AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(painless): depth + source-length guards (unauth remote stack-overflow)
xerj-orgdf1c005ec777
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex b738724f4..f57fab7a6 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -3679,12 +3679,70 @@ fn default_size() -> usize {}/// Build a `SearchRequest` from the ES body, forwarding all relevant options.+/// Walk a request JSON subtree for inline Painless script sources — values+/// under a `script` key or any `*_script` key (e.g. scripted_metric's+/// `map_script`) — and validate each against the engine's script resource+/// limits. Returns the first limit-violation message, if any.+///+/// Only genuine limit violations (oversized source, or nesting depth beyond+/// the parser's cap) are reported. Ordinary scripts — including ones outside+/// our Painless subset — return `None` so they keep degrading gracefully at+/// score time rather than turning into spurious 400s (which would break+/// otherwise-passing requests and the ES-compat gate). This is the request-time+/// guard that converts an adversarial deeply-nested `script_score` from a+/// process-aborting stack overflow into a clean 400.+fn find_script_limit_violation(v: &Value) -> Option<String> {+ match v {+ Value::Object(map) => {+ for (k, val) in map {+ if k == "script" || k.ends_with("_script") {+ let src = match val {+ Value::String(s) => Some(s.as_str()),+ Value::Object(o) => o.get("source").and_then(Value::as_str),+ _ => None,+ };+ if let Some(src) = src {+ if let Err(e) = xerj_engine::painless::check_script_limits(src) {+ return Some(e);+ }+ }+ }+ if let Some(hit) = find_script_limit_violation(val) {+ return Some(hit);+ }+ }+ None+ }+ Value::Array(arr) => arr.iter().find_map(find_script_limit_violation),+ _ => None,+ }+}+fn build_search_request(body: &EsSearchBody,aggs_value: Option<Value>,) -> Result<xerj_query::ast::SearchRequest, xerj_common::XerjError> {use xerj_query::ast::SourceFilter;+ // Reject scripts that exceed the engine's resource limits up front with a+ // 400, before they reach the per-document scoring path (where a deeply+ // nested script would otherwise overflow the stack and abort the server).+ for v in [+ body.query.as_ref(),+ body.rescore.as_ref(),+ body.sort.as_ref(),+ body.script_fields.as_ref(),+ body.runtime_mappings.as_ref(),+ aggs_value.as_ref(),+ ]+ .into_iter()+ .flatten()+ {+ if let Some(msg) = find_script_limit_violation(v) {+ return Err(xerj_common::XerjError::invalid_query(msg));+ }+ }+let query_val = body.query.clone().unwrap_or(json!({ "match_all": {} }));// Build a JSON blob for parse_request (handles the query + from/size).diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex 4ab575c75..38e891c12 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -10555,11 +10555,34 @@ fn lex_script(src: &str) -> Result<Vec<PlTok>, String> {Ok(out)}+/// Maximum recursive nesting depth accepted by the scripted-metric+/// recursive-descent parser. Mirrors `painless::MAX_PARSE_DEPTH`: an+/// unauthenticated deeply-nested script would otherwise overflow the worker+/// stack and abort the process.+const MAX_SCRIPT_PARSE_DEPTH: usize = 100;+/// Maximum recursive AST-evaluation depth for scripted-metric scripts.+const MAX_SCRIPT_EVAL_DEPTH: usize = 500;+/// Maximum accepted scripted-metric script source length (bytes).+const MAX_SCRIPT_SRC_LEN: usize = 64 * 1024;+struct PlParser {toks: Vec<PlTok>,pos: usize,+ /// Current recursive-descent nesting depth.+ depth: usize,}impl PlParser {+ fn descend(&mut self) -> Result<(), String> {+ self.depth += 1;+ if self.depth > MAX_SCRIPT_PARSE_DEPTH {+ Err("compile error: script exceeds maximum nesting depth".to_string())+ } else {+ Ok(())+ }+ }+ fn ascend(&mut self) {+ self.depth = self.depth.saturating_sub(1);+ }fn peek(&self) -> Option<&PlTok> {self.toks.get(self.pos)}@@ -10592,6 +10615,13 @@ impl PlParser {Ok(out)}fn parse_stmt(&mut self) -> Result<PlStmt, String> {+ // Depth-guard statement nesting (for / if / block bodies).+ self.descend()?;+ let out = self.parse_stmt_inner();+ self.ascend();+ out+ }+ fn parse_stmt_inner(&mut self) -> Result<PlStmt, String> {if let Some(PlTok::Ident(kw)) = self.peek().cloned() {if kw == "return" {self.bump();@@ -10774,6 +10804,15 @@ impl PlParser {self.parse_ternary()}fn parse_ternary(&mut self) -> Result<PlExpr, String> {+ // parse_expr → parse_ternary funnels every expression re-entry+ // (parens, array elems, ctor/call args, subscripts, ternary arms), so+ // guarding it bounds the whole expression-grammar recursion.+ self.descend()?;+ let out = self.parse_ternary_inner();+ self.ascend();+ out+ }+ fn parse_ternary_inner(&mut self) -> Result<PlExpr, String> {let cond = self.parse_binary(1)?;if matches!(self.peek(), Some(PlTok::Question)) {self.bump();@@ -10812,6 +10851,14 @@ impl PlParser {}}fn parse_unary(&mut self) -> Result<PlExpr, String> {+ // Guard unary chains (`----x`, `!!!x`) which recurse through+ // parse_unary directly and bypass the parse_ternary guard.+ self.descend()?;+ let out = self.parse_unary_inner();+ self.ascend();+ out+ }+ fn parse_unary_inner(&mut self) -> Result<PlExpr, String> {if let Some(PlTok::Op("-")) = self.peek() {self.bump();let e = self.parse_unary()?;@@ -10977,11 +11024,52 @@ fn binop_prec(op: &str) -> u8 {}fn parse_script(src: &str) -> Result<Vec<PlStmt>, String> {+ if src.len() > MAX_SCRIPT_SRC_LEN {+ return Err(format!(+ "compile error: script source is {} bytes, exceeds the {}-byte limit",+ src.len(),+ MAX_SCRIPT_SRC_LEN+ ));+ }let toks = lex_script(src)?;- let mut p = PlParser { toks, pos: 0 };+ let mut p = PlParser {+ toks,+ pos: 0,+ depth: 0,+ };p.parse_stmts(false)}+thread_local! {+ /// Per-thread AST-evaluation recursion depth for scripted-metric scripts.+ /// `ScriptCtx` is threaded by `&mut`, so a `Cell` field can't be borrowed+ /// alongside the recursive `&mut ctx` calls; a thread-local sidesteps the+ /// borrow conflict (eval runs synchronously on one thread per script).+ static SCRIPT_EVAL_DEPTH: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };+}++/// RAII guard bounding scripted-metric AST-evaluation recursion depth so a+/// deep (or long flat) AST returns an error instead of overflowing the stack.+struct ScriptEvalDepthGuard;+impl ScriptEvalDepthGuard {+ fn enter() -> Result<Self, String> {+ SCRIPT_EVAL_DEPTH.with(|d| {+ let cur = d.get();+ if cur >= MAX_SCRIPT_EVAL_DEPTH {+ Err("script evaluation exceeded maximum depth".to_string())+ } else {+ d.set(cur + 1);+ Ok(ScriptEvalDepthGuard)+ }+ })+ }+}+impl Drop for ScriptEvalDepthGuard {+ fn drop(&mut self) {+ SCRIPT_EVAL_DEPTH.with(|d| d.set(d.get().saturating_sub(1)));+ }+}+fn exec_stmts(stmts: &[PlStmt], ctx: &mut ScriptCtx) -> Result<Option<Value>, String> {for s in stmts {if let Some(v) = exec_stmt(s, ctx)? {@@ -10992,6 +11080,7 @@ fn exec_stmts(stmts: &[PlStmt], ctx: &mut ScriptCtx) -> Result<Option<Value>, St}fn exec_stmt(stmt: &PlStmt, ctx: &mut ScriptCtx) -> Result<Option<Value>, String> {+ let _guard = ScriptEvalDepthGuard::enter()?;match stmt {PlStmt::Return(e) => Ok(Some(eval_expr(e, ctx)?)),PlStmt::Decl(name, e) => {@@ -11062,6 +11151,7 @@ fn is_truthy(v: &Value) -> bool {/// Resolve `doc.field.value` or `doc["field"].value` by accessing the/// current doc's source; return the first value for multi-valued fields.fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {+ let _guard = ScriptEvalDepthGuard::enter()?;match e {PlExpr::Num(n) => Ok(json!(n)),PlExpr::Str(s) => Ok(Value::String(s.clone())),@@ -11597,6 +11687,31 @@ mod tests {use super::*;use serde_json::json;+ // Regression: a deeply nested scripted-metric script must not overflow the+ // recursive-descent parser (which would abort the whole server). The guard+ // returns an Err instead. If it regresses, this test process itself+ // stack-overflows and aborts, so it can never silently pass.+ #[test]+ fn scripted_metric_deep_nesting_does_not_overflow() {+ let src = format!("return {}1.0{};", "(".repeat(5000), ")".repeat(5000));+ let r = parse_script(&src);+ assert!(r.is_err(), "expected depth-limit error, got {:?}", r);+ }++ #[test]+ fn scripted_metric_oversized_source_rejected() {+ let src = format!("return {};", "1.0".repeat(MAX_SCRIPT_SRC_LEN));+ let r = parse_script(&src);+ assert!(r.is_err(), "expected length-limit error, got {:?}", r);+ }++ #[test]+ fn scripted_metric_normal_script_still_parses() {+ // A realistic scripted-metric map body still parses fine.+ let r = parse_script("state.sum = 0.0; for (t in doc['v']) { state.sum += t; }");+ assert!(r.is_ok(), "normal script should parse: {:?}", r);+ }+fn docs() -> Vec<Value> {vec![json!({"tag": "rust", "score": 10.0, "@timestamp": "2024-01-01T00:00:00Z"}),diff --git a/engine/crates/xerj-engine/src/painless.rs b/engine/crates/xerj-engine/src/painless.rsindex f724a14d9..988b78017 100644--- a/engine/crates/xerj-engine/src/painless.rs+++ b/engine/crates/xerj-engine/src/painless.rs@@ -92,6 +92,9 @@ pub struct PainlessCtx<'a> {/// non-runtime contexts (script_score, rescore, etc.) where emit()/// is not used.pub emits: std::cell::RefCell<Vec<PainlessValue>>,+ /// Current AST-evaluation recursion depth. Guards `eval_expr`/`exec_stmt`+ /// against stack overflow on a deeply-nested (or long flat) AST.+ eval_depth: std::cell::Cell<usize>,}impl<'a> PainlessCtx<'a> {@@ -101,6 +104,7 @@ impl<'a> PainlessCtx<'a> {params,score,emits: std::cell::RefCell::new(Vec::new()),+ eval_depth: std::cell::Cell::new(0),}}pub fn take_emits(&self) -> Vec<PainlessValue> {@@ -341,16 +345,65 @@ enum Stmt {Block(Vec<Stmt>),}+// ── Resource limits ──────────────────────────────────────────────────────────++/// Maximum recursive nesting depth accepted by the recursive-descent parser+/// (nested parens, unary chains, ternaries, nested blocks). An unauthenticated+/// caller could otherwise submit a ~3 KB script of nested parens whose parse+/// recursion overflows the (2 MB) worker-thread stack and aborts the entire+/// process. ~100 is far below the empirically-observed overflow (~3000) yet+/// comfortably above any legitimate hand-written or generated script.+pub(crate) const MAX_PARSE_DEPTH: usize = 100;++/// Maximum recursive AST-evaluation depth. Bounds `eval_expr`/`exec_stmt`+/// recursion so a deep AST (including flat left-associative operator chains+/// like `1+1+1+…` which the parser builds with a loop, not recursion, and so+/// are NOT limited by [`MAX_PARSE_DEPTH`]) cannot overflow the stack at score+/// time.+pub(crate) const MAX_EVAL_DEPTH: usize = 500;++/// Maximum accepted script source length in bytes. Matches Elasticsearch's+/// default `script.max_size_in_bytes` (64 KiB) and bounds the size of any AST+/// we build (and later drop) from a single request.+pub(crate) const MAX_SCRIPT_LEN: usize = 64 * 1024;++/// Sentinel error returned when [`MAX_PARSE_DEPTH`] is exceeded. Callers+/// (`check_script_limits`) match on it to distinguish "too complex" (a 400)+/// from ordinary syntax errors (which degrade gracefully at runtime).+pub(crate) const TOO_DEEP_MSG: &str = "compile error: script exceeds maximum nesting depth";+// ── Parser ───────────────────────────────────────────────────────────────────struct Parser<'a> {toks: &'a [Tok],pos: usize,+ /// Current recursive-descent nesting depth (guards against stack overflow+ /// on adversarial deeply-nested input).+ depth: usize,}impl<'a> Parser<'a> {fn new(toks: &'a [Tok]) -> Self {- Self { toks, pos: 0 }+ Self {+ toks,+ pos: 0,+ depth: 0,+ }+ }+ /// Enter one recursion level, failing (instead of overflowing the stack)+ /// once [`MAX_PARSE_DEPTH`] is exceeded. Paired with [`Parser::ascend`] on+ /// the success path; on the error path the whole parse is abandoned so the+ /// unbalanced counter is irrelevant.+ fn descend(&mut self) -> Result<(), String> {+ self.depth += 1;+ if self.depth > MAX_PARSE_DEPTH {+ Err(TOO_DEEP_MSG.to_string())+ } else {+ Ok(())+ }+ }+ fn ascend(&mut self) {+ self.depth = self.depth.saturating_sub(1);}fn peek(&self) -> Option<&Tok> {self.toks.get(self.pos)@@ -394,6 +447,13 @@ impl<'a> Parser<'a> {Ok(out)}fn parse_stmt(&mut self) -> Result<Stmt, String> {+ // Depth-guard statement nesting (blocks / if / for bodies).+ self.descend()?;+ let out = self.parse_stmt_inner();+ self.ascend();+ out+ }+ fn parse_stmt_inner(&mut self) -> Result<Stmt, String> {// `if (...) { ... } else { ... }`if self.match_keyword("if") {self.expect_punct('(')?;@@ -464,6 +524,15 @@ impl<'a> Parser<'a> {self.parse_assign()}fn parse_assign(&mut self) -> Result<Expr, String> {+ // Every expression re-entry (parens, index keys, call args, ternary+ // arms, assignment RHS) funnels through parse_assign, so guarding it+ // here bounds the whole expression-grammar recursion by nesting depth.+ self.descend()?;+ let out = self.parse_assign_inner();+ self.ascend();+ out+ }+ fn parse_assign_inner(&mut self) -> Result<Expr, String> {let lhs = self.parse_ternary()?;if self.match_punct('=') {// Disambiguate from `==` already consumed by parse_compare.@@ -573,6 +642,14 @@ impl<'a> Parser<'a> {Ok(lhs)}fn parse_unary(&mut self) -> Result<Expr, String> {+ // Guard unary chains (`----x`, `!!!x`), which recurse through+ // parse_unary itself and so bypass the parse_assign guard.+ self.descend()?;+ let out = self.parse_unary_inner();+ self.ascend();+ out+ }+ fn parse_unary_inner(&mut self) -> Result<Expr, String> {if self.match_punct('-') {let e = self.parse_unary()?;return Ok(Expr::Unary("-".into(), Box::new(e)));@@ -662,7 +739,64 @@ impl<'a> Parser<'a> {// ── Evaluation ───────────────────────────────────────────────────────────────+/// RAII guard that bounds AST-evaluation recursion depth. Incremented on+/// entry to `eval_expr`/`exec_stmt`, decremented on scope exit (including the+/// `?` early-return paths), so a pathological AST returns an error instead of+/// overflowing the worker-thread stack.+struct EvalDepthGuard<'a>(&'a std::cell::Cell<usize>);+impl<'a> EvalDepthGuard<'a> {+ fn enter(cell: &'a std::cell::Cell<usize>) -> Result<Self, String> {+ let d = cell.get();+ if d >= MAX_EVAL_DEPTH {+ return Err("script evaluation exceeded maximum depth".to_string());+ }+ cell.set(d + 1);+ Ok(EvalDepthGuard(cell))+ }+}+impl Drop for EvalDepthGuard<'_> {+ fn drop(&mut self) {+ self.0.set(self.0.get().saturating_sub(1));+ }+}++/// Validate a script against the parser/length resource limits WITHOUT running+/// it, so the request layer can reject an abusive script with a 400 up front.+///+/// Returns `Err` **only** for limit violations (source too long, or nesting+/// depth beyond [`MAX_PARSE_DEPTH`]). Ordinary syntax errors — including+/// constructs outside our Painless subset — return `Ok(())` so they keep+/// degrading gracefully at runtime (unchanged behavior), rather than becoming+/// spurious 400s that would break otherwise-passing requests.+pub fn check_script_limits(src: &str) -> Result<(), String> {+ if src.len() > MAX_SCRIPT_LEN {+ return Err(format!(+ "compile error: script source is {} bytes, exceeds the {}-byte limit",+ src.len(),+ MAX_SCRIPT_LEN+ ));+ }+ // The tokenizer is non-recursive; a tokenizer error is a plain syntax+ // problem, so let the runtime path handle it (don't 400).+ let toks = match tokenize(src) {+ Ok(t) => t,+ Err(_) => return Ok(()),+ };+ let mut p = Parser::new(&toks);+ match p.parse_program() {+ Err(e) if e == TOO_DEEP_MSG => Err(e),+ _ => Ok(()),+ }+}+pub fn eval_painless(src: &str, ctx: &PainlessCtx) -> Result<PainlessValue, String> {+ if src.len() > MAX_SCRIPT_LEN {+ return Err(format!(+ "script source is {} bytes, exceeds the {}-byte limit",+ src.len(),+ MAX_SCRIPT_LEN+ ));+ }let toks = tokenize(src)?;let mut p = Parser::new(&toks);let stmts = p.parse_program()?;@@ -693,6 +827,7 @@ fn exec_stmt(ctx: &PainlessCtx,env: &mut HashMap<String, PainlessValue>,) -> Result<ExecOutcome, String> {+ let _guard = EvalDepthGuard::enter(&ctx.eval_depth)?;match s {Stmt::Return(opt) => {let v = match opt {@@ -730,6 +865,7 @@ fn eval_expr(ctx: &PainlessCtx,env: &mut HashMap<String, PainlessValue>,) -> Result<PainlessValue, String> {+ let _guard = EvalDepthGuard::enter(&ctx.eval_depth)?;match e {Expr::Number(n) => Ok(PainlessValue::Number(*n)),Expr::String(s) => Ok(PainlessValue::String(s.clone())),@@ -1288,4 +1424,100 @@ mod tests {let v = eval_painless("Math.max(1.5, 2.5)", &ctx(&doc, ¶ms, 0.0)).unwrap();assert!((v.as_f64().unwrap() - 2.5).abs() < 1e-9);}++ // ── Resource-limit / stack-overflow guards ───────────────────────────────+ // Regression tests for the unauthenticated remote crash: a deeply nested+ // script used to overflow the parser's (or evaluator's) recursion and abort+ // the whole process. These MUST return an `Err` — if the guard regresses,+ // the test process itself stack-overflows and aborts (a hard failure), so+ // the test can never silently pass.++ #[test]+ fn deeply_nested_parens_do_not_overflow_parser() {+ let doc = json!({});+ let params = json!({});+ // ~5000 nested parens — well beyond the ~3000 that overflowed the+ // real server before the guard.+ let src = format!("{}1.0{}", "(".repeat(5000), ")".repeat(5000));+ let r = eval_painless(&src, &ctx(&doc, ¶ms, 0.0));+ assert!(r.is_err(), "expected depth-limit error, got {:?}", r);+ assert_eq!(r.unwrap_err(), TOO_DEEP_MSG);+ }++ #[test]+ fn deeply_nested_unary_do_not_overflow_parser() {+ let doc = json!({});+ let params = json!({});+ // Unary chains recurse through parse_unary directly. Use `!` (logical+ // NOT): unlike `-`, consecutive `!` are NOT collapsed into a multi-char+ // token by the lexer, so this genuinely drives deep unary recursion.+ let src = format!("{}true", "!".repeat(5000));+ let r = eval_painless(&src, &ctx(&doc, ¶ms, 0.0));+ assert!(r.is_err(), "expected depth-limit error, got {:?}", r);+ assert_eq!(r.unwrap_err(), TOO_DEEP_MSG);+ }++ #[test]+ fn long_flat_binary_chain_does_not_overflow_evaluator() {+ let doc = json!({});+ let params = json!({});+ // A flat `1+1+1+…` chain is parsed with a LOOP (not deep recursion),+ // so it passes the parser but builds a deep left-leaning AST that the+ // evaluator would recurse over. The eval-depth guard must catch it.+ let src = format!("1{}", "+1".repeat(5000));+ let r = eval_painless(&src, &ctx(&doc, ¶ms, 0.0));+ assert!(+ r.is_err(),+ "expected eval-depth error on deep AST, got {:?}",+ r+ );+ }++ #[test]+ fn oversized_source_rejected() {+ let doc = json!({});+ let params = json!({});+ let src = format!("{} + 1.0", "1.0".repeat(MAX_SCRIPT_LEN));+ let r = eval_painless(&src, &ctx(&doc, ¶ms, 0.0));+ assert!(r.is_err(), "expected length-limit error, got {:?}", r);+ }++ #[test]+ fn check_script_limits_flags_nesting_but_ignores_plain_syntax_errors() {+ // Nesting past the cap → reported (becomes a 400 at the request layer).+ let deep = format!("{}1.0{}", "(".repeat(5000), ")".repeat(5000));+ assert!(check_script_limits(&deep).is_err());++ // Oversized → reported.+ let big = "1.0".repeat(MAX_SCRIPT_LEN);+ assert!(check_script_limits(&big).is_err());++ // Deep unary chain past the cap → reported (use `!`; see note above).+ let deep_unary = format!("{}true", "!".repeat(5000));+ assert!(+ check_script_limits(&deep_unary).is_err(),+ "deep unary should be flagged by the parse-depth guard"+ );++ // A normal, valid script → OK.+ assert!(check_script_limits("doc['x'].value * 2 + _score").is_ok());++ // An unsupported-but-shallow script (syntax our subset rejects) must+ // NOT be flagged — it should keep degrading gracefully at runtime, not+ // turn into a spurious 400.+ assert!(check_script_limits("some garbage )(").is_ok());+ }++ #[test]+ fn normal_script_still_evaluates_after_guards() {+ let doc = json!({"x": 4});+ let params = json!({"m": 3});+ // Moderate nesting well within limits still works.+ let v = eval_painless(+ "((doc['x'].value + 1) * params.m)",+ &ctx(&doc, ¶ms, 0.0),+ )+ .unwrap();+ assert!((v.as_f64().unwrap() - 15.0).abs() < 1e-9);+ }}