AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(query): add script query support (#87)
Vinz216879ec079b896e
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex d330d0621..f0dd79203 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -13267,6 +13267,93 @@ mod malformed_bulk_route_tests {}}+#[cfg(test)]+mod script_query_end_to_end_tests {+ //! Regression coverage for a real bug: the `script` query's unit tests+ //! (in xerj-engine's index.rs) all exercised `doc_matches_query`+ //! directly, never a real `POST /{index}/_search` — so they passed+ //! while the query silently returned zero hits for every real search,+ //! because `is_doc_scan_query` never routed `QueryNode::Script` into+ //! doc-scan. This test goes through the actual HTTP route so a+ //! regression here fails loudly.+ use super::*;+ use axum::{+ body::{to_bytes, Body},+ http::Request,+ };+ use tower::ServiceExt;+ use xerj_common::{+ config::{Config, WalSync},+ metrics::Metrics,+ };+ use xerj_engine::Engine;++ fn test_state() -> AppState {+ let dir = tempfile::tempdir().expect("tempdir").keep();+ let mut config = Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = WalSync::Async;+ let metrics = Metrics::new().expect("metrics");+ let engine = Engine::new(config.clone()).expect("engine");+ AppState::new(config, engine, metrics)+ }++ #[tokio::test]+ async fn script_query_matches_an_indexed_document_via_real_search() {+ let router = crate::router::build_es_compat_router(test_state());++ let index_response = router+ .clone()+ .oneshot(+ Request::post("/script-query-e2e/_doc/1")+ .header("content-type", "application/json")+ .body(Body::from(r#"{"x": 10}"#))+ .expect("request"),+ )+ .await+ .expect("index response");+ assert!(+ index_response.status().is_success(),+ "indexing failed: {:?}",+ index_response.status()+ );++ let search_body = serde_json::to_vec(&json!({+ "query": {+ "script": {+ "script": {"source": "doc['x'].value > 5"}+ }+ }+ }))+ .expect("serialize query");++ let search_response = router+ .oneshot(+ Request::post("/script-query-e2e/_search")+ .header("content-type", "application/json")+ .body(Body::from(search_body))+ .expect("request"),+ )+ .await+ .expect("search response");+ assert!(+ search_response.status().is_success(),+ "search failed: {:?}",+ search_response.status()+ );++ let bytes = to_bytes(search_response.into_body(), usize::MAX)+ .await+ .expect("response bytes");+ let response: Value = serde_json::from_slice(&bytes).expect("JSON response");+ let total = response["hits"]["total"]["value"].as_u64().unwrap_or(0);+ assert_eq!(+ total, 1,+ "script query should match the freshly indexed document via a real search, got: {response}"+ );+ }+}+// ─────────────────────────────────────────────────────────────────────────────// Schema conversion helpers// ─────────────────────────────────────────────────────────────────────────────diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 43f55f4fe..12e8f8e15 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -19518,7 +19518,7 @@ impl Index {// a malformed section bails early and leaves the vec incomplete.mut offsets_out: Option<&mut Vec<(u32, u32)>>,// Cooperative timeout: when `Some`, the walk polls the wall clock- // every 4 096 docs and aborts (setting `*timed_out`) once past the+ // every 256 docs and aborts (setting `*timed_out`) once past the// deadline. One giant merged segment is otherwise a multi-second// uninterruptible unit — the between-segment deadline check in// `search_inner` can't bound it.@@ -19639,9 +19639,28 @@ impl Index {doc_pos += 1;*dbg_walked += 1;- // Cooperative timeout poll — every 4 096 docs (~1 µs of clock- // reads per million docs; the parse below costs ~3 µs/doc).- if doc_pos & 0xFFF == 0 {+ // Cooperative timeout poll — every 256 docs (~16 µs of clock+ // reads per million docs, against a per-doc cost that is at+ // minimum the ~3 µs JSON parse below; the read is noise).+ //+ // The interval used to be 4 096, sized for that ~3 µs/doc floor.+ // A `script` query breaks that assumption: even with the Painless+ // AST cached (see `painless::compile_script_cached`), *executing*+ // a script dominates the per-doc cost. Measured on a release+ // build: a benign 64 KiB script runs 385 µs/doc, so 4 096 docs is+ // a ~1.6 s stretch in which a `timeout` cannot take effect, and+ // 256 brings that to ~98 ms.+ //+ // 256 is an improvement, not a bound. A deliberately adversarial+ // script of legal size — flat, no closure calls, tripping none of+ // MAX_CALL_DEPTH / MAX_CALL_COUNT / MAX_EVAL_DEPTH /+ // MAX_PARSE_DEPTH / MAX_PAINLESS_STRING_LEN — was measured at+ // 1.27 s/doc, which is ~325 s per poll interval. Nothing here+ // bounds that, because there is no per-evaluation CPU budget; the+ // poll interval only decides how often an already-cheap document+ // boundary is checked. Closing that needs a real time or+ // instruction budget inside `eval_painless`, tracked separately.+ if doc_pos & 0xFF == 0 {if let Some(dl) = deadline {if std::time::Instant::now() >= dl {*timed_out = true;@@ -23817,6 +23836,7 @@ fn is_doc_scan_query(q: &QueryNode) -> bool {| QueryNode::Boosted { .. }| QueryNode::Intervals { .. }| QueryNode::Percolate { .. }+ | QueryNode::Script { .. })}@@ -24010,7 +24030,8 @@ fn query_needs_id_injection(q: &QueryNode) -> bool {| QueryNode::GeoShape { .. }| QueryNode::SpanTerm { .. }| QueryNode::MoreLikeThis { .. }- | QueryNode::Percolate { .. } => false,+ | QueryNode::Percolate { .. }+ | QueryNode::Script { .. } => false,QueryNode::Bool {must,should,@@ -24249,6 +24270,25 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {.unwrap_or(false)}+ QueryNode::Script {+ source: src,+ params,+ } => {+ let empty = Value::Object(serde_json::Map::new());+ let p = params.as_ref().unwrap_or(&empty);+ let ctx = crate::painless::PainlessCtx::new(source, p, 0.0);+ // ES requires a `script` query's script to return a boolean;+ // a script that errors, or returns anything else (a number, a+ // string, ...), never matches — real ES throws a script-cast+ // exception in that case, which drops the doc in filter+ // contexts, so this fails closed the same way rather than+ // truthy-coercing a non-boolean result.+ matches!(+ crate::painless::eval_painless(src, &ctx),+ Ok(crate::painless::PainlessValue::Bool(true))+ )+ }+QueryNode::Ids { values } => {// _id is passed via the source's "_id" metadata field when present,// but for memtable doc-scan calls the source may not contain it.@@ -31267,6 +31307,112 @@ mod percolate_tests {}}+#[cfg(test)]+mod script_query_tests {+ //! A standalone `script` query is a boolean filter (no scoring), used+ //! e.g. by OpenSearch's UBI sample dashboards to filter on a computed+ //! field via a Painless boolean lambda. Distinct from `script_score`.+ use super::*;+ use serde_json::json;+ use xerj_query::ast::QueryNode;++ #[test]+ fn script_query_matches_when_truthy() {+ let doc = json!({"x": 10});+ let q = QueryNode::Script {+ source: "doc['x'].value > params.min".to_string(),+ params: Some(json!({"min": 5})),+ };+ assert!(doc_matches_query(&q, &doc));+ }++ #[test]+ fn script_query_no_match_when_falsy() {+ let doc = json!({"x": 1});+ let q = QueryNode::Script {+ source: "doc['x'].value > params.min".to_string(),+ params: Some(json!({"min": 5})),+ };+ assert!(!doc_matches_query(&q, &doc));+ }++ #[test]+ fn script_query_fails_closed_on_script_error() {+ let doc = json!({"x": 1});+ let q = QueryNode::Script {+ source: "this is not valid painless".to_string(),+ params: None,+ };+ assert!(!doc_matches_query(&q, &doc));+ }++ #[test]+ fn script_query_fails_closed_on_non_boolean_result() {+ // A script that evaluates to a number (not a boolean) must not match+ // — ES itself requires `script` queries to return a boolean.+ let doc = json!({"x": 1});+ let q = QueryNode::Script {+ source: "1 + 1".to_string(),+ params: None,+ };+ assert!(!doc_matches_query(&q, &doc));+ }++ #[test]+ fn script_query_null_guard_filters_rather_than_dropping_everything() {+ // The standard shape for a script filter over a field that isn't on+ // every document, written the way Elasticsearch documents it. If the+ // guard errors instead of yielding a boolean, the script is+ // unrunnable and EVERY document is dropped — including the ones that+ // should match. This is what a `script` query looks like in+ // practice, so it is checked at the query surface and not only in+ // the Painless unit tests.+ let q = QueryNode::Script {+ source: "doc['x'].size() == 0 ? false : doc['x'].value > params.min".to_string(),+ params: Some(json!({"min": 5})),+ };+ assert!(+ doc_matches_query(&q, &json!({"x": 10})),+ "guarded script must still match the document it selects for"+ );+ assert!(!doc_matches_query(&q, &json!({"x": 1})));+ assert!(+ !doc_matches_query(&q, &json!({"other": 1})),+ "a document without the field must be filtered out, not matched"+ );+ }++ #[test]+ fn script_query_with_a_typoed_field_matches_nothing() {+ // The bug the fail-closed `.value` exists for: reading a missing+ // field as 0 made `doc['typo'].value > -1` true for every document,+ // turning a filter into a no-op. It must match NOTHING instead —+ // including documents that do have the correctly-spelled field.+ let q = QueryNode::Script {+ source: "doc['coutn'].value > -1".to_string(),+ params: None,+ };+ for doc in [+ json!({"count": 5}),+ json!({"count": 0}),+ json!({}),+ json!({"coutn": null}),+ ] {+ assert!(+ !doc_matches_query(&q, &doc),+ "a typo'd field name must not match {doc}"+ );+ }+ // And the correctly-spelled query still works, so this is a spelling+ // check and not a blanket refusal.+ let ok = QueryNode::Script {+ source: "doc['count'].value > -1".to_string(),+ params: None,+ };+ assert!(doc_matches_query(&ok, &json!({"count": 5})));+ }+}+#[cfg(test)]mod highlight_multibyte_tests {//! Regression tests for the multibyte-highlight SIGABRT: a term whosediff --git a/engine/crates/xerj-engine/src/painless.rs b/engine/crates/xerj-engine/src/painless.rsindex f91a6572c..37835aee5 100644--- a/engine/crates/xerj-engine/src/painless.rs+++ b/engine/crates/xerj-engine/src/painless.rs@@ -45,7 +45,9 @@//! should fall back to a no-op score on script error.use serde_json::Value;+use std::cell::RefCell;use std::collections::HashMap;+use std::rc::Rc;#[derive(Debug, Clone)]pub enum PainlessValue {@@ -458,6 +460,16 @@ pub(crate) const MAX_EVAL_DEPTH: usize = 500;/// we build (and later drop) from a single request.pub(crate) const MAX_SCRIPT_LEN: usize = 64 * 1024;+/// Safety ceiling on any single string value produced during evaluation.+/// String concatenation (`+`) is the only operation that can grow a value+/// beyond the size of its parts, so a handful of flat (non-nested, non-deep)+/// `s = s + s;` statements — none of which trip [`MAX_EVAL_DEPTH`] or+/// [`MAX_PARSE_DEPTH`], since they're neither deeply nested nor deeply+/// recursive — doubles a string exponentially per statement. Without this+/// cap a script well under [`MAX_SCRIPT_LEN`] can exhaust available memory+/// before any other limit fires.+const MAX_PAINLESS_STRING_LEN: usize = 1024 * 1024;+/// Sentinel error returned when [`MAX_PARSE_DEPTH`] is exceeded. Callers/// (`check_script_limits`) match on it to distinguish "too complex" (a 400)/// from ordinary syntax errors (which degrade gracefully at runtime).@@ -1167,6 +1179,110 @@ impl Drop for CallGuard<'_> {}}+// ── Compiled-script cache ────────────────────────────────────────────────────+//+// A script is evaluated once per *document*. Tokenizing and parsing it per+// document made the per-doc cost scale with the script's SIZE rather than+// with its complexity: a legal 64 KiB script (see [`MAX_SCRIPT_LEN`]) cost+// ~2 ms/doc, essentially all of it re-parsing an AST identical to the one+// built for the previous document. That is also what made the doc-scan's+// cooperative timeout ineffective — `scan_stored_section_into` polls its+// deadline every N docs on the assumption that per-doc work is a few+// microseconds, so a 2 ms/doc script stretched the uninterruptible quantum+// from milliseconds to seconds and a `timeout` could not cut the scan short.+//+// Scripts are pure source text, so the AST is a pure function of the source+// and can simply be memoised. The cache MUST be bounded: the key is+// attacker-supplied source, so an unbounded map is itself a memory-exhaustion+// vector — a caller need only issue queries with unique scripts.++/// Maximum number of distinct compiled scripts retained at once.+const MAX_SCRIPT_CACHE_ENTRIES: usize = 128;++/// Maximum total *source* bytes retained across cached entries. Bounds the+/// cache independently of the entry count, since [`MAX_SCRIPT_LEN`] allows a+/// single 64 KiB script and 128 of those would be 8 MiB of source (and a+/// considerably larger multiple of that in AST nodes).+const MAX_SCRIPT_CACHE_SRC_BYTES: usize = 512 * 1024;++/// A parse result shared between evaluations. Parse *failures* are cached+/// too — a syntactically invalid script is also evaluated once per document,+/// so leaving errors uncached would leave the same per-doc parse cost (and+/// the same unbounded quantum) reachable through a malformed script.+///+/// `Rc`, not `Arc`, because [`Stmt::FnDecl`]/[`Expr::Lambda`] bodies are+/// themselves `Rc`-shared — deliberately non-atomic, since a closure literal+/// is cloned on every invocation. The cache is therefore per-thread (see+/// [`SCRIPT_CACHE`]) rather than a shared static.+type CompiledScript = Rc<Result<Vec<Stmt>, String>>;++#[derive(Default)]+struct ScriptCache {+ entries: HashMap<String, CompiledScript>,+ /// Sum of `entries`' key lengths; maintained incrementally.+ src_bytes: usize,+}++impl ScriptCache {+ fn insert(&mut self, src: &str, compiled: CompiledScript) {+ // Never cacheable on its own — would evict everything and still not fit.+ if src.len() > MAX_SCRIPT_CACHE_SRC_BYTES || self.entries.contains_key(src) {+ return;+ }+ // Evict (in arbitrary hash order — the workload is "one script for a+ // whole query", so recency bookkeeping buys nothing worth its cost)+ // until the newcomer fits under BOTH bounds.+ while self.entries.len() >= MAX_SCRIPT_CACHE_ENTRIES+ || self.src_bytes.saturating_add(src.len()) > MAX_SCRIPT_CACHE_SRC_BYTES+ {+ let Some(victim) = self.entries.keys().next().cloned() else {+ // Empty and still over budget is impossible given the guard+ // above, but never spin.+ return;+ };+ self.entries.remove(&victim);+ self.src_bytes = self.src_bytes.saturating_sub(victim.len());+ }+ self.src_bytes = self.src_bytes.saturating_add(src.len());+ self.entries.insert(src.to_string(), compiled);+ }+}++thread_local! {+ /// One cache per thread. A compiled AST contains `Rc`s and so cannot+ /// cross threads; a `Mutex<Arc<...>>` would not help, since the `Rc`s+ /// *inside* the AST are the non-`Sync` part.+ ///+ /// The bound is therefore per-thread, and the total is bounded by+ /// (search worker threads) × [`MAX_SCRIPT_CACHE_SRC_BYTES`] — still a+ /// fixed ceiling, because the worker pool is sized from the core count,+ /// not from the request rate. Per-thread caching also removes the lock+ /// from a path taken once per document, and each worker scanning the+ /// same segment converges on the same one entry.+ static SCRIPT_CACHE: RefCell<ScriptCache> = RefCell::new(ScriptCache::default());+}++fn compile_script(src: &str) -> Result<Vec<Stmt>, String> {+ let toks = tokenize(src)?;+ let mut p = Parser::new(&toks);+ p.parse_program()+}++/// Tokenize + parse `src`, reusing a previously compiled AST when possible.+fn compile_script_cached(src: &str) -> CompiledScript {+ SCRIPT_CACHE.with(|cache| {+ // Borrow, look up, release — the insert below needs a fresh mutable+ // borrow, and `compile_script` must not run while either is held.+ let hit = cache.borrow().entries.get(src).map(Rc::clone);+ if let Some(hit) = hit {+ return hit;+ }+ let compiled: CompiledScript = Rc::new(compile_script(src));+ cache.borrow_mut().insert(src, Rc::clone(&compiled));+ compiled+ })+}+/// Validate a script against the parser/length resource limits WITHOUT running/// it, so the request layer can reject an abusive script with a 400 up front.///@@ -1183,15 +1299,13 @@ pub fn check_script_limits(src: &str) -> Result<(), String> {MAX_SCRIPT_LEN));}- // The tokenizer is non-recursive; a tokenizer error is a plain syntax- // problem, so let the runtime path handle it (don't 400).- let toks = match tokenize(src) {- Ok(t) => t,- Err(_) => return Ok(()),- };- let mut p = Parser::new(&toks);- match p.parse_program() {- Err(e) if e == TOO_DEEP_MSG || e == EVAL_TOO_DEEP_MSG => Err(e),+ // Anything other than a depth violation — including tokenizer errors and+ // constructs outside our subset — is a plain syntax problem, so let the+ // runtime path handle it (don't 400). Compiled through the cache so that+ // repeated admission checks of the same script are free; whether the+ // evaluating thread reuses this entry depends on which one it is.+ match &*compile_script_cached(src) {+ Err(e) if e == TOO_DEEP_MSG || e == EVAL_TOO_DEEP_MSG => Err(e.clone()),_ => Ok(()),}}@@ -1221,11 +1335,16 @@ fn eval_painless_inner(src: &str, ctx: &PainlessCtx) -> Result<PainlessValue, StMAX_SCRIPT_LEN));}- let toks = tokenize(src)?;- let mut p = Parser::new(&toks);- let stmts = p.parse_program()?;+ // Compiled once per distinct source, not once per document. The AST is+ // shared (and immutable), so execution still gets its own `env` and its+ // own `ctx` counters — the closure guards below are per-evaluation.+ let compiled = compile_script_cached(src);+ let stmts = match &*compiled {+ Ok(stmts) => stmts,+ Err(e) => return Err(e.clone()),+ };let mut env: HashMap<String, PainlessValue> = HashMap::new();- exec_body(&stmts, ctx, &mut env)+ exec_body(stmts, ctx, &mut env)}/// Run a statement list with implicit-last-value return semantics: an@@ -1371,7 +1490,10 @@ fn eval_expr(Expr::Unary(op, x) => {let v = eval_expr(x, ctx, env)?;match op.as_str() {- "-" => Ok(PainlessValue::Number(-v.as_f64().unwrap_or(0.0))),+ "-" => v+ .as_f64()+ .map(|n| PainlessValue::Number(-n))+ .ok_or_else(|| "cannot apply unary '-' to a non-numeric value".to_string()),"!" => Ok(PainlessValue::Bool(!v.as_bool())),_ => Err(format!("bad unary {op}")),}@@ -1650,11 +1772,14 @@ fn apply_binary(PainlessValue::Bool(b) => b.to_string(),_ => "null".to_string(),};- return Ok(PainlessValue::String(format!(- "{}{}",- render(&left),- render(&right)- )));+ let left_r = render(&left);+ let right_r = render(&right);+ if left_r.len().saturating_add(right_r.len()) > MAX_PAINLESS_STRING_LEN {+ return Err(format!(+ "string concatenation result exceeds the {MAX_PAINLESS_STRING_LEN}-byte limit"+ ));+ }+ return Ok(PainlessValue::String(format!("{left_r}{right_r}")));}// ES Painless compares Strings as strings. Equality is false across@@ -1687,8 +1812,34 @@ fn apply_binary(}}- let left = left.as_f64().unwrap_or(0.0);- let right = right.as_f64().unwrap_or(0.0);+ // Null-aware equality. `params` is a `Map<String, Object>` in real+ // Painless, so a key that wasn't supplied reads as `null` and+ // `params.y == null` is an ordinary reference comparison that yields a+ // boolean — the numeric coercion below would otherwise reject the null+ // operand and turn every params-guarded script into an error.+ //+ // Only `==`/`!=` are null-aware. Real Painless's *relational* operators+ // and arithmetic unbox their operands and throw on a null, and so do+ // ours (below) — which is what keeps a typo'd field name from matching+ // every document.+ if matches!(op, "==" | "!=")+ && (matches!(left, PainlessValue::Null) || matches!(right, PainlessValue::Null))+ {+ let equal = matches!((&left, &right), (PainlessValue::Null, PainlessValue::Null));+ return Ok(PainlessValue::Bool(if op == "==" { equal } else { !equal }));+ }++ // A value that can't coerce to a number (most commonly `Null` — a+ // missing `params` field) errors rather than silently acting as+ // 0: real Painless throws unboxing a null into a primitive, and a+ // script that quietly matched-as-zero on every doc with a typo'd field+ // name would be far worse than one that errors and excludes the doc.+ let left = left+ .as_f64()+ .ok_or_else(|| format!("cannot apply '{op}' to a non-numeric value"))?;+ let right = right+ .as_f64()+ .ok_or_else(|| format!("cannot apply '{op}' to a non-numeric value"))?;let value = match op {"+" => left + right,"-" => left - right,@@ -1761,18 +1912,47 @@ fn resolve_doc_member(let raw = get_doc_value(ctx.doc, field);match member {"value" => {- // Return first scalar.+ // Return first scalar. A field that's genuinely missing (or a+ // multi-valued field with zero actual values) ERRORS — this is+ // literally what Elasticsearch does: since 7.0, an empty+ // `ScriptDocValues` throws+ // "A document doesn't have a value for a field! Use+ // doc[<field>].size()==0 to check if a document is missing a+ // field!"+ // (6.x returned a type default with a deprecation warning; 7.0+ // removed that). Returning `null` here instead would be neither+ // ES's behaviour nor safe: callers (script query filter,+ // script_score, ...) treat an error as "doesn't match" /+ // "no-op score", so erroring is what keeps `doc['typo'].value > -1`+ // from matching every document.+ //+ // Guarding a field that isn't on every document therefore uses+ // ES's own documented idiom — `doc['x'].size() == 0 ? <default>+ // : doc['x'].value` — which works because `.size()` (below) is+ // defined on a missing field and the ternary only evaluates the+ // branch it takes. `params.<key> == null` still works too, since+ // `params` really is a nullable map (see `apply_binary`).+ let missing = || {+ format!(+ "A document doesn't have a value for a field! \+ Use doc[{field}].size()==0 to check if a document is missing a field!"+ )+ };match raw {- Value::Array(arr) => Ok(arr+ Value::Array(arr) => arr.first().map(PainlessValue::from_json)- .unwrap_or(PainlessValue::Number(0.0))),+ .ok_or_else(missing),Value::Number(n) => Ok(PainlessValue::Number(n.as_f64().unwrap_or(0.0))),Value::String(s) => Ok(PainlessValue::String(s)),Value::Bool(b) => Ok(PainlessValue::Bool(b)),- _ => Ok(PainlessValue::Number(0.0)),+ _ => Err(missing()),}}+ // `.size()` / `.length` / `.empty` are defined on a MISSING field —+ // they are how a script asks whether the field is there at all, so+ // they must never error. This is the other half of the fail-closed+ // contract on `.value` above."size" | "length" => {if args.is_some() {// doc[...].size() with explicit call@@ -2829,4 +3009,313 @@ mod tests {assert!(eval_bool("1 + 1 == 2", &doc));assert!(eval_bool("doc['color'].value.length() == 5", &doc));}++ #[test]+ fn missing_doc_field_value_errors() {+ // This is ES's own behaviour, not a local hardening choice: since+ // 7.0 an empty `ScriptDocValues` throws rather than yielding a+ // default (6.x returned 0/"" with a deprecation warning). Returning+ // `null` here instead would reopen the hole this closes — see+ // `comparison_against_missing_field_errors_not_matches_everything`.+ let doc = json!({});+ let params = json!({});+ let r = eval_painless("doc['missing'].value", &ctx(&doc, ¶ms, 0.0));+ let e = r.expect_err("a missing field must error, matching ES 7+");+ // The message points at the supported guard, exactly as ES's does.+ assert!(+ e.contains("size()==0"),+ "the error must name the .size()==0 guard: {e}"+ );+ }++ #[test]+ fn comparison_against_missing_field_errors_not_matches_everything() {+ let doc = json!({});+ let params = json!({});+ let r = eval_painless("doc['missing'].value > -1", &ctx(&doc, ¶ms, 0.0));+ assert!(+ r.is_err(),+ "a comparison against a missing field must error, not silently match: got {:?}",+ r+ );+ }++ #[test]+ fn empty_multi_value_field_errors() {+ let doc = json!({"tags": []});+ let params = json!({});+ let r = eval_painless("doc['tags'].value", &ctx(&doc, ¶ms, 0.0));+ assert!(r.is_err(), "expected an error, got {:?}", r);+ }++ #[test]+ fn present_field_still_works_after_fail_closed_change() {+ let doc = json!({"x": 10});+ let params = json!({});+ let v = eval_painless("doc['x'].value > 5", &ctx(&doc, ¶ms, 0.0)).unwrap();+ assert!(v.as_bool());+ }++ #[test]+ fn exponential_string_doubling_is_bounded_not_a_crash() {+ let doc = json!({});+ let params = json!({});+ // 30 doublings from a 1-byte string would reach ~1 GiB unbounded;+ // the cap must trip well before that.+ let src = format!("def s = \"a\";{}return s;", "s = s + s;".repeat(30));+ let r = eval_painless(&src, &ctx(&doc, ¶ms, 0.0));+ assert!(+ r.is_err(),+ "expected the string-length cap to trip, got {:?}",+ r+ );+ }++ #[test]+ fn ordinary_string_concatenation_still_works() {+ let doc = json!({});+ let params = json!({});+ let v = eval_painless("'a' + 'b' + 'c'", &ctx(&doc, ¶ms, 0.0)).unwrap();+ match v {+ PainlessValue::String(s) => assert_eq!(s, "abc"),+ other => panic!("expected a string, got {:?}", other),+ }+ }++ // ── The null-guard idiom ──────────────────────────────────────────────────+ // Making a missing field error (above) is only safe if a script can still+ // ASK whether the field is there. Elasticsearch's answer is+ // `doc['x'].size() == 0`, which is also the remedy its own exception+ // message names, so that is the idiom that has to work here. It relies on+ // two properties, both asserted below: `.size()` is defined on a missing+ // field, and the ternary evaluates only the branch it takes.++ #[test]+ fn size_guard_is_defined_on_a_missing_field() {+ let params = json!({});+ for (doc, expected_size, expected_empty) in [+ (json!({}), 0.0, true),+ (json!({"x": null}), 0.0, true),+ (json!({"x": []}), 0.0, true),+ (json!({"x": 7}), 1.0, false),+ (json!({"x": [1, 2, 3]}), 3.0, false),+ ] {+ let c = ctx(&doc, ¶ms, 0.0);+ let size = eval_painless("doc['x'].size()", &c)+ .unwrap_or_else(|e| panic!(".size() must never error on {doc}: {e}"));+ assert_eq!(size.as_f64().unwrap(), expected_size, "doc {doc}");+ let empty = eval_painless("doc['x'].empty", &c)+ .unwrap_or_else(|e| panic!(".empty must never error on {doc}: {e}"));+ assert_eq!(empty.as_bool(), expected_empty, "doc {doc}");+ }+ }++ #[test]+ fn null_guard_idiom_evaluates_on_both_shapes() {+ // The canonical ES form, verbatim.+ let src = "doc['x'].size() == 0 ? 0 : doc['x'].value";+ let params = json!({});++ for (doc, expected) in [+ (json!({}), 0.0),+ (json!({"x": []}), 0.0),+ (json!({"x": 42}), 42.0),+ (json!({"x": [7, 8]}), 7.0),+ ] {+ let v = eval_painless(src, &ctx(&doc, ¶ms, 0.0))+ .unwrap_or_else(|e| panic!("the null-guard idiom must not error on {doc}: {e}"));+ assert_eq!(v.as_f64().unwrap(), expected, "doc {doc}");+ }+ }++ #[test]+ fn guarded_script_filters_rather_than_erroring_everywhere() {+ // The whole point of the idiom: one script, run over documents that+ // do and don't have the field, returning a boolean for each.+ let params = json!({"min": 5});+ let src = "doc['x'].size() == 0 ? false : doc['x'].value > params.min";++ for (doc, expected) in [+ (json!({"x": 10}), true),+ (json!({"x": 1}), false),+ (json!({}), false),+ ] {+ let v = eval_painless(src, &ctx(&doc, ¶ms, 0.0))+ .unwrap_or_else(|e| panic!("guarded script errored on {doc}: {e}"));+ assert_eq!(v.as_bool(), expected, "doc {doc}");+ }+ }++ #[test]+ fn null_guard_on_a_missing_param_is_a_boolean() {+ // `params` is a real `Map<String, Object>` in Painless, so a key+ // that wasn't supplied reads as null and comparing it to null is an+ // ordinary reference comparison — unlike `doc[...]`, which throws.+ let doc = json!({});++ let empty = json!({});+ let v = eval_painless("params.y == null", &ctx(&doc, &empty, 0.0))+ .expect("a params null guard must evaluate, not error");+ assert!(v.as_bool(), "missing param: `== null` must be true");++ let supplied = json!({"y": 3});+ let v = eval_painless("params.y == null", &ctx(&doc, &supplied, 0.0))+ .expect("a params null guard must evaluate, not error");+ assert!(!v.as_bool(), "supplied param: `== null` must be false");++ // `!=` is the same comparison inverted, and a present-but-null value+ // is indistinguishable from an absent one, as in ES.+ let explicit_null = json!({"y": null});+ let v = eval_painless("params.y != null", &ctx(&doc, &explicit_null, 0.0)).unwrap();+ assert!(!v.as_bool());+ }++ #[test]+ fn genuinely_invalid_arithmetic_still_errors() {+ let doc = json!({});+ let params = json!({"obj": {"a": 1}});+ // Null-awareness is scoped to `==`/`!=`; nothing else got looser, so+ // the fail-closed property the script query depends on is intact.+ for src in [+ "'abc' - params.obj",+ "params.missing + 1",+ "params.missing > 0",+ "doc['nope'].value * 2",+ "doc['nope'].value >= 0",+ "-params.missing",+ ] {+ let r = eval_painless(src, &ctx(&doc, ¶ms, 0.0));+ assert!(r.is_err(), "`{src}` must still error, got {:?}", r);+ }+ }++ // ── Compiled-script cache ─────────────────────────────────────────────────+ // The cache is thread-local, and each `#[test]` gets its own thread, so+ // every assertion below observes only what its own test evaluated.++ /// `(entries, src_bytes, actual bytes held)` for the calling thread.+ fn cache_stats() -> (usize, usize, usize) {+ SCRIPT_CACHE.with(|c| {+ let c = c.borrow();+ (+ c.entries.len(),+ c.src_bytes,+ c.entries.keys().map(|k| k.len()).sum(),+ )+ })+ }++ #[test]+ fn compiled_script_cache_is_bounded_by_entry_count() {+ let doc = json!({});+ let params = json!({});+ // Far more distinct sources than the cache may retain. Each is tiny,+ // so the entry-count bound is the one under test.+ for i in 0..(MAX_SCRIPT_CACHE_ENTRIES * 8) {+ let src = format!("{i} + 1");+ eval_painless(&src, &ctx(&doc, ¶ms, 0.0)).unwrap();+ }+ let (entries, src_bytes, actual) = cache_stats();+ assert!(+ entries <= MAX_SCRIPT_CACHE_ENTRIES,+ "cache grew to {entries} entries, bound is {MAX_SCRIPT_CACHE_ENTRIES}"+ );+ assert!(+ src_bytes <= MAX_SCRIPT_CACHE_SRC_BYTES,+ "cache holds {src_bytes} source bytes, bound is {MAX_SCRIPT_CACHE_SRC_BYTES}"+ );+ // The accounting the bound rests on must match reality.+ assert_eq!(actual, src_bytes, "src_bytes accounting drifted");+ }++ #[test]+ fn compiled_script_cache_is_bounded_by_source_bytes() {+ let doc = json!({});+ let params = json!({});+ // Distinct ~33 KiB scripts (each legal under MAX_SCRIPT_LEN, and+ // flat rather than deep so they compile and run cleanly). Well under+ // MAX_SCRIPT_CACHE_ENTRIES of them already exceed the byte budget,+ // so the byte bound has to be what stops the growth.+ for i in 0..24 {+ let src = format!("{}return {i};", "def a = 1;".repeat(3300));+ assert!(src.len() > 32 * 1024);+ eval_painless(&src, &ctx(&doc, ¶ms, 0.0)).unwrap();+ }+ let (entries, src_bytes, actual) = cache_stats();+ assert!(+ src_bytes <= MAX_SCRIPT_CACHE_SRC_BYTES,+ "cache holds {src_bytes} source bytes, bound is {MAX_SCRIPT_CACHE_SRC_BYTES}"+ );+ assert!(entries <= MAX_SCRIPT_CACHE_ENTRIES);+ assert!(+ entries < 24,+ "the byte bound must have evicted something: {entries} entries retained"+ );+ assert_eq!(actual, src_bytes, "src_bytes accounting drifted");+ }++ #[test]+ fn cached_script_is_parsed_once_and_still_correct() {+ let params = json!({"multiplier": 3});+ let src = "doc['n'].value * params.multiplier /* cache-once probe */";+ for n in 1..=5 {+ let doc = json!({ "n": n });+ let v = eval_painless(src, &ctx(&doc, ¶ms, 0.0)).unwrap();+ assert!((v.as_f64().unwrap() - (n * 3) as f64).abs() < 1e-9);+ }+ assert!(+ SCRIPT_CACHE.with(|c| c.borrow().entries.contains_key(src)),+ "repeated evaluation of one source must leave a cached AST"+ );+ assert_eq!(+ cache_stats().0,+ 1,+ "five evaluations of one source must leave exactly one entry"+ );+ }++ #[test]+ fn cached_parse_failure_still_reports_the_same_error() {+ let doc = json!({});+ let params = json!({});+ let src = "return ( ( ( /* unbalanced, cache-error probe */";+ let first = eval_painless(src, &ctx(&doc, ¶ms, 0.0)).unwrap_err();+ let second = eval_painless(src, &ctx(&doc, ¶ms, 0.0)).unwrap_err();+ assert_eq!(first, second);+ }++ #[test]+ fn caching_does_not_share_state_between_evaluations() {+ // The AST is shared but mutable evaluation state is not: a script+ // that mutates a local must start from scratch on every document.+ let params = json!({});+ let src = "def acc = 0; acc = acc + doc['n'].value; \+ acc = acc + doc['n'].value; acc = acc + doc['n'].value; \+ return acc; /* shared-state probe */";+ for n in 1..=4 {+ let doc = json!({ "n": n });+ let v = eval_painless(src, &ctx(&doc, ¶ms, 0.0)).unwrap();+ assert_eq!(v.as_f64().unwrap(), (n * 3) as f64, "n = {n}");+ }+ }++ #[test]+ fn cached_ast_does_not_leak_the_call_budget_across_evaluations() {+ // MAX_CALL_COUNT lives on the per-evaluation `PainlessCtx`, not on+ // the AST, so memoising the AST must not let one document's closure+ // invocations count against the next document's budget. A script+ // that spends a large slice of the budget has to keep succeeding+ // when it is re-run against document after document.+ let doc = json!({});+ let params = json!({});+ // 2^11 = 2048 invocations per evaluation, comfortably under+ // MAX_CALL_COUNT (10_000) but far more than a per-process budget+ // would survive being run 8 times.+ let src = "def f = (g, n) -> { if (n <= 0) { return 1; } return g(g, n - 1) + g(g, n - 1); }; return f(f, 11); /* budget-reset probe */";+ for round in 0..8 {+ let v = eval_painless(src, &ctx(&doc, ¶ms, 0.0))+ .unwrap_or_else(|e| panic!("round {round} must stay within budget: {e}"));+ assert!((v.as_f64().unwrap() - 2048.0).abs() < 1e-9, "round {round}");+ }+ }}diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rsindex af8787ee0..fb99d2fa0 100644--- a/engine/crates/xerj-query/src/ast.rs+++ b/engine/crates/xerj-query/src/ast.rs@@ -377,6 +377,18 @@ pub enum QueryNode {/// Matches documents whose `_id` is in the given list.Ids { values: Vec<String> },+ /// A standalone Painless `script` query — matches when the script+ /// (given the doc's `_source` via `doc`/`params`) evaluates truthy.+ /// Distinct from `script_score` (a `function_score` scoring function):+ /// this one is a boolean filter, no scoring involved. Used e.g. by+ /// OpenSearch's UBI sample dashboards ("Top Searches Without Results")+ /// to filter on a computed field via a boolean Supplier-lambda script.+ Script {+ source: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ params: Option<serde_json::Value>,+ },+// ── Full-text ─────────────────────────────────────────────────────────────/// Analysed text query against a single field.Match {diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rsindex dde49a216..f56870abf 100644--- a/engine/crates/xerj-query/src/parser.rs+++ b/engine/crates/xerj-query/src/parser.rs@@ -296,6 +296,7 @@ pub fn parse_query(json: &Value) -> Result<QueryNode> {"wildcard" => parse_wildcard(params),"exists" => parse_exists(params),"ids" => parse_ids(params),+ "script" => parse_script(params),"bool" => parse_bool(params),"query_string" => parse_query_string(params),"constant_score" => parse_constant_score(params),@@ -1093,6 +1094,23 @@ fn parse_ids(params: &Value) -> Result<QueryNode> {Ok(node)}+fn parse_script(params: &Value) -> Result<QueryNode> {+ let obj = params+ .as_object()+ .ok_or_else(|| qerr("`script` must be an object"))?;+ let script = obj+ .get("script")+ .and_then(Value::as_object)+ .ok_or_else(|| qerr("`script.script` is required"))?;+ let source = script+ .get("source")+ .and_then(Value::as_str)+ .ok_or_else(|| qerr("`script.script.source` is required"))?+ .to_string();+ let params = script.get("params").cloned();+ Ok(QueryNode::Script { source, params })+}+fn parse_query_string(params: &Value) -> Result<QueryNode> {let obj = params.as_object()@@ -4348,6 +4366,43 @@ mod tests {}}+ // ── script ────────────────────────────────────────────────────────────────++ #[test]+ fn test_script_query() {+ let node = q(json!({+ "script": {+ "script": {+ "source": "doc['x'].value > params.min",+ "params": {"min": 5}+ }+ }+ }));+ if let QueryNode::Script { source, params } = node {+ assert_eq!(source, "doc['x'].value > params.min");+ assert_eq!(params, Some(json!({"min": 5})));+ } else {+ panic!("wrong variant");+ }+ }++ #[test]+ fn test_script_query_no_params() {+ let node = q(json!({"script": {"script": {"source": "true"}}}));+ if let QueryNode::Script { source, params } = node {+ assert_eq!(source, "true");+ assert_eq!(params, None);+ } else {+ panic!("wrong variant");+ }+ }++ #[test]+ fn test_script_query_missing_source_errs() {+ let result = parse_query(&json!({"script": {"script": {}}}));+ assert!(result.is_err());+ }+// ── bool ──────────────────────────────────────────────────────────────────#[test]diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rsindex ffda47fa0..f84402527 100644--- a/engine/crates/xerj-query/src/planner.rs+++ b/engine/crates/xerj-query/src/planner.rs@@ -590,6 +590,10 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {// engine's doc-scan path; MatchAll satisfies exhaustiveness here.QueryNode::Percolate { .. } => ExecutionPlan::MatchAll,+ // Script: no index can back an arbitrary Painless predicate — pure+ // doc-scan, same as Percolate above.+ QueryNode::Script { .. } => ExecutionPlan::MatchAll,+// ── Span queries — treat as doc-scans ────────────────────────────────QueryNode::SpanTerm { field, value } => ExecutionPlan::FtsScan {field,