Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(query): add script query support (#87)

Vinz216879ec079b896e
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index d330d0621..f0dd79203 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -13267,6 +13267,93 @@ mod malformed_bulk_route_tests {
}
}
+#[cfg(test)]
+mod script_query_end_to_end_tests {
+ //! Regression coverage for a real bug: the `script` query's unit tests
+ //! (in xerj-engine's index.rs) all exercised `doc_matches_query`
+ //! directly, never a real `POST /{index}/_search` — so they passed
+ //! while the query silently returned zero hits for every real search,
+ //! because `is_doc_scan_query` never routed `QueryNode::Script` into
+ //! doc-scan. This test goes through the actual HTTP route so a
+ //! regression here fails loudly.
+ use super::*;
+ use axum::{
+ body::{to_bytes, Body},
+ http::Request,
+ };
+ use tower::ServiceExt;
+ use xerj_common::{
+ config::{Config, WalSync},
+ metrics::Metrics,
+ };
+ use xerj_engine::Engine;
+
+ fn test_state() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+ let metrics = Metrics::new().expect("metrics");
+ let engine = Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ #[tokio::test]
+ async fn script_query_matches_an_indexed_document_via_real_search() {
+ let router = crate::router::build_es_compat_router(test_state());
+
+ let index_response = router
+ .clone()
+ .oneshot(
+ Request::post("/script-query-e2e/_doc/1")
+ .header("content-type", "application/json")
+ .body(Body::from(r#"{"x": 10}"#))
+ .expect("request"),
+ )
+ .await
+ .expect("index response");
+ assert!(
+ index_response.status().is_success(),
+ "indexing failed: {:?}",
+ index_response.status()
+ );
+
+ let search_body = serde_json::to_vec(&json!({
+ "query": {
+ "script": {
+ "script": {"source": "doc['x'].value > 5"}
+ }
+ }
+ }))
+ .expect("serialize query");
+
+ let search_response = router
+ .oneshot(
+ Request::post("/script-query-e2e/_search")
+ .header("content-type", "application/json")
+ .body(Body::from(search_body))
+ .expect("request"),
+ )
+ .await
+ .expect("search response");
+ assert!(
+ search_response.status().is_success(),
+ "search failed: {:?}",
+ search_response.status()
+ );
+
+ let bytes = to_bytes(search_response.into_body(), usize::MAX)
+ .await
+ .expect("response bytes");
+ let response: Value = serde_json::from_slice(&bytes).expect("JSON response");
+ let total = response["hits"]["total"]["value"].as_u64().unwrap_or(0);
+ assert_eq!(
+ total, 1,
+ "script query should match the freshly indexed document via a real search, got: {response}"
+ );
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Schema conversion helpers
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 43f55f4fe..12e8f8e15 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -19518,7 +19518,7 @@ impl Index {
// a malformed section bails early and leaves the vec incomplete.
mut offsets_out: Option<&mut Vec<(u32, u32)>>,
// Cooperative timeout: when `Some`, the walk polls the wall clock
- // every 4 096 docs and aborts (setting `*timed_out`) once past the
+ // every 256 docs and aborts (setting `*timed_out`) once past the
// deadline. One giant merged segment is otherwise a multi-second
// uninterruptible unit — the between-segment deadline check in
// `search_inner` can't bound it.
@@ -19639,9 +19639,28 @@ impl Index {
doc_pos += 1;
*dbg_walked += 1;
- // Cooperative timeout poll — every 4 096 docs (~1 µs of clock
- // reads per million docs; the parse below costs ~3 µs/doc).
- if doc_pos & 0xFFF == 0 {
+ // Cooperative timeout poll — every 256 docs (~16 µs of clock
+ // reads per million docs, against a per-doc cost that is at
+ // minimum the ~3 µs JSON parse below; the read is noise).
+ //
+ // The interval used to be 4 096, sized for that ~3 µs/doc floor.
+ // A `script` query breaks that assumption: even with the Painless
+ // AST cached (see `painless::compile_script_cached`), *executing*
+ // a script dominates the per-doc cost. Measured on a release
+ // build: a benign 64 KiB script runs 385 µs/doc, so 4 096 docs is
+ // a ~1.6 s stretch in which a `timeout` cannot take effect, and
+ // 256 brings that to ~98 ms.
+ //
+ // 256 is an improvement, not a bound. A deliberately adversarial
+ // script of legal size — flat, no closure calls, tripping none of
+ // MAX_CALL_DEPTH / MAX_CALL_COUNT / MAX_EVAL_DEPTH /
+ // MAX_PARSE_DEPTH / MAX_PAINLESS_STRING_LEN — was measured at
+ // 1.27 s/doc, which is ~325 s per poll interval. Nothing here
+ // bounds that, because there is no per-evaluation CPU budget; the
+ // poll interval only decides how often an already-cheap document
+ // boundary is checked. Closing that needs a real time or
+ // instruction budget inside `eval_painless`, tracked separately.
+ if doc_pos & 0xFF == 0 {
if let Some(dl) = deadline {
if std::time::Instant::now() >= dl {
*timed_out = true;
@@ -23817,6 +23836,7 @@ fn is_doc_scan_query(q: &QueryNode) -> bool {
| QueryNode::Boosted { .. }
| QueryNode::Intervals { .. }
| QueryNode::Percolate { .. }
+ | QueryNode::Script { .. }
)
}
@@ -24010,7 +24030,8 @@ fn query_needs_id_injection(q: &QueryNode) -> bool {
| QueryNode::GeoShape { .. }
| QueryNode::SpanTerm { .. }
| QueryNode::MoreLikeThis { .. }
- | QueryNode::Percolate { .. } => false,
+ | QueryNode::Percolate { .. }
+ | QueryNode::Script { .. } => false,
QueryNode::Bool {
must,
should,
@@ -24249,6 +24270,25 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {
.unwrap_or(false)
}
+ QueryNode::Script {
+ source: src,
+ params,
+ } => {
+ let empty = Value::Object(serde_json::Map::new());
+ let p = params.as_ref().unwrap_or(&empty);
+ let ctx = crate::painless::PainlessCtx::new(source, p, 0.0);
+ // ES requires a `script` query's script to return a boolean;
+ // a script that errors, or returns anything else (a number, a
+ // string, ...), never matches — real ES throws a script-cast
+ // exception in that case, which drops the doc in filter
+ // contexts, so this fails closed the same way rather than
+ // truthy-coercing a non-boolean result.
+ matches!(
+ crate::painless::eval_painless(src, &ctx),
+ Ok(crate::painless::PainlessValue::Bool(true))
+ )
+ }
+
QueryNode::Ids { values } => {
// _id is passed via the source's "_id" metadata field when present,
// but for memtable doc-scan calls the source may not contain it.
@@ -31267,6 +31307,112 @@ mod percolate_tests {
}
}
+#[cfg(test)]
+mod script_query_tests {
+ //! A standalone `script` query is a boolean filter (no scoring), used
+ //! e.g. by OpenSearch's UBI sample dashboards to filter on a computed
+ //! field via a Painless boolean lambda. Distinct from `script_score`.
+ use super::*;
+ use serde_json::json;
+ use xerj_query::ast::QueryNode;
+
+ #[test]
+ fn script_query_matches_when_truthy() {
+ let doc = json!({"x": 10});
+ let q = QueryNode::Script {
+ source: "doc['x'].value > params.min".to_string(),
+ params: Some(json!({"min": 5})),
+ };
+ assert!(doc_matches_query(&q, &doc));
+ }
+
+ #[test]
+ fn script_query_no_match_when_falsy() {
+ let doc = json!({"x": 1});
+ let q = QueryNode::Script {
+ source: "doc['x'].value > params.min".to_string(),
+ params: Some(json!({"min": 5})),
+ };
+ assert!(!doc_matches_query(&q, &doc));
+ }
+
+ #[test]
+ fn script_query_fails_closed_on_script_error() {
+ let doc = json!({"x": 1});
+ let q = QueryNode::Script {
+ source: "this is not valid painless".to_string(),
+ params: None,
+ };
+ assert!(!doc_matches_query(&q, &doc));
+ }
+
+ #[test]
+ fn script_query_fails_closed_on_non_boolean_result() {
+ // A script that evaluates to a number (not a boolean) must not match
+ // — ES itself requires `script` queries to return a boolean.
+ let doc = json!({"x": 1});
+ let q = QueryNode::Script {
+ source: "1 + 1".to_string(),
+ params: None,
+ };
+ assert!(!doc_matches_query(&q, &doc));
+ }
+
+ #[test]
+ fn script_query_null_guard_filters_rather_than_dropping_everything() {
+ // The standard shape for a script filter over a field that isn't on
+ // every document, written the way Elasticsearch documents it. If the
+ // guard errors instead of yielding a boolean, the script is
+ // unrunnable and EVERY document is dropped — including the ones that
+ // should match. This is what a `script` query looks like in
+ // practice, so it is checked at the query surface and not only in
+ // the Painless unit tests.
+ let q = QueryNode::Script {
+ source: "doc['x'].size() == 0 ? false : doc['x'].value > params.min".to_string(),
+ params: Some(json!({"min": 5})),
+ };
+ assert!(
+ doc_matches_query(&q, &json!({"x": 10})),
+ "guarded script must still match the document it selects for"
+ );
+ assert!(!doc_matches_query(&q, &json!({"x": 1})));
+ assert!(
+ !doc_matches_query(&q, &json!({"other": 1})),
+ "a document without the field must be filtered out, not matched"
+ );
+ }
+
+ #[test]
+ fn script_query_with_a_typoed_field_matches_nothing() {
+ // The bug the fail-closed `.value` exists for: reading a missing
+ // field as 0 made `doc['typo'].value > -1` true for every document,
+ // turning a filter into a no-op. It must match NOTHING instead —
+ // including documents that do have the correctly-spelled field.
+ let q = QueryNode::Script {
+ source: "doc['coutn'].value > -1".to_string(),
+ params: None,
+ };
+ for doc in [
+ json!({"count": 5}),
+ json!({"count": 0}),
+ json!({}),
+ json!({"coutn": null}),
+ ] {
+ assert!(
+ !doc_matches_query(&q, &doc),
+ "a typo'd field name must not match {doc}"
+ );
+ }
+ // And the correctly-spelled query still works, so this is a spelling
+ // check and not a blanket refusal.
+ let ok = QueryNode::Script {
+ source: "doc['count'].value > -1".to_string(),
+ params: None,
+ };
+ assert!(doc_matches_query(&ok, &json!({"count": 5})));
+ }
+}
+
#[cfg(test)]
mod highlight_multibyte_tests {
//! Regression tests for the multibyte-highlight SIGABRT: a term whose
diff --git a/engine/crates/xerj-engine/src/painless.rs b/engine/crates/xerj-engine/src/painless.rs
index f91a6572c..37835aee5 100644
--- a/engine/crates/xerj-engine/src/painless.rs
+++ b/engine/crates/xerj-engine/src/painless.rs
@@ -45,7 +45,9 @@
//! should fall back to a no-op score on script error.
use serde_json::Value;
+use std::cell::RefCell;
use std::collections::HashMap;
+use std::rc::Rc;
#[derive(Debug, Clone)]
pub enum PainlessValue {
@@ -458,6 +460,16 @@ pub(crate) const MAX_EVAL_DEPTH: usize = 500;
/// we build (and later drop) from a single request.
pub(crate) const MAX_SCRIPT_LEN: usize = 64 * 1024;
+/// Safety ceiling on any single string value produced during evaluation.
+/// String concatenation (`+`) is the only operation that can grow a value
+/// beyond the size of its parts, so a handful of flat (non-nested, non-deep)
+/// `s = s + s;` statements — none of which trip [`MAX_EVAL_DEPTH`] or
+/// [`MAX_PARSE_DEPTH`], since they're neither deeply nested nor deeply
+/// recursive — doubles a string exponentially per statement. Without this
+/// cap a script well under [`MAX_SCRIPT_LEN`] can exhaust available memory
+/// before any other limit fires.
+const MAX_PAINLESS_STRING_LEN: usize = 1024 * 1024;
+
/// Sentinel error returned when [`MAX_PARSE_DEPTH`] is exceeded. Callers
/// (`check_script_limits`) match on it to distinguish "too complex" (a 400)
/// from ordinary syntax errors (which degrade gracefully at runtime).
@@ -1167,6 +1179,110 @@ impl Drop for CallGuard<'_> {
}
}
+// ── Compiled-script cache ────────────────────────────────────────────────────
+//
+// A script is evaluated once per *document*. Tokenizing and parsing it per
+// document made the per-doc cost scale with the script's SIZE rather than
+// with its complexity: a legal 64 KiB script (see [`MAX_SCRIPT_LEN`]) cost
+// ~2 ms/doc, essentially all of it re-parsing an AST identical to the one
+// built for the previous document. That is also what made the doc-scan's
+// cooperative timeout ineffective — `scan_stored_section_into` polls its
+// deadline every N docs on the assumption that per-doc work is a few
+// microseconds, so a 2 ms/doc script stretched the uninterruptible quantum
+// from milliseconds to seconds and a `timeout` could not cut the scan short.
+//
+// Scripts are pure source text, so the AST is a pure function of the source
+// and can simply be memoised. The cache MUST be bounded: the key is
+// attacker-supplied source, so an unbounded map is itself a memory-exhaustion
+// vector — a caller need only issue queries with unique scripts.
+
+/// Maximum number of distinct compiled scripts retained at once.
+const MAX_SCRIPT_CACHE_ENTRIES: usize = 128;
+
+/// Maximum total *source* bytes retained across cached entries. Bounds the
+/// cache independently of the entry count, since [`MAX_SCRIPT_LEN`] allows a
+/// single 64 KiB script and 128 of those would be 8 MiB of source (and a
+/// considerably larger multiple of that in AST nodes).
+const MAX_SCRIPT_CACHE_SRC_BYTES: usize = 512 * 1024;
+
+/// A parse result shared between evaluations. Parse *failures* are cached
+/// too — a syntactically invalid script is also evaluated once per document,
+/// so leaving errors uncached would leave the same per-doc parse cost (and
+/// the same unbounded quantum) reachable through a malformed script.
+///
+/// `Rc`, not `Arc`, because [`Stmt::FnDecl`]/[`Expr::Lambda`] bodies are
+/// themselves `Rc`-shared — deliberately non-atomic, since a closure literal
+/// is cloned on every invocation. The cache is therefore per-thread (see
+/// [`SCRIPT_CACHE`]) rather than a shared static.
+type CompiledScript = Rc<Result<Vec<Stmt>, String>>;
+
+#[derive(Default)]
+struct ScriptCache {
+ entries: HashMap<String, CompiledScript>,
+ /// Sum of `entries`' key lengths; maintained incrementally.
+ src_bytes: usize,
+}
+
+impl ScriptCache {
+ fn insert(&mut self, src: &str, compiled: CompiledScript) {
+ // Never cacheable on its own — would evict everything and still not fit.
+ if src.len() > MAX_SCRIPT_CACHE_SRC_BYTES || self.entries.contains_key(src) {
+ return;
+ }
+ // Evict (in arbitrary hash order — the workload is "one script for a
+ // whole query", so recency bookkeeping buys nothing worth its cost)
+ // until the newcomer fits under BOTH bounds.
+ while self.entries.len() >= MAX_SCRIPT_CACHE_ENTRIES
+ || self.src_bytes.saturating_add(src.len()) > MAX_SCRIPT_CACHE_SRC_BYTES
+ {
+ let Some(victim) = self.entries.keys().next().cloned() else {
+ // Empty and still over budget is impossible given the guard
+ // above, but never spin.
+ return;
+ };
+ self.entries.remove(&victim);
+ self.src_bytes = self.src_bytes.saturating_sub(victim.len());
+ }
+ self.src_bytes = self.src_bytes.saturating_add(src.len());
+ self.entries.insert(src.to_string(), compiled);
+ }
+}
+
+thread_local! {
+ /// One cache per thread. A compiled AST contains `Rc`s and so cannot
+ /// cross threads; a `Mutex<Arc<...>>` would not help, since the `Rc`s
+ /// *inside* the AST are the non-`Sync` part.
+ ///
+ /// The bound is therefore per-thread, and the total is bounded by
+ /// (search worker threads) × [`MAX_SCRIPT_CACHE_SRC_BYTES`] — still a
+ /// fixed ceiling, because the worker pool is sized from the core count,
+ /// not from the request rate. Per-thread caching also removes the lock
+ /// from a path taken once per document, and each worker scanning the
+ /// same segment converges on the same one entry.
+ static SCRIPT_CACHE: RefCell<ScriptCache> = RefCell::new(ScriptCache::default());
+}
+
+fn compile_script(src: &str) -> Result<Vec<Stmt>, String> {
+ let toks = tokenize(src)?;
+ let mut p = Parser::new(&toks);
+ p.parse_program()
+}
+
+/// Tokenize + parse `src`, reusing a previously compiled AST when possible.
+fn compile_script_cached(src: &str) -> CompiledScript {
+ SCRIPT_CACHE.with(|cache| {
+ // Borrow, look up, release — the insert below needs a fresh mutable
+ // borrow, and `compile_script` must not run while either is held.
+ let hit = cache.borrow().entries.get(src).map(Rc::clone);
+ if let Some(hit) = hit {
+ return hit;
+ }
+ let compiled: CompiledScript = Rc::new(compile_script(src));
+ cache.borrow_mut().insert(src, Rc::clone(&compiled));
+ compiled
+ })
+}
+
/// Validate a script against the parser/length resource limits WITHOUT running
/// it, so the request layer can reject an abusive script with a 400 up front.
///
@@ -1183,15 +1299,13 @@ pub fn check_script_limits(src: &str) -> Result<(), String> {
MAX_SCRIPT_LEN
));
}
- // The tokenizer is non-recursive; a tokenizer error is a plain syntax
- // problem, so let the runtime path handle it (don't 400).
- let toks = match tokenize(src) {
- Ok(t) => t,
- Err(_) => return Ok(()),
- };
- let mut p = Parser::new(&toks);
- match p.parse_program() {
- Err(e) if e == TOO_DEEP_MSG || e == EVAL_TOO_DEEP_MSG => Err(e),
+ // Anything other than a depth violation — including tokenizer errors and
+ // constructs outside our subset — is a plain syntax problem, so let the
+ // runtime path handle it (don't 400). Compiled through the cache so that
+ // repeated admission checks of the same script are free; whether the
+ // evaluating thread reuses this entry depends on which one it is.
+ match &*compile_script_cached(src) {
+ Err(e) if e == TOO_DEEP_MSG || e == EVAL_TOO_DEEP_MSG => Err(e.clone()),
_ => Ok(()),
}
}
@@ -1221,11 +1335,16 @@ fn eval_painless_inner(src: &str, ctx: &PainlessCtx) -> Result<PainlessValue, St
MAX_SCRIPT_LEN
));
}
- let toks = tokenize(src)?;
- let mut p = Parser::new(&toks);
- let stmts = p.parse_program()?;
+ // Compiled once per distinct source, not once per document. The AST is
+ // shared (and immutable), so execution still gets its own `env` and its
+ // own `ctx` counters — the closure guards below are per-evaluation.
+ let compiled = compile_script_cached(src);
+ let stmts = match &*compiled {
+ Ok(stmts) => stmts,
+ Err(e) => return Err(e.clone()),
+ };
let mut env: HashMap<String, PainlessValue> = HashMap::new();
- exec_body(&stmts, ctx, &mut env)
+ exec_body(stmts, ctx, &mut env)
}
/// Run a statement list with implicit-last-value return semantics: an
@@ -1371,7 +1490,10 @@ fn eval_expr(
Expr::Unary(op, x) => {
let v = eval_expr(x, ctx, env)?;
match op.as_str() {
- "-" => Ok(PainlessValue::Number(-v.as_f64().unwrap_or(0.0))),
+ "-" => v
+ .as_f64()
+ .map(|n| PainlessValue::Number(-n))
+ .ok_or_else(|| "cannot apply unary '-' to a non-numeric value".to_string()),
"!" => Ok(PainlessValue::Bool(!v.as_bool())),
_ => Err(format!("bad unary {op}")),
}
@@ -1650,11 +1772,14 @@ fn apply_binary(
PainlessValue::Bool(b) => b.to_string(),
_ => "null".to_string(),
};
- return Ok(PainlessValue::String(format!(
- "{}{}",
- render(&left),
- render(&right)
- )));
+ let left_r = render(&left);
+ let right_r = render(&right);
+ if left_r.len().saturating_add(right_r.len()) > MAX_PAINLESS_STRING_LEN {
+ return Err(format!(
+ "string concatenation result exceeds the {MAX_PAINLESS_STRING_LEN}-byte limit"
+ ));
+ }
+ return Ok(PainlessValue::String(format!("{left_r}{right_r}")));
}
// ES Painless compares Strings as strings. Equality is false across
@@ -1687,8 +1812,34 @@ fn apply_binary(
}
}
- let left = left.as_f64().unwrap_or(0.0);
- let right = right.as_f64().unwrap_or(0.0);
+ // Null-aware equality. `params` is a `Map<String, Object>` in real
+ // Painless, so a key that wasn't supplied reads as `null` and
+ // `params.y == null` is an ordinary reference comparison that yields a
+ // boolean — the numeric coercion below would otherwise reject the null
+ // operand and turn every params-guarded script into an error.
+ //
+ // Only `==`/`!=` are null-aware. Real Painless's *relational* operators
+ // and arithmetic unbox their operands and throw on a null, and so do
+ // ours (below) — which is what keeps a typo'd field name from matching
+ // every document.
+ if matches!(op, "==" | "!=")
+ && (matches!(left, PainlessValue::Null) || matches!(right, PainlessValue::Null))
+ {
+ let equal = matches!((&left, &right), (PainlessValue::Null, PainlessValue::Null));
+ return Ok(PainlessValue::Bool(if op == "==" { equal } else { !equal }));
+ }
+
+ // A value that can't coerce to a number (most commonly `Null` — a
+ // missing `params` field) errors rather than silently acting as
+ // 0: real Painless throws unboxing a null into a primitive, and a
+ // script that quietly matched-as-zero on every doc with a typo'd field
+ // name would be far worse than one that errors and excludes the doc.
+ let left = left
+ .as_f64()
+ .ok_or_else(|| format!("cannot apply '{op}' to a non-numeric value"))?;
+ let right = right
+ .as_f64()
+ .ok_or_else(|| format!("cannot apply '{op}' to a non-numeric value"))?;
let value = match op {
"+" => left + right,
"-" => left - right,
@@ -1761,18 +1912,47 @@ fn resolve_doc_member(
let raw = get_doc_value(ctx.doc, field);
match member {
"value" => {
- // Return first scalar.
+ // Return first scalar. A field that's genuinely missing (or a
+ // multi-valued field with zero actual values) ERRORS — this is
+ // literally what Elasticsearch does: since 7.0, an empty
+ // `ScriptDocValues` throws
+ // "A document doesn't have a value for a field! Use
+ // doc[<field>].size()==0 to check if a document is missing a
+ // field!"
+ // (6.x returned a type default with a deprecation warning; 7.0
+ // removed that). Returning `null` here instead would be neither
+ // ES's behaviour nor safe: callers (script query filter,
+ // script_score, ...) treat an error as "doesn't match" /
+ // "no-op score", so erroring is what keeps `doc['typo'].value > -1`
+ // from matching every document.
+ //
+ // Guarding a field that isn't on every document therefore uses
+ // ES's own documented idiom — `doc['x'].size() == 0 ? <default>
+ // : doc['x'].value` — which works because `.size()` (below) is
+ // defined on a missing field and the ternary only evaluates the
+ // branch it takes. `params.<key> == null` still works too, since
+ // `params` really is a nullable map (see `apply_binary`).
+ let missing = || {
+ format!(
+ "A document doesn't have a value for a field! \
+ Use doc[{field}].size()==0 to check if a document is missing a field!"
+ )
+ };
match raw {
- Value::Array(arr) => Ok(arr
+ Value::Array(arr) => arr
.first()
.map(PainlessValue::from_json)
- .unwrap_or(PainlessValue::Number(0.0))),
+ .ok_or_else(missing),
Value::Number(n) => Ok(PainlessValue::Number(n.as_f64().unwrap_or(0.0))),
Value::String(s) => Ok(PainlessValue::String(s)),
Value::Bool(b) => Ok(PainlessValue::Bool(b)),
- _ => Ok(PainlessValue::Number(0.0)),
+ _ => Err(missing()),
}
}
+ // `.size()` / `.length` / `.empty` are defined on a MISSING field —
+ // they are how a script asks whether the field is there at all, so
+ // they must never error. This is the other half of the fail-closed
+ // contract on `.value` above.
"size" | "length" => {
if args.is_some() {
// doc[...].size() with explicit call
@@ -2829,4 +3009,313 @@ mod tests {
assert!(eval_bool("1 + 1 == 2", &doc));
assert!(eval_bool("doc['color'].value.length() == 5", &doc));
}
+
+ #[test]
+ fn missing_doc_field_value_errors() {
+ // This is ES's own behaviour, not a local hardening choice: since
+ // 7.0 an empty `ScriptDocValues` throws rather than yielding a
+ // default (6.x returned 0/"" with a deprecation warning). Returning
+ // `null` here instead would reopen the hole this closes — see
+ // `comparison_against_missing_field_errors_not_matches_everything`.
+ let doc = json!({});
+ let params = json!({});
+ let r = eval_painless("doc['missing'].value", &ctx(&doc, &params, 0.0));
+ let e = r.expect_err("a missing field must error, matching ES 7+");
+ // The message points at the supported guard, exactly as ES's does.
+ assert!(
+ e.contains("size()==0"),
+ "the error must name the .size()==0 guard: {e}"
+ );
+ }
+
+ #[test]
+ fn comparison_against_missing_field_errors_not_matches_everything() {
+ let doc = json!({});
+ let params = json!({});
+ let r = eval_painless("doc['missing'].value > -1", &ctx(&doc, &params, 0.0));
+ assert!(
+ r.is_err(),
+ "a comparison against a missing field must error, not silently match: got {:?}",
+ r
+ );
+ }
+
+ #[test]
+ fn empty_multi_value_field_errors() {
+ let doc = json!({"tags": []});
+ let params = json!({});
+ let r = eval_painless("doc['tags'].value", &ctx(&doc, &params, 0.0));
+ assert!(r.is_err(), "expected an error, got {:?}", r);
+ }
+
+ #[test]
+ fn present_field_still_works_after_fail_closed_change() {
+ let doc = json!({"x": 10});
+ let params = json!({});
+ let v = eval_painless("doc['x'].value > 5", &ctx(&doc, &params, 0.0)).unwrap();
+ assert!(v.as_bool());
+ }
+
+ #[test]
+ fn exponential_string_doubling_is_bounded_not_a_crash() {
+ let doc = json!({});
+ let params = json!({});
+ // 30 doublings from a 1-byte string would reach ~1 GiB unbounded;
+ // the cap must trip well before that.
+ let src = format!("def s = \"a\";{}return s;", "s = s + s;".repeat(30));
+ let r = eval_painless(&src, &ctx(&doc, &params, 0.0));
+ assert!(
+ r.is_err(),
+ "expected the string-length cap to trip, got {:?}",
+ r
+ );
+ }
+
+ #[test]
+ fn ordinary_string_concatenation_still_works() {
+ let doc = json!({});
+ let params = json!({});
+ let v = eval_painless("'a' + 'b' + 'c'", &ctx(&doc, &params, 0.0)).unwrap();
+ match v {
+ PainlessValue::String(s) => assert_eq!(s, "abc"),
+ other => panic!("expected a string, got {:?}", other),
+ }
+ }
+
+ // ── The null-guard idiom ──────────────────────────────────────────────────
+ // Making a missing field error (above) is only safe if a script can still
+ // ASK whether the field is there. Elasticsearch's answer is
+ // `doc['x'].size() == 0`, which is also the remedy its own exception
+ // message names, so that is the idiom that has to work here. It relies on
+ // two properties, both asserted below: `.size()` is defined on a missing
+ // field, and the ternary evaluates only the branch it takes.
+
+ #[test]
+ fn size_guard_is_defined_on_a_missing_field() {
+ let params = json!({});
+ for (doc, expected_size, expected_empty) in [
+ (json!({}), 0.0, true),
+ (json!({"x": null}), 0.0, true),
+ (json!({"x": []}), 0.0, true),
+ (json!({"x": 7}), 1.0, false),
+ (json!({"x": [1, 2, 3]}), 3.0, false),
+ ] {
+ let c = ctx(&doc, &params, 0.0);
+ let size = eval_painless("doc['x'].size()", &c)
+ .unwrap_or_else(|e| panic!(".size() must never error on {doc}: {e}"));
+ assert_eq!(size.as_f64().unwrap(), expected_size, "doc {doc}");
+ let empty = eval_painless("doc['x'].empty", &c)
+ .unwrap_or_else(|e| panic!(".empty must never error on {doc}: {e}"));
+ assert_eq!(empty.as_bool(), expected_empty, "doc {doc}");
+ }
+ }
+
+ #[test]
+ fn null_guard_idiom_evaluates_on_both_shapes() {
+ // The canonical ES form, verbatim.
+ let src = "doc['x'].size() == 0 ? 0 : doc['x'].value";
+ let params = json!({});
+
+ for (doc, expected) in [
+ (json!({}), 0.0),
+ (json!({"x": []}), 0.0),
+ (json!({"x": 42}), 42.0),
+ (json!({"x": [7, 8]}), 7.0),
+ ] {
+ let v = eval_painless(src, &ctx(&doc, &params, 0.0))
+ .unwrap_or_else(|e| panic!("the null-guard idiom must not error on {doc}: {e}"));
+ assert_eq!(v.as_f64().unwrap(), expected, "doc {doc}");
+ }
+ }
+
+ #[test]
+ fn guarded_script_filters_rather_than_erroring_everywhere() {
+ // The whole point of the idiom: one script, run over documents that
+ // do and don't have the field, returning a boolean for each.
+ let params = json!({"min": 5});
+ let src = "doc['x'].size() == 0 ? false : doc['x'].value > params.min";
+
+ for (doc, expected) in [
+ (json!({"x": 10}), true),
+ (json!({"x": 1}), false),
+ (json!({}), false),
+ ] {
+ let v = eval_painless(src, &ctx(&doc, &params, 0.0))
+ .unwrap_or_else(|e| panic!("guarded script errored on {doc}: {e}"));
+ assert_eq!(v.as_bool(), expected, "doc {doc}");
+ }
+ }
+
+ #[test]
+ fn null_guard_on_a_missing_param_is_a_boolean() {
+ // `params` is a real `Map<String, Object>` in Painless, so a key
+ // that wasn't supplied reads as null and comparing it to null is an
+ // ordinary reference comparison — unlike `doc[...]`, which throws.
+ let doc = json!({});
+
+ let empty = json!({});
+ let v = eval_painless("params.y == null", &ctx(&doc, &empty, 0.0))
+ .expect("a params null guard must evaluate, not error");
+ assert!(v.as_bool(), "missing param: `== null` must be true");
+
+ let supplied = json!({"y": 3});
+ let v = eval_painless("params.y == null", &ctx(&doc, &supplied, 0.0))
+ .expect("a params null guard must evaluate, not error");
+ assert!(!v.as_bool(), "supplied param: `== null` must be false");
+
+ // `!=` is the same comparison inverted, and a present-but-null value
+ // is indistinguishable from an absent one, as in ES.
+ let explicit_null = json!({"y": null});
+ let v = eval_painless("params.y != null", &ctx(&doc, &explicit_null, 0.0)).unwrap();
+ assert!(!v.as_bool());
+ }
+
+ #[test]
+ fn genuinely_invalid_arithmetic_still_errors() {
+ let doc = json!({});
+ let params = json!({"obj": {"a": 1}});
+ // Null-awareness is scoped to `==`/`!=`; nothing else got looser, so
+ // the fail-closed property the script query depends on is intact.
+ for src in [
+ "'abc' - params.obj",
+ "params.missing + 1",
+ "params.missing > 0",
+ "doc['nope'].value * 2",
+ "doc['nope'].value >= 0",
+ "-params.missing",
+ ] {
+ let r = eval_painless(src, &ctx(&doc, &params, 0.0));
+ assert!(r.is_err(), "`{src}` must still error, got {:?}", r);
+ }
+ }
+
+ // ── Compiled-script cache ─────────────────────────────────────────────────
+ // The cache is thread-local, and each `#[test]` gets its own thread, so
+ // every assertion below observes only what its own test evaluated.
+
+ /// `(entries, src_bytes, actual bytes held)` for the calling thread.
+ fn cache_stats() -> (usize, usize, usize) {
+ SCRIPT_CACHE.with(|c| {
+ let c = c.borrow();
+ (
+ c.entries.len(),
+ c.src_bytes,
+ c.entries.keys().map(|k| k.len()).sum(),
+ )
+ })
+ }
+
+ #[test]
+ fn compiled_script_cache_is_bounded_by_entry_count() {
+ let doc = json!({});
+ let params = json!({});
+ // Far more distinct sources than the cache may retain. Each is tiny,
+ // so the entry-count bound is the one under test.
+ for i in 0..(MAX_SCRIPT_CACHE_ENTRIES * 8) {
+ let src = format!("{i} + 1");
+ eval_painless(&src, &ctx(&doc, &params, 0.0)).unwrap();
+ }
+ let (entries, src_bytes, actual) = cache_stats();
+ assert!(
+ entries <= MAX_SCRIPT_CACHE_ENTRIES,
+ "cache grew to {entries} entries, bound is {MAX_SCRIPT_CACHE_ENTRIES}"
+ );
+ assert!(
+ src_bytes <= MAX_SCRIPT_CACHE_SRC_BYTES,
+ "cache holds {src_bytes} source bytes, bound is {MAX_SCRIPT_CACHE_SRC_BYTES}"
+ );
+ // The accounting the bound rests on must match reality.
+ assert_eq!(actual, src_bytes, "src_bytes accounting drifted");
+ }
+
+ #[test]
+ fn compiled_script_cache_is_bounded_by_source_bytes() {
+ let doc = json!({});
+ let params = json!({});
+ // Distinct ~33 KiB scripts (each legal under MAX_SCRIPT_LEN, and
+ // flat rather than deep so they compile and run cleanly). Well under
+ // MAX_SCRIPT_CACHE_ENTRIES of them already exceed the byte budget,
+ // so the byte bound has to be what stops the growth.
+ for i in 0..24 {
+ let src = format!("{}return {i};", "def a = 1;".repeat(3300));
+ assert!(src.len() > 32 * 1024);
+ eval_painless(&src, &ctx(&doc, &params, 0.0)).unwrap();
+ }
+ let (entries, src_bytes, actual) = cache_stats();
+ assert!(
+ src_bytes <= MAX_SCRIPT_CACHE_SRC_BYTES,
+ "cache holds {src_bytes} source bytes, bound is {MAX_SCRIPT_CACHE_SRC_BYTES}"
+ );
+ assert!(entries <= MAX_SCRIPT_CACHE_ENTRIES);
+ assert!(
+ entries < 24,
+ "the byte bound must have evicted something: {entries} entries retained"
+ );
+ assert_eq!(actual, src_bytes, "src_bytes accounting drifted");
+ }
+
+ #[test]
+ fn cached_script_is_parsed_once_and_still_correct() {
+ let params = json!({"multiplier": 3});
+ let src = "doc['n'].value * params.multiplier /* cache-once probe */";
+ for n in 1..=5 {
+ let doc = json!({ "n": n });
+ let v = eval_painless(src, &ctx(&doc, &params, 0.0)).unwrap();
+ assert!((v.as_f64().unwrap() - (n * 3) as f64).abs() < 1e-9);
+ }
+ assert!(
+ SCRIPT_CACHE.with(|c| c.borrow().entries.contains_key(src)),
+ "repeated evaluation of one source must leave a cached AST"
+ );
+ assert_eq!(
+ cache_stats().0,
+ 1,
+ "five evaluations of one source must leave exactly one entry"
+ );
+ }
+
+ #[test]
+ fn cached_parse_failure_still_reports_the_same_error() {
+ let doc = json!({});
+ let params = json!({});
+ let src = "return ( ( ( /* unbalanced, cache-error probe */";
+ let first = eval_painless(src, &ctx(&doc, &params, 0.0)).unwrap_err();
+ let second = eval_painless(src, &ctx(&doc, &params, 0.0)).unwrap_err();
+ assert_eq!(first, second);
+ }
+
+ #[test]
+ fn caching_does_not_share_state_between_evaluations() {
+ // The AST is shared but mutable evaluation state is not: a script
+ // that mutates a local must start from scratch on every document.
+ let params = json!({});
+ let src = "def acc = 0; acc = acc + doc['n'].value; \
+ acc = acc + doc['n'].value; acc = acc + doc['n'].value; \
+ return acc; /* shared-state probe */";
+ for n in 1..=4 {
+ let doc = json!({ "n": n });
+ let v = eval_painless(src, &ctx(&doc, &params, 0.0)).unwrap();
+ assert_eq!(v.as_f64().unwrap(), (n * 3) as f64, "n = {n}");
+ }
+ }
+
+ #[test]
+ fn cached_ast_does_not_leak_the_call_budget_across_evaluations() {
+ // MAX_CALL_COUNT lives on the per-evaluation `PainlessCtx`, not on
+ // the AST, so memoising the AST must not let one document's closure
+ // invocations count against the next document's budget. A script
+ // that spends a large slice of the budget has to keep succeeding
+ // when it is re-run against document after document.
+ let doc = json!({});
+ let params = json!({});
+ // 2^11 = 2048 invocations per evaluation, comfortably under
+ // MAX_CALL_COUNT (10_000) but far more than a per-process budget
+ // would survive being run 8 times.
+ let src = "def f = (g, n) -> { if (n <= 0) { return 1; } return g(g, n - 1) + g(g, n - 1); }; return f(f, 11); /* budget-reset probe */";
+ for round in 0..8 {
+ let v = eval_painless(src, &ctx(&doc, &params, 0.0))
+ .unwrap_or_else(|e| panic!("round {round} must stay within budget: {e}"));
+ assert!((v.as_f64().unwrap() - 2048.0).abs() < 1e-9, "round {round}");
+ }
+ }
}
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index af8787ee0..fb99d2fa0 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -377,6 +377,18 @@ pub enum QueryNode {
/// Matches documents whose `_id` is in the given list.
Ids { values: Vec<String> },
+ /// A standalone Painless `script` query — matches when the script
+ /// (given the doc's `_source` via `doc`/`params`) evaluates truthy.
+ /// Distinct from `script_score` (a `function_score` scoring function):
+ /// this one is a boolean filter, no scoring involved. Used e.g. by
+ /// OpenSearch's UBI sample dashboards ("Top Searches Without Results")
+ /// to filter on a computed field via a boolean Supplier-lambda script.
+ Script {
+ source: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ params: Option<serde_json::Value>,
+ },
+
// ── Full-text ─────────────────────────────────────────────────────────────
/// Analysed text query against a single field.
Match {
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index dde49a216..f56870abf 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -296,6 +296,7 @@ pub fn parse_query(json: &Value) -> Result<QueryNode> {
"wildcard" => parse_wildcard(params),
"exists" => parse_exists(params),
"ids" => parse_ids(params),
+ "script" => parse_script(params),
"bool" => parse_bool(params),
"query_string" => parse_query_string(params),
"constant_score" => parse_constant_score(params),
@@ -1093,6 +1094,23 @@ fn parse_ids(params: &Value) -> Result<QueryNode> {
Ok(node)
}
+fn parse_script(params: &Value) -> Result<QueryNode> {
+ let obj = params
+ .as_object()
+ .ok_or_else(|| qerr("`script` must be an object"))?;
+ let script = obj
+ .get("script")
+ .and_then(Value::as_object)
+ .ok_or_else(|| qerr("`script.script` is required"))?;
+ let source = script
+ .get("source")
+ .and_then(Value::as_str)
+ .ok_or_else(|| qerr("`script.script.source` is required"))?
+ .to_string();
+ let params = script.get("params").cloned();
+ Ok(QueryNode::Script { source, params })
+}
+
fn parse_query_string(params: &Value) -> Result<QueryNode> {
let obj = params
.as_object()
@@ -4348,6 +4366,43 @@ mod tests {
}
}
+ // ── script ────────────────────────────────────────────────────────────────
+
+ #[test]
+ fn test_script_query() {
+ let node = q(json!({
+ "script": {
+ "script": {
+ "source": "doc['x'].value > params.min",
+ "params": {"min": 5}
+ }
+ }
+ }));
+ if let QueryNode::Script { source, params } = node {
+ assert_eq!(source, "doc['x'].value > params.min");
+ assert_eq!(params, Some(json!({"min": 5})));
+ } else {
+ panic!("wrong variant");
+ }
+ }
+
+ #[test]
+ fn test_script_query_no_params() {
+ let node = q(json!({"script": {"script": {"source": "true"}}}));
+ if let QueryNode::Script { source, params } = node {
+ assert_eq!(source, "true");
+ assert_eq!(params, None);
+ } else {
+ panic!("wrong variant");
+ }
+ }
+
+ #[test]
+ fn test_script_query_missing_source_errs() {
+ let result = parse_query(&json!({"script": {"script": {}}}));
+ assert!(result.is_err());
+ }
+
// ── bool ──────────────────────────────────────────────────────────────────
#[test]
diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rs
index ffda47fa0..f84402527 100644
--- a/engine/crates/xerj-query/src/planner.rs
+++ b/engine/crates/xerj-query/src/planner.rs
@@ -590,6 +590,10 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
// engine's doc-scan path; MatchAll satisfies exhaustiveness here.
QueryNode::Percolate { .. } => ExecutionPlan::MatchAll,
+ // Script: no index can back an arbitrary Painless predicate — pure
+ // doc-scan, same as Percolate above.
+ QueryNode::Script { .. } => ExecutionPlan::MatchAll,
+
// ── Span queries — treat as doc-scans ────────────────────────────────
QueryNode::SpanTerm { field, value } => ExecutionPlan::FtsScan {
field,