AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(query): scope the slop claim to the phrase entry points; coerce slop like ES
xerj-orga2cfc6ebd215
diff --git a/engine/crates/xerj-engine/tests/multi_match_phrase_positions.rs b/engine/crates/xerj-engine/tests/multi_match_phrase_positions.rsindex 579d8e968..007861e1e 100644--- a/engine/crates/xerj-engine/tests/multi_match_phrase_positions.rs+++ b/engine/crates/xerj-engine/tests/multi_match_phrase_positions.rs@@ -31,6 +31,7 @@ use tempfile::TempDir;use xerj_common::config::Config;use xerj_common::types::Schema;use xerj_engine::{Engine, Index};+use xerj_query::ast::QueryNode;use xerj_query::parse_request;fn make_engine(dir: &TempDir) -> Engine {@@ -357,12 +358,22 @@ async fn phrase_prefix_slop_is_honoured_at_both_entry_points() {.await;}-/// A negative `slop` is a client error in ES from EVERY builder, so every-/// entry point must answer alike — a 400 from one spelling and a silently-/// coerced 0 from the other is the same accept-and-ignore defect wearing a-/// different hat.+/// A negative `slop` is a client error in ES from all three *phrase* builders+/// (`MatchPhraseQueryBuilder` / `MatchPhrasePrefixQueryBuilder:103` /+/// `MultiMatchQueryBuilder:350`, all "No negative slop allowed"), so all three+/// must answer alike here — a 400 from one spelling and a silently coerced 0+/// from the other is the same accept-and-ignore defect wearing a different hat.+///+/// SCOPE, stated so the name cannot be read as more than it is: these three+/// query types are the whole of it. `span_near` also takes a `slop` and is+/// **not** covered — it keeps `as_u64().unwrap_or(0)`, so `span_near` with+/// `slop: -1` returns 200 with the value silently 0. That is a pre-existing+/// gap, recorded at `parse_span_near`, and it is left alone because ES applies+/// no validation to `span_near.slop` either+/// (`SpanNearQueryBuilder.java:62`) — rejecting it would break a query ES+/// answers.#[test]-fn negative_slop_is_rejected_at_every_entry_point() {+fn negative_slop_is_rejected_at_the_three_phrase_entry_points() {for bad in [json!({"multi_match": {"query": "merge policy", "fields": ["body"],"type": "phrase", "slop": -1}}),@@ -378,6 +389,54 @@ fn negative_slop_is_rejected_at_every_entry_point() {}}+/// A float-encoded `slop` (`2.0`) must be honoured as 2, not refused and not+/// dropped. ES reads slop with `XContentParser.intValue()` under its default+/// coercing policy, which truncates a float token and narrows a numeric string+/// (`AbstractXContentParser.java:74`/`:162`/`:177`), so `{"slop": 2.0}` is a+/// query ES answers with slop 2.+///+/// This test exists because an earlier revision of this branch tightened the+/// parse to `as_i64()` and turned `2.0` into a 400 — a new refusal on a+/// request `main` answered, which is exactly the wire break this PR declined+/// to make elsewhere. The two wrong answers are symmetrical: refusing the+/// value, or silently substituting 0 as `main` did.+#[test]+fn float_and_string_encoded_slop_are_coerced_like_es() {+ for (body, want) in [+ (json!({"query": "merge policy", "slop": 2.0}), 2u32),+ (json!({"query": "merge policy", "slop": 2.7}), 2),+ (json!({"query": "merge policy", "slop": "2"}), 2),+ ] {+ let q = json!({"match_phrase": {"body": body.clone()}});+ let parsed = parse_request(&json!({ "query": q }))+ .unwrap_or_else(|e| panic!("ES coerces this slop, so it must parse: {body} -> {e}"));+ match parsed.query {+ QueryNode::MatchPhrase { slop, .. } => {+ assert_eq!(slop, want, "expected slop {want} from {body}, got {slop}")+ }+ other => panic!("expected a MatchPhrase node, got {other:?}"),+ }+ }++ // The same coercion, and the same refusal to ignore, on `max_expansions`:+ // a float is read, and a value that cannot be read is a 400 rather than+ // a silent fall back to 50 (which is what `main` did here).+ let parsed = parse_request(&json!({"query": {"match_phrase_prefix": {+ "body": {"query": "merge poli", "max_expansions": 7.0}}}}))+ .expect("float max_expansions is coerced, not refused");+ match parsed.query {+ QueryNode::MatchPhrasePrefix { max_expansions, .. } => assert_eq!(max_expansions, 7),+ other => panic!("expected a MatchPhrasePrefix node, got {other:?}"),+ }+ for bad in [+ json!({"match_phrase_prefix": {"body": {"query": "merge poli", "max_expansions": -1}}}),+ json!({"match_phrase_prefix": {"body": {"query": "merge poli",+ "max_expansions": "seven"}}}),+ ] {+ parse_request(&json!({ "query": bad.clone() })).unwrap_err();+ }+}+/// The stored-scan/memtable evaluator and the positional segment clause/// must tokenize with the SAME analyzer, or they answer different queries/// and the hit set changes at `_flush` — the regression class #218/#222diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rsindex d0ebf8dc0..43f8560d4 100644--- a/engine/crates/xerj-query/src/parser.rs+++ b/engine/crates/xerj-query/src/parser.rs@@ -557,14 +557,53 @@ fn parse_match_phrase(params: &Value) -> Result<QueryNode> {Ok(maybe_named(node, name))}-/// Parse the `slop` parameter of a phrase-shaped query, identically for every-/// entry point that accepts one (`match_phrase`, `match_phrase_prefix`,-/// `multi_match`). Sharing it is the point: the same JSON value must produce-/// the same answer whichever query wraps it, or a client gets a 400 from one-/// form and a silently different answer from the semantically identical other-/// form.+/// Coerce a JSON scalar to an integer the way ES's `XContentParser.intValue()`+/// does under its default number policy (`DEFAULT_NUMBER_COERCE_POLICY = true`):+/// an integer is taken as written, a **float is truncated** toward zero, and a+/// numeric **string** is read as a double and then narrowed+/// (`libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java:74`+/// — "the 3rd party parsers we rely on are known to silently truncate+/// fractions" — plus `:162` `parseInt` and `:177` `intValue`; AGPL, read for+/// semantics only, nothing copied).///-/// ES rejects a negative slop from every builder+/// Why it exists: JSON encoders that carry every number as a double emit+/// `"slop": 2.0`, and ES answers that query with slop 2. Reading it with+/// `as_i64()` alone leaves two bad options — a 400 on a request ES answers, or+/// a silent fall back to the default, which is the accept-and-ignore (#204)+/// this parser exists to remove.+///+/// The `as i64` cast saturates on an out-of-range float, so an absurd value+/// lands on the bound and the caller's range check turns it into a 400 rather+/// than a wrapped, silently honoured small number.+fn coerce_es_int(v: &Value) -> Option<i64> {+ if let Some(n) = v.as_i64() {+ return Some(n);+ }+ let d = match v {+ Value::Number(_) => v.as_f64()?,+ Value::String(s) => s.trim().parse::<f64>().ok()?,+ _ => return None,+ };+ d.is_finite().then(|| d.trunc() as i64)+}++/// Parse the `slop` parameter of a phrase-shaped query, identically at each of+/// the three entry points that accept one and lower it to a phrase:+/// `match_phrase`, `match_phrase_prefix`, `multi_match`. Sharing it is the+/// point: the same JSON value must produce the same answer whichever of those+/// three wraps it, or a client gets a 400 from one spelling and a silently+/// different answer from the semantically identical other.+///+/// Deliberately **not** covered: `span_near`, which also takes a `slop` and+/// keeps its own `as_u64().unwrap_or(0)` read (see `parse_span_near`). ES+/// applies no validation there either — `SpanNearQueryBuilder` stores the int+/// unchecked (`.../index/query/SpanNearQueryBuilder.java:62`), so a negative+/// `span_near` slop is a query ES *answers*, and routing it through here would+/// turn a 200 into a 400. XERJ still coerces it to 0 instead of honouring it,+/// which is a real (pre-existing) gap — recorded in `parse_span_near`, not+/// papered over here.+///+/// ES rejects a negative slop from all three phrase builders/// (`MatchPhraseQueryBuilder.slop` / `MatchPhrasePrefixQueryBuilder.slop:103`/// / `MultiMatchQueryBuilder.slop:350` all throw "No negative slop allowed";/// AGPL, read for semantics only). `u32::try_from` rather than `as u32`@@ -573,17 +612,43 @@ fn parse_match_phrase(params: &Value) -> Result<QueryNode> {fn parse_phrase_slop(v: Option<&Value>, ctx: &str) -> Result<u32> {match v {None | Some(Value::Null) => Ok(0),- Some(v) => match v- .as_i64()- .or_else(|| v.as_str().and_then(|s| s.parse().ok()))- {+ Some(v) => match coerce_es_int(v) {Some(n) if n < 0 => invalid(format!("`{ctx}.slop` must not be negative")),Some(n) => u32::try_from(n).map_err(|_| qerr(format!("`{ctx}.slop` is out of range"))),- None => invalid(format!("`{ctx}.slop` must be an integer")),+ None => invalid(format!("`{ctx}.slop` must be a number")),+ },+ }+}++/// Parse `max_expansions`, with the same refusal-to-ignore as+/// [`parse_phrase_slop`]: a value XERJ cannot read is a 400, never a silent+/// fall back to the default of 50.+///+/// The floor differs between the two entry points, and that asymmetry is ES's+/// rather than an oversight here: `MultiMatchQueryBuilder.maxExpansions`+/// throws on `<= 0` (`.../index/query/MultiMatchQueryBuilder.java:389`) while+/// `MatchPhrasePrefixQueryBuilder.maxExpansions` throws only on `< 0`+/// (`.../index/query/MatchPhrasePrefixQueryBuilder.java:118`, "No negative+/// maxExpansions allowed"). So `0` is a 400 on `multi_match` and an accepted+/// (expand-nothing) value on `match_phrase_prefix`; `allow_zero` carries that+/// distinction instead of us inventing one ES does not have.+fn parse_max_expansions(v: Option<&Value>, ctx: &str, allow_zero: bool) -> Result<u32> {+ match v {+ None | Some(Value::Null) => Ok(DEFAULT_MAX_EXPANSIONS),+ Some(v) => match coerce_es_int(v) {+ Some(n) if n < 0 => invalid(format!("`{ctx}.max_expansions` must not be negative")),+ Some(0) if !allow_zero => invalid(format!("`{ctx}.max_expansions` must be positive")),+ Some(n) => u32::try_from(n)+ .map_err(|_| qerr(format!("`{ctx}.max_expansions` is out of range"))),+ None => invalid(format!("`{ctx}.max_expansions` must be a number")),},}}+/// ES's `FuzzyQuery.defaultMaxExpansions`, which both phrase-prefix builders+/// default to.+const DEFAULT_MAX_EXPANSIONS: u32 = 50;+/// ES `combined_fields` query — introduced in 7.13, treats N text fields as a/// single virtual field for term-frequency scoring. Our approximation routes/// it to `multi_match` with `type: "cross_fields"`, which is the closest@@ -636,20 +701,8 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {// ES: negative slop and non-positive max_expansions are 400s, and// `slop` is not allowed with `type: bool_prefix`.let slop: u32 = parse_phrase_slop(obj.get("slop"), "multi_match")?;- let max_expansions: u32 = match obj.get("max_expansions") {- None | Some(Value::Null) => 50,- Some(v) => match v- .as_i64()- .or_else(|| v.as_str().and_then(|s| s.parse().ok()))- {- Some(n) if n <= 0 => return invalid("`multi_match.max_expansions` must be positive"),- Some(n) => match u32::try_from(n) {- Ok(m) => m,- Err(_) => return invalid("`multi_match.max_expansions` is out of range"),- },- None => return invalid("`multi_match.max_expansions` must be an integer"),- },- };+ let max_expansions: u32 =+ parse_max_expansions(obj.get("max_expansions"), "multi_match", false)?;if slop != 0 && type_str == "bool_prefix" {// ES's own wording — `310_match_bool_prefix.yml` catches this exact// regex. The suite's runner only checks that the call is allowed to@@ -2008,10 +2061,17 @@ fn parse_match_phrase_prefix(params: &Value) -> Result<QueryNode> {.get("query").and_then(scalar_to_string).ok_or_else(|| qerr("`match_phrase_prefix.query` must be a non-empty scalar"))?;- let max_expansions = inner- .get("max_expansions")- .and_then(Value::as_u64)- .unwrap_or(50) as u32;+ // `max_expansions` used to be read with `as_u64().unwrap_or(50)`, so+ // `-1`, `"7"` and `7.0` all silently became 50 — the same accept-and-ignore+ // (#204) as the dropped `slop` below, one line apart. It is now the shared+ // parse: a value we cannot read is a 400, and `0` stays accepted because+ // ES accepts it *here* (it rejects it on `multi_match`; the asymmetry is+ // ES's own, see `parse_max_expansions`).+ let max_expansions = parse_max_expansions(+ inner.get("max_expansions"),+ "match_phrase_prefix",+ /* allow_zero */ true,+ )?;// `slop` used to be read nowhere here, so// `{"match_phrase_prefix": {"f": {"query": …, "slop": 2}}}` was accepted// and answered as slop 0 — accept-and-ignore (#204 class), and a silent@@ -3467,6 +3527,20 @@ fn parse_span_near(params: &Value) -> Result<QueryNode> {return Ok(QueryNode::MatchNone);}+ // KNOWN GAP (pre-existing, deliberately not changed by #230): this read+ // accepts anything and silently substitutes 0 — `slop: -1`, `slop: 2.0`+ // and `slop: "2"` all answer as slop 0. It is NOT routed through+ // `parse_phrase_slop`, and the phrase work in #230 does not cover it.+ //+ // It is left alone rather than "fixed" because ES applies no validation+ // here either: `SpanNearQueryBuilder` takes the int unchecked+ // (`.../index/query/SpanNearQueryBuilder.java:62`, no negative test)+ // while every phrase builder throws on a negative slop. So a negative+ // `span_near` slop is a query ES answers, and rejecting it would be a+ // wire-compat break on a request that works today. Closing it properly+ // means honouring the value (float coercion via `coerce_es_int`, and a+ // decision about what a negative span slop should match), which is a+ // span-query change, not a phrase one.let slop = obj.get("slop").and_then(|v| v.as_u64()).unwrap_or(0) as u32;let in_order = obj.get("in_order")@@ -4498,6 +4572,13 @@ mod tests {json!({"match_phrase": {"body": {"query": "a b", "slop": -1}}}),json!({"match_phrase_prefix": {"body": {"query": "a b", "slop": -1}}}),json!({"match_phrase_prefix": {"body": {"query": "a b", "slop": 4294967296i64}}}),+ // `max_expansions` used to be read here as+ // `as_u64().unwrap_or(50)`, so both of these were accepted and+ // silently answered with 50 (#204 class). ES throws "No negative+ // maxExpansions allowed" (MatchPhrasePrefixQueryBuilder.java:118).+ json!({"match_phrase_prefix": {"body": {"query": "a b", "max_expansions": -1}}}),+ json!({"match_phrase_prefix": {"body": {"query": "a b", "max_expansions": "many"}}}),+ json!({"match_phrase": {"body": {"query": "a b", "slop": "two"}}}),] {assert!(parse_query(&bad).is_err(),@@ -4506,6 +4587,50 @@ mod tests {}}+ /// Values ES *accepts by coercing* must be accepted here too. ES reads+ /// `slop` and `max_expansions` with `XContentParser.intValue()` under the+ /// default coercing policy, which truncates a float token and narrows a+ /// numeric string (`AbstractXContentParser.java:74`/`:162`/`:177`), so+ /// `{"slop": 2.0}` is a request ES answers with slop 2. Refusing it would+ /// be a new 400 on a query that works today, and reading it as 0 would be+ /// the accept-and-ignore this parser exists to remove — both are wrong.+ #[test]+ fn test_phrase_params_are_coerced_like_es() {+ for (v, want) in [(json!(2.0), 2u32), (json!(2.9), 2), (json!("2"), 2)] {+ let node = q(json!({"match_phrase": {"body": {"query": "a b", "slop": v}}}));+ assert!(+ matches!(node, QueryNode::MatchPhrase { slop, .. } if slop == want),+ "slop {v} must coerce to {want}, parsed to {node:?}"+ );+ let node = q(json!({"multi_match": {"query": "a b", "fields": ["body"],+ "type": "phrase", "slop": v}}));+ assert!(+ matches!(node, QueryNode::MultiMatch { slop, .. } if slop == want),+ "multi_match slop {v} must coerce to {want}, parsed to {node:?}"+ );+ }++ // `max_expansions: 0` is a 400 on `multi_match`+ // (MultiMatchQueryBuilder.java:389 rejects `<= 0`) and ACCEPTED on+ // `match_phrase_prefix` (MatchPhrasePrefixQueryBuilder.java:118+ // rejects only `< 0`). The asymmetry is ES's; XERJ mirrors it rather+ // than inventing a rule ES does not have.+ let node = q(json!({"match_phrase_prefix": {"body": {"query": "a b",+ "max_expansions": 0}}}));+ assert!(+ matches!(node, QueryNode::MatchPhrasePrefix { max_expansions, .. }+ if max_expansions == 0),+ "ES accepts max_expansions 0 on match_phrase_prefix, parsed to {node:?}"+ );+ assert!(+ parse_query(&json!({"multi_match": {"query": "a b", "fields": ["body"],+ "type": "phrase_prefix",+ "max_expansions": 0}}))+ .is_err(),+ "ES rejects max_expansions 0 on multi_match"+ );+ }+/// `slop` + `type: phrase_prefix` is ACCEPTED and carried, at BOTH/// entry points. ES honours it (`MultiMatchQueryBuilder` rejects slop/// only for `bool_prefix`; `TextFieldMapper.createPhrasePrefixQuery`