Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(query): scope the slop claim to the phrase entry points; coerce slop like ES

xerj-orga2cfc6ebd215
diff --git a/engine/crates/xerj-engine/tests/multi_match_phrase_positions.rs b/engine/crates/xerj-engine/tests/multi_match_phrase_positions.rs
index 579d8e968..007861e1e 100644
--- a/engine/crates/xerj-engine/tests/multi_match_phrase_positions.rs
+++ b/engine/crates/xerj-engine/tests/multi_match_phrase_positions.rs
@@ -31,6 +31,7 @@ use tempfile::TempDir;
use xerj_common::config::Config;
use xerj_common::types::Schema;
use xerj_engine::{Engine, Index};
+use xerj_query::ast::QueryNode;
use xerj_query::parse_request;
fn make_engine(dir: &TempDir) -> Engine {
@@ -357,12 +358,22 @@ async fn phrase_prefix_slop_is_honoured_at_both_entry_points() {
.await;
}
-/// A negative `slop` is a client error in ES from EVERY builder, so every
-/// entry point must answer alike — a 400 from one spelling and a silently
-/// coerced 0 from the other is the same accept-and-ignore defect wearing a
-/// different hat.
+/// A negative `slop` is a client error in ES from all three *phrase* builders
+/// (`MatchPhraseQueryBuilder` / `MatchPhrasePrefixQueryBuilder:103` /
+/// `MultiMatchQueryBuilder:350`, all "No negative slop allowed"), so all three
+/// must answer alike here — a 400 from one spelling and a silently coerced 0
+/// from the other is the same accept-and-ignore defect wearing a different hat.
+///
+/// SCOPE, stated so the name cannot be read as more than it is: these three
+/// query types are the whole of it. `span_near` also takes a `slop` and is
+/// **not** covered — it keeps `as_u64().unwrap_or(0)`, so `span_near` with
+/// `slop: -1` returns 200 with the value silently 0. That is a pre-existing
+/// gap, recorded at `parse_span_near`, and it is left alone because ES applies
+/// no validation to `span_near.slop` either
+/// (`SpanNearQueryBuilder.java:62`) — rejecting it would break a query ES
+/// answers.
#[test]
-fn negative_slop_is_rejected_at_every_entry_point() {
+fn negative_slop_is_rejected_at_the_three_phrase_entry_points() {
for bad in [
json!({"multi_match": {"query": "merge policy", "fields": ["body"],
"type": "phrase", "slop": -1}}),
@@ -378,6 +389,54 @@ fn negative_slop_is_rejected_at_every_entry_point() {
}
}
+/// A float-encoded `slop` (`2.0`) must be honoured as 2, not refused and not
+/// dropped. ES reads slop with `XContentParser.intValue()` under its default
+/// coercing policy, which truncates a float token and narrows a numeric string
+/// (`AbstractXContentParser.java:74`/`:162`/`:177`), so `{"slop": 2.0}` is a
+/// query ES answers with slop 2.
+///
+/// This test exists because an earlier revision of this branch tightened the
+/// parse to `as_i64()` and turned `2.0` into a 400 — a new refusal on a
+/// request `main` answered, which is exactly the wire break this PR declined
+/// to make elsewhere. The two wrong answers are symmetrical: refusing the
+/// value, or silently substituting 0 as `main` did.
+#[test]
+fn float_and_string_encoded_slop_are_coerced_like_es() {
+ for (body, want) in [
+ (json!({"query": "merge policy", "slop": 2.0}), 2u32),
+ (json!({"query": "merge policy", "slop": 2.7}), 2),
+ (json!({"query": "merge policy", "slop": "2"}), 2),
+ ] {
+ let q = json!({"match_phrase": {"body": body.clone()}});
+ let parsed = parse_request(&json!({ "query": q }))
+ .unwrap_or_else(|e| panic!("ES coerces this slop, so it must parse: {body} -> {e}"));
+ match parsed.query {
+ QueryNode::MatchPhrase { slop, .. } => {
+ assert_eq!(slop, want, "expected slop {want} from {body}, got {slop}")
+ }
+ other => panic!("expected a MatchPhrase node, got {other:?}"),
+ }
+ }
+
+ // The same coercion, and the same refusal to ignore, on `max_expansions`:
+ // a float is read, and a value that cannot be read is a 400 rather than
+ // a silent fall back to 50 (which is what `main` did here).
+ let parsed = parse_request(&json!({"query": {"match_phrase_prefix": {
+ "body": {"query": "merge poli", "max_expansions": 7.0}}}}))
+ .expect("float max_expansions is coerced, not refused");
+ match parsed.query {
+ QueryNode::MatchPhrasePrefix { max_expansions, .. } => assert_eq!(max_expansions, 7),
+ other => panic!("expected a MatchPhrasePrefix node, got {other:?}"),
+ }
+ for bad in [
+ json!({"match_phrase_prefix": {"body": {"query": "merge poli", "max_expansions": -1}}}),
+ json!({"match_phrase_prefix": {"body": {"query": "merge poli",
+ "max_expansions": "seven"}}}),
+ ] {
+ parse_request(&json!({ "query": bad.clone() })).unwrap_err();
+ }
+}
+
/// The stored-scan/memtable evaluator and the positional segment clause
/// must tokenize with the SAME analyzer, or they answer different queries
/// and the hit set changes at `_flush` — the regression class #218/#222
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index d0ebf8dc0..43f8560d4 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -557,14 +557,53 @@ fn parse_match_phrase(params: &Value) -> Result<QueryNode> {
Ok(maybe_named(node, name))
}
-/// Parse the `slop` parameter of a phrase-shaped query, identically for every
-/// entry point that accepts one (`match_phrase`, `match_phrase_prefix`,
-/// `multi_match`). Sharing it is the point: the same JSON value must produce
-/// the same answer whichever query wraps it, or a client gets a 400 from one
-/// form and a silently different answer from the semantically identical other
-/// form.
+/// Coerce a JSON scalar to an integer the way ES's `XContentParser.intValue()`
+/// does under its default number policy (`DEFAULT_NUMBER_COERCE_POLICY = true`):
+/// an integer is taken as written, a **float is truncated** toward zero, and a
+/// numeric **string** is read as a double and then narrowed
+/// (`libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java:74`
+/// — "the 3rd party parsers we rely on are known to silently truncate
+/// fractions" — plus `:162` `parseInt` and `:177` `intValue`; AGPL, read for
+/// semantics only, nothing copied).
///
-/// ES rejects a negative slop from every builder
+/// Why it exists: JSON encoders that carry every number as a double emit
+/// `"slop": 2.0`, and ES answers that query with slop 2. Reading it with
+/// `as_i64()` alone leaves two bad options — a 400 on a request ES answers, or
+/// a silent fall back to the default, which is the accept-and-ignore (#204)
+/// this parser exists to remove.
+///
+/// The `as i64` cast saturates on an out-of-range float, so an absurd value
+/// lands on the bound and the caller's range check turns it into a 400 rather
+/// than a wrapped, silently honoured small number.
+fn coerce_es_int(v: &Value) -> Option<i64> {
+ if let Some(n) = v.as_i64() {
+ return Some(n);
+ }
+ let d = match v {
+ Value::Number(_) => v.as_f64()?,
+ Value::String(s) => s.trim().parse::<f64>().ok()?,
+ _ => return None,
+ };
+ d.is_finite().then(|| d.trunc() as i64)
+}
+
+/// Parse the `slop` parameter of a phrase-shaped query, identically at each of
+/// the three entry points that accept one and lower it to a phrase:
+/// `match_phrase`, `match_phrase_prefix`, `multi_match`. Sharing it is the
+/// point: the same JSON value must produce the same answer whichever of those
+/// three wraps it, or a client gets a 400 from one spelling and a silently
+/// different answer from the semantically identical other.
+///
+/// Deliberately **not** covered: `span_near`, which also takes a `slop` and
+/// keeps its own `as_u64().unwrap_or(0)` read (see `parse_span_near`). ES
+/// applies no validation there either — `SpanNearQueryBuilder` stores the int
+/// unchecked (`.../index/query/SpanNearQueryBuilder.java:62`), so a negative
+/// `span_near` slop is a query ES *answers*, and routing it through here would
+/// turn a 200 into a 400. XERJ still coerces it to 0 instead of honouring it,
+/// which is a real (pre-existing) gap — recorded in `parse_span_near`, not
+/// papered over here.
+///
+/// ES rejects a negative slop from all three phrase builders
/// (`MatchPhraseQueryBuilder.slop` / `MatchPhrasePrefixQueryBuilder.slop:103`
/// / `MultiMatchQueryBuilder.slop:350` all throw "No negative slop allowed";
/// AGPL, read for semantics only). `u32::try_from` rather than `as u32`
@@ -573,17 +612,43 @@ fn parse_match_phrase(params: &Value) -> Result<QueryNode> {
fn parse_phrase_slop(v: Option<&Value>, ctx: &str) -> Result<u32> {
match v {
None | Some(Value::Null) => Ok(0),
- Some(v) => match v
- .as_i64()
- .or_else(|| v.as_str().and_then(|s| s.parse().ok()))
- {
+ Some(v) => match coerce_es_int(v) {
Some(n) if n < 0 => invalid(format!("`{ctx}.slop` must not be negative")),
Some(n) => u32::try_from(n).map_err(|_| qerr(format!("`{ctx}.slop` is out of range"))),
- None => invalid(format!("`{ctx}.slop` must be an integer")),
+ None => invalid(format!("`{ctx}.slop` must be a number")),
+ },
+ }
+}
+
+/// Parse `max_expansions`, with the same refusal-to-ignore as
+/// [`parse_phrase_slop`]: a value XERJ cannot read is a 400, never a silent
+/// fall back to the default of 50.
+///
+/// The floor differs between the two entry points, and that asymmetry is ES's
+/// rather than an oversight here: `MultiMatchQueryBuilder.maxExpansions`
+/// throws on `<= 0` (`.../index/query/MultiMatchQueryBuilder.java:389`) while
+/// `MatchPhrasePrefixQueryBuilder.maxExpansions` throws only on `< 0`
+/// (`.../index/query/MatchPhrasePrefixQueryBuilder.java:118`, "No negative
+/// maxExpansions allowed"). So `0` is a 400 on `multi_match` and an accepted
+/// (expand-nothing) value on `match_phrase_prefix`; `allow_zero` carries that
+/// distinction instead of us inventing one ES does not have.
+fn parse_max_expansions(v: Option<&Value>, ctx: &str, allow_zero: bool) -> Result<u32> {
+ match v {
+ None | Some(Value::Null) => Ok(DEFAULT_MAX_EXPANSIONS),
+ Some(v) => match coerce_es_int(v) {
+ Some(n) if n < 0 => invalid(format!("`{ctx}.max_expansions` must not be negative")),
+ Some(0) if !allow_zero => invalid(format!("`{ctx}.max_expansions` must be positive")),
+ Some(n) => u32::try_from(n)
+ .map_err(|_| qerr(format!("`{ctx}.max_expansions` is out of range"))),
+ None => invalid(format!("`{ctx}.max_expansions` must be a number")),
},
}
}
+/// ES's `FuzzyQuery.defaultMaxExpansions`, which both phrase-prefix builders
+/// default to.
+const DEFAULT_MAX_EXPANSIONS: u32 = 50;
+
/// ES `combined_fields` query — introduced in 7.13, treats N text fields as a
/// single virtual field for term-frequency scoring. Our approximation routes
/// it to `multi_match` with `type: "cross_fields"`, which is the closest
@@ -636,20 +701,8 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
// ES: negative slop and non-positive max_expansions are 400s, and
// `slop` is not allowed with `type: bool_prefix`.
let slop: u32 = parse_phrase_slop(obj.get("slop"), "multi_match")?;
- let max_expansions: u32 = match obj.get("max_expansions") {
- None | Some(Value::Null) => 50,
- Some(v) => match v
- .as_i64()
- .or_else(|| v.as_str().and_then(|s| s.parse().ok()))
- {
- Some(n) if n <= 0 => return invalid("`multi_match.max_expansions` must be positive"),
- Some(n) => match u32::try_from(n) {
- Ok(m) => m,
- Err(_) => return invalid("`multi_match.max_expansions` is out of range"),
- },
- None => return invalid("`multi_match.max_expansions` must be an integer"),
- },
- };
+ let max_expansions: u32 =
+ parse_max_expansions(obj.get("max_expansions"), "multi_match", false)?;
if slop != 0 && type_str == "bool_prefix" {
// ES's own wording — `310_match_bool_prefix.yml` catches this exact
// regex. The suite's runner only checks that the call is allowed to
@@ -2008,10 +2061,17 @@ fn parse_match_phrase_prefix(params: &Value) -> Result<QueryNode> {
.get("query")
.and_then(scalar_to_string)
.ok_or_else(|| qerr("`match_phrase_prefix.query` must be a non-empty scalar"))?;
- let max_expansions = inner
- .get("max_expansions")
- .and_then(Value::as_u64)
- .unwrap_or(50) as u32;
+ // `max_expansions` used to be read with `as_u64().unwrap_or(50)`, so
+ // `-1`, `"7"` and `7.0` all silently became 50 — the same accept-and-ignore
+ // (#204) as the dropped `slop` below, one line apart. It is now the shared
+ // parse: a value we cannot read is a 400, and `0` stays accepted because
+ // ES accepts it *here* (it rejects it on `multi_match`; the asymmetry is
+ // ES's own, see `parse_max_expansions`).
+ let max_expansions = parse_max_expansions(
+ inner.get("max_expansions"),
+ "match_phrase_prefix",
+ /* allow_zero */ true,
+ )?;
// `slop` used to be read nowhere here, so
// `{"match_phrase_prefix": {"f": {"query": …, "slop": 2}}}` was accepted
// and answered as slop 0 — accept-and-ignore (#204 class), and a silent
@@ -3467,6 +3527,20 @@ fn parse_span_near(params: &Value) -> Result<QueryNode> {
return Ok(QueryNode::MatchNone);
}
+ // KNOWN GAP (pre-existing, deliberately not changed by #230): this read
+ // accepts anything and silently substitutes 0 — `slop: -1`, `slop: 2.0`
+ // and `slop: "2"` all answer as slop 0. It is NOT routed through
+ // `parse_phrase_slop`, and the phrase work in #230 does not cover it.
+ //
+ // It is left alone rather than "fixed" because ES applies no validation
+ // here either: `SpanNearQueryBuilder` takes the int unchecked
+ // (`.../index/query/SpanNearQueryBuilder.java:62`, no negative test)
+ // while every phrase builder throws on a negative slop. So a negative
+ // `span_near` slop is a query ES answers, and rejecting it would be a
+ // wire-compat break on a request that works today. Closing it properly
+ // means honouring the value (float coercion via `coerce_es_int`, and a
+ // decision about what a negative span slop should match), which is a
+ // span-query change, not a phrase one.
let slop = obj.get("slop").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
let in_order = obj
.get("in_order")
@@ -4498,6 +4572,13 @@ mod tests {
json!({"match_phrase": {"body": {"query": "a b", "slop": -1}}}),
json!({"match_phrase_prefix": {"body": {"query": "a b", "slop": -1}}}),
json!({"match_phrase_prefix": {"body": {"query": "a b", "slop": 4294967296i64}}}),
+ // `max_expansions` used to be read here as
+ // `as_u64().unwrap_or(50)`, so both of these were accepted and
+ // silently answered with 50 (#204 class). ES throws "No negative
+ // maxExpansions allowed" (MatchPhrasePrefixQueryBuilder.java:118).
+ json!({"match_phrase_prefix": {"body": {"query": "a b", "max_expansions": -1}}}),
+ json!({"match_phrase_prefix": {"body": {"query": "a b", "max_expansions": "many"}}}),
+ json!({"match_phrase": {"body": {"query": "a b", "slop": "two"}}}),
] {
assert!(
parse_query(&bad).is_err(),
@@ -4506,6 +4587,50 @@ mod tests {
}
}
+ /// Values ES *accepts by coercing* must be accepted here too. ES reads
+ /// `slop` and `max_expansions` with `XContentParser.intValue()` under the
+ /// default coercing policy, which truncates a float token and narrows a
+ /// numeric string (`AbstractXContentParser.java:74`/`:162`/`:177`), so
+ /// `{"slop": 2.0}` is a request ES answers with slop 2. Refusing it would
+ /// be a new 400 on a query that works today, and reading it as 0 would be
+ /// the accept-and-ignore this parser exists to remove — both are wrong.
+ #[test]
+ fn test_phrase_params_are_coerced_like_es() {
+ for (v, want) in [(json!(2.0), 2u32), (json!(2.9), 2), (json!("2"), 2)] {
+ let node = q(json!({"match_phrase": {"body": {"query": "a b", "slop": v}}}));
+ assert!(
+ matches!(node, QueryNode::MatchPhrase { slop, .. } if slop == want),
+ "slop {v} must coerce to {want}, parsed to {node:?}"
+ );
+ let node = q(json!({"multi_match": {"query": "a b", "fields": ["body"],
+ "type": "phrase", "slop": v}}));
+ assert!(
+ matches!(node, QueryNode::MultiMatch { slop, .. } if slop == want),
+ "multi_match slop {v} must coerce to {want}, parsed to {node:?}"
+ );
+ }
+
+ // `max_expansions: 0` is a 400 on `multi_match`
+ // (MultiMatchQueryBuilder.java:389 rejects `<= 0`) and ACCEPTED on
+ // `match_phrase_prefix` (MatchPhrasePrefixQueryBuilder.java:118
+ // rejects only `< 0`). The asymmetry is ES's; XERJ mirrors it rather
+ // than inventing a rule ES does not have.
+ let node = q(json!({"match_phrase_prefix": {"body": {"query": "a b",
+ "max_expansions": 0}}}));
+ assert!(
+ matches!(node, QueryNode::MatchPhrasePrefix { max_expansions, .. }
+ if max_expansions == 0),
+ "ES accepts max_expansions 0 on match_phrase_prefix, parsed to {node:?}"
+ );
+ assert!(
+ parse_query(&json!({"multi_match": {"query": "a b", "fields": ["body"],
+ "type": "phrase_prefix",
+ "max_expansions": 0}}))
+ .is_err(),
+ "ES rejects max_expansions 0 on multi_match"
+ );
+ }
+
/// `slop` + `type: phrase_prefix` is ACCEPTED and carried, at BOTH
/// entry points. ES honours it (`MultiMatchQueryBuilder` rejects slop
/// only for `bool_prefix`; `TextFieldMapper.createPhrasePrefixQuery`