Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(query): lenient-aware type check for full-text clauses (#1284, part 2)

Vincenzo Lombardo5d71865ecd6c
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 767a40351..d5e902d75 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -11293,20 +11293,39 @@ async fn search_impl(
// de-boosted `{"match": {kw: {"query": …,
// "boost": N}}}`.
let mut boost: Option<Value> = None;
+ let mut lenient = false;
let value = match raw {
Value::Object(inner) => {
boost = inner.get("boost").cloned();
+ lenient =
+ matches!(
+ inner.get("lenient"),
+ Some(Value::Bool(true))
+ ) || inner.get("lenient").and_then(Value::as_str)
+ == Some("true");
inner.get("query").cloned().unwrap_or(raw.clone())
}
_ => raw.clone(),
};
+ // #1284: a `term` has no `lenient`, and the
+ // engine 400s a non-numeric term value on a
+ // numeric field. A lenient `match` whose value
+ // is not a number stays a `match`, which the
+ // engine answers with no hits, as ES does.
+ let lenient_non_numeric = lenient
+ && exact.contains(field)
+ && value
+ .as_str()
+ .is_some_and(|s| s.trim().parse::<f64>().is_err());
let s_owned = match &value {
Value::String(s) => Some(s.clone()),
Value::Number(n) => Some(n.to_string()),
_ => None,
};
let split_field = split.contains(field);
- if split_field {
+ if lenient_non_numeric {
+ // Leave the clause as written.
+ } else if split_field {
if let Some(s) = s_owned.as_deref() {
let toks: Vec<&str> = s.split_whitespace().collect();
if toks.len() > 1 {
diff --git a/engine/crates/xerj-api/tests/non_numeric_value_on_numeric_field_400s.rs b/engine/crates/xerj-api/tests/non_numeric_value_on_numeric_field_400s.rs
index f68d83364..6d311273d 100644
--- a/engine/crates/xerj-api/tests/non_numeric_value_on_numeric_field_400s.rs
+++ b/engine/crates/xerj-api/tests/non_numeric_value_on_numeric_field_400s.rs
@@ -1,8 +1,9 @@
-//! Issue #1284 (term-level part): a string that is not a number, sent by a
-//! `term` / `terms` / `range` clause to a numeric field, is a query ES cannot
-//! build. ES 8.13.4 answers 400 with root cause `failed to create query: For
-//! input string: "abc"` (`query_shard_exception` / `number_format_exception`).
-//! XERJ answered 200 with 0 hits, so a type error looked like an empty result.
+//! Issue #1284: a string that is not a number, sent to a numeric field by a
+//! `term` / `terms` / `range` clause, or by a full-text clause that is not
+//! lenient, is a query ES cannot build. ES 8.13.4 answers 400 with root cause
+//! `failed to create query: For input string: "abc"` (`query_shard_exception`
+//! / `number_format_exception`). XERJ answered 200 with 0 hits, so a type
+//! error looked like an empty result.
//!
//! Elasticsearch is referenced for wire semantics only; no ES code is
//! reproduced here.
@@ -140,6 +141,121 @@ async fn numeric_strings_and_non_numeric_fields_stay_valid() {
}
}
+/// Full-text clauses that are not lenient. Every shape here returned 400 on
+/// ES 8.13.4 with this exact reason: `match` with `lenient` absent or false,
+/// and `multi_match` / `query_string` / `simple_query_string` whose field set
+/// is narrower than every field (so ES defaults `lenient` to false), or that
+/// say `lenient: false` outright.
+#[tokio::test]
+async fn non_lenient_full_text_clauses_are_a_400() {
+ let (app, _dir) = seeded().await;
+ for query in [
+ json!({"match": {"bytes": "abc"}}),
+ json!({"match": {"bytes": {"query": "abc", "lenient": false}}}),
+ json!({"match": {"ms": "abc"}}),
+ json!({"match_phrase": {"bytes": "abc"}}),
+ json!({"multi_match": {"query": "abc", "fields": ["bytes", "msg"]}}),
+ json!({"multi_match": {"query": "abc", "fields": ["b*"]}}),
+ json!({"multi_match": {"query": "abc", "fields": ["bytes"], "type": "phrase"}}),
+ json!({"query_string": {"query": "bytes:abc", "lenient": false}}),
+ json!({"query_string": {"query": "abc", "default_field": "bytes"}}),
+ json!({"query_string": {"query": "abc", "fields": ["bytes", "msg"]}}),
+ json!({"query_string": {"query": "bytes:(1 OR abc)", "lenient": false}}),
+ json!({"query_string": {"query": "bytes:>abc", "lenient": false}}),
+ json!({"query_string": {"query": "bytes:\"abc\"", "lenient": false}}),
+ json!({"query_string": {"query": "msg:abc AND bytes:abc", "lenient": false}}),
+ json!({"simple_query_string": {"query": "abc", "fields": ["bytes"]}}),
+ json!({"simple_query_string": {"query": "abc", "fields": ["bytes", "msg"]}}),
+ json!({"bool": {"filter": [{"match": {"bytes": "abc"}}]}}),
+ ] {
+ for path in ["/t84/_count", "/t84/_search"] {
+ let (st, resp) = query_on(&app, path, &query).await;
+ assert_eq!(st, StatusCode::BAD_REQUEST, "{path} {query}: {resp}");
+ assert!(
+ resp.to_string()
+ .contains(r#"failed to create query: For input string: \"abc\""#),
+ "{path} {query}: reason must name the bad value: {resp}"
+ );
+ }
+ }
+}
+
+/// Lenient full-text clauses stay a 200, as on ES 8.13.4: an explicit
+/// `lenient: true`, and the every-field default (`query_string` with no
+/// `default_field`, or `["*"]`), which is lenient even for a clause that names
+/// the numeric field itself. The two `query_string` ranges were a 200 on main
+/// and must not become a 400 now that ranges are checked.
+#[tokio::test]
+async fn lenient_full_text_clauses_stay_a_200() {
+ let (app, _dir) = seeded().await;
+ for (query, hits) in [
+ (
+ json!({"match": {"bytes": {"query": "abc", "lenient": true}}}),
+ 0,
+ ),
+ (
+ json!({"match": {"bytes": {"query": "abc", "lenient": "true"}}}),
+ 0,
+ ),
+ (
+ json!({"multi_match": {"query": "abc", "fields": ["bytes", "msg"], "lenient": true}}),
+ 0,
+ ),
+ (json!({"query_string": {"query": "bytes:abc"}}), 0),
+ (
+ json!({"query_string": {"query": "bytes:abc", "lenient": true}}),
+ 0,
+ ),
+ (json!({"query_string": {"query": "bytes:>abc"}}), 0),
+ (json!({"query_string": {"query": "bytes:[abc TO 5]"}}), 0),
+ (json!({"query_string": {"query": "bytes:\"abc\""}}), 0),
+ (
+ json!({"query_string": {"query": "bytes:>abc", "default_field": "*"}}),
+ 0,
+ ),
+ (
+ json!({"simple_query_string": {"query": "abc", "fields": ["bytes"], "lenient": true}}),
+ 0,
+ ),
+ // Valid values on a numeric field, and full-text on text fields.
+ (json!({"match": {"bytes": "1"}}), 1),
+ (json!({"match": {"bytes": "1.5"}}), 0),
+ (
+ json!({"multi_match": {"query": "1", "fields": ["bytes"]}}),
+ 1,
+ ),
+ (
+ json!({"query_string": {"query": "bytes:1", "lenient": false}}),
+ 1,
+ ),
+ (json!({"match": {"msg": "hello"}}), 1),
+ ] {
+ // `_search` matters on its own: its ES-compat layer rewrites a `match`
+ // on a numeric field to a `term` before the engine sees it.
+ for path in ["/t84/_count", "/t84/_search"] {
+ let (st, resp) = query_on(&app, path, &query).await;
+ assert_eq!(st, StatusCode::OK, "{path} {query}: {resp}");
+ let got = if path.ends_with("_count") {
+ &resp["count"]
+ } else {
+ &resp["hits"]["total"]["value"]
+ };
+ assert_eq!(got, &json!(hits), "{path} {query}: {resp}");
+ }
+ }
+}
+
+async fn query_on(app: &axum::Router, path: &str, query: &Value) -> (StatusCode, Value) {
+ send(
+ app,
+ "POST",
+ path,
+ "application/json",
+ json!({ "query": query }).to_string(),
+ )
+ .await
+}
+
/// `_msearch` fails only the offending item.
#[tokio::test]
async fn non_numeric_value_fails_only_that_msearch_item() {
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 669e6b127..4c514684d 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -41262,26 +41262,39 @@ fn unsearchable_query_field(q: &QueryNode, schema: &Schema) -> Option<String> {
}
}
-/// The first string value a `term` / `terms` / `range` clause sends to a
-/// numeric (`long` / `double`) field that does not parse as a number (#1284).
+/// The first string value a clause sends to a numeric (`long` / `double`)
+/// field that does not parse as a number (#1284).
///
/// ES 8.13.4 answers every one of these with a 400 whose root cause reads
/// `failed to create query: For input string: "abc"`, top-level or under
/// `bool.filter`; XERJ answered a silent 0-hit 200, so a type error in a
-/// dashboard query looked like an empty result. These clause types have no
-/// `lenient` option in ES, so there is nothing to honour here. A decimal
-/// against a `long` (`"1.5"`) is a valid query that matches nothing on ES, so
-/// only an unparseable string is rejected. `match` / `multi_match` /
-/// `query_string` are left alone: they DO take `lenient`, which the AST does
-/// not carry yet.
+/// dashboard query looked like an empty result. A decimal against a `long`
+/// (`"1.5"`) is a valid query that matches nothing on ES, so only an
+/// unparseable string is rejected.
+///
+/// `term` / `terms` / `range` have no `lenient` option in ES and are always
+/// checked. The full-text leaves (`match`, `match_phrase`, `multi_match`, and
+/// everything `query_string` / `simple_query_string` lower to, ranges
+/// included) carry the clause's effective `lenient`, which the parser sets.
+/// A lenient leaf is skipped, which is how `query_string` with no
+/// `default_field` keeps answering 200, as on ES. A full-text leaf on a
+/// numeric field takes its whole query string as one value, as ES does
+/// (`match {bytes: "1 2"}` fails on `"1 2"`). A field pattern (`b*`, `*`) is
+/// checked against every numeric field it matches.
fn non_numeric_term_value(q: &QueryNode, schema: &Schema) -> Option<String> {
- let numeric = |field: &str| {
- declared_field(schema, field)
- .is_some_and(|fc| matches!(fc.field_type, FieldType::Long | FieldType::Double))
- };
- let bad = |v: &serde_json::Value| -> Option<String> {
- let s = v.as_str()?;
- s.trim().parse::<f64>().is_err().then(|| s.to_string())
+ let numeric = |field: &str| numeric_field_matches(schema, field);
+ let bad_str =
+ |s: &str| -> Option<String> { s.trim().parse::<f64>().is_err().then(|| s.to_string()) };
+ let bad = |v: &serde_json::Value| -> Option<String> { bad_str(v.as_str()?) };
+ // An empty full-text string carries no value to check: the parser makes a
+ // top-level empty `match` a `match_none`, and the only other source is the
+ // `field:` stub of a `field:(…)` group, whose terms are checked as leaves.
+ let bad_text = |s: &str| -> Option<String> {
+ if s.is_empty() {
+ None
+ } else {
+ bad_str(s)
+ }
};
match q {
QueryNode::Term { field, value, .. } if numeric(field) => bad(value),
@@ -41292,8 +41305,32 @@ fn non_numeric_term_value(q: &QueryNode, schema: &Schema) -> Option<String> {
gt,
lte,
lt,
+ lenient: false,
..
} if numeric(field) => [gte, gt, lte, lt].into_iter().flatten().find_map(bad),
+ QueryNode::Match {
+ field,
+ query,
+ lenient: false,
+ ..
+ }
+ | QueryNode::MatchPhrase {
+ field,
+ query,
+ lenient: false,
+ ..
+ } if numeric(field) => bad_text(query),
+ QueryNode::MultiMatch {
+ fields,
+ query,
+ lenient: false,
+ ..
+ } if fields
+ .iter()
+ .any(|f| numeric(f.split('^').next().unwrap_or(f))) =>
+ {
+ bad_text(query)
+ }
QueryNode::Bool {
must,
should,
@@ -41326,6 +41363,32 @@ fn non_numeric_term_value(q: &QueryNode, schema: &Schema) -> Option<String> {
}
}
+/// Whether `field` names a numeric (`long` / `double`) field or, for a pattern
+/// containing `*` / `?`, whether any declared numeric field matches it
+/// (#1284). Dotted paths are matched in full, as `multi_match` and
+/// `query_string` expand a `fields` pattern.
+fn numeric_field_matches(schema: &Schema, field: &str) -> bool {
+ fn is_numeric(fc: &FieldConfig) -> bool {
+ matches!(fc.field_type, FieldType::Long | FieldType::Double)
+ }
+ fn any_match(fields: &[FieldConfig], prefix: &str, pattern: &str) -> bool {
+ fields.iter().any(|fc| {
+ let path = if prefix.is_empty() {
+ fc.name.clone()
+ } else {
+ format!("{prefix}.{}", fc.name)
+ };
+ (is_numeric(fc) && crate::engine::glob_match(pattern, &path))
+ || any_match(&fc.fields, &path, pattern)
+ })
+ }
+ if field.contains(['*', '?']) {
+ any_match(&schema.fields, "", field)
+ } else {
+ declared_field(schema, field).is_some_and(is_numeric)
+ }
+}
+
/// Lower every LEXICAL clause that names a lexically typeless field to
/// `QueryNode::MatchNone` — the query half of #328.
///
@@ -41501,6 +41564,7 @@ fn lower_lexically_typeless_clauses(
boost,
slop,
max_expansions,
+ lenient,
} => match prune_specs(fields) {
None => q.clone(),
Some(kept) if kept.is_empty() => QueryNode::MatchNone,
@@ -41513,6 +41577,7 @@ fn lower_lexically_typeless_clauses(
boost: *boost,
slop: *slop,
max_expansions: *max_expansions,
+ lenient: *lenient,
},
},
QueryNode::SimpleQueryString { query, fields } => match prune_specs(fields) {
@@ -41886,6 +41951,7 @@ mod lexically_typeless_lowering_tests {
lte: Some(Value::from(2.0)),
lt: None,
boost: None,
+ lenient: false,
},
QueryNode::Prefix {
field: "emb".into(),
@@ -41907,6 +41973,7 @@ mod lexically_typeless_lowering_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
term("emb_chunks", "0.5"),
] {
@@ -41990,6 +42057,7 @@ mod lexically_typeless_lowering_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
match lower(&q) {
QueryNode::MultiMatch { fields, .. } => {
@@ -42015,6 +42083,7 @@ mod lexically_typeless_lowering_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
assert!(lower(&q).is_match_none());
}
@@ -42032,6 +42101,7 @@ mod lexically_typeless_lowering_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
assert_eq!(lower(&q), q);
}
@@ -42640,6 +42710,7 @@ mod keyword_rewrite_filter_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
}
}
@@ -42731,6 +42802,7 @@ mod keyword_rewrite_filter_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
})),
boost: None,
similarity: None,
@@ -42782,6 +42854,7 @@ mod keyword_rewrite_filter_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
query_weight: 1.0,
rescore_query_weight: 1.0,
@@ -42947,6 +43020,7 @@ fn rewrite_query_aliases(q: &QueryNode, schema: &Schema) -> QueryNode {
lte: Some(Value::String(hi)),
lt: None,
boost: *boost,
+ lenient: false,
};
}
}
@@ -42984,6 +43058,7 @@ fn rewrite_query_aliases(q: &QueryNode, schema: &Schema) -> QueryNode {
lte: Some(epoch),
lt: None,
boost: *boost,
+ lenient: false,
};
}
}
@@ -43055,6 +43130,7 @@ fn rewrite_query_aliases(q: &QueryNode, schema: &Schema) -> QueryNode {
lte: Some(epoch),
lt: None,
boost: *boost,
+ lenient: false,
}
})
}
@@ -43069,6 +43145,7 @@ fn rewrite_query_aliases(q: &QueryNode, schema: &Schema) -> QueryNode {
lte: Some(Value::String(hi)),
lt: None,
boost: *boost,
+ lenient: false,
})
};
match rewritten {
@@ -43106,6 +43183,7 @@ fn rewrite_query_aliases(q: &QueryNode, schema: &Schema) -> QueryNode {
lte,
lt,
boost,
+ lenient,
} => {
let resolved = resolve_field_alias(schema, field);
QueryNode::Range {
@@ -43115,6 +43193,7 @@ fn rewrite_query_aliases(q: &QueryNode, schema: &Schema) -> QueryNode {
lte: lte.clone(),
lt: lt.clone(),
boost: *boost,
+ lenient: *lenient,
}
}
QueryNode::Exists { field } => {
@@ -43128,6 +43207,7 @@ fn rewrite_query_aliases(q: &QueryNode, schema: &Schema) -> QueryNode {
operator,
analyzer,
minimum_should_match,
+ lenient,
} => {
let resolved = resolve_field_alias(schema, field);
QueryNode::Match {
@@ -43137,6 +43217,7 @@ fn rewrite_query_aliases(q: &QueryNode, schema: &Schema) -> QueryNode {
operator: *operator,
analyzer: analyzer.clone(),
minimum_should_match: minimum_should_match.clone(),
+ lenient: *lenient,
}
}
QueryNode::Bool {
@@ -43207,6 +43288,7 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {
lte,
lt,
boost,
+ lenient,
} => QueryNode::Range {
field: strip(field),
gte: gte.clone(),
@@ -43214,6 +43296,7 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {
lte: lte.clone(),
lt: lt.clone(),
boost: *boost,
+ lenient: *lenient,
},
QueryNode::Prefix {
field,
@@ -43249,6 +43332,7 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {
operator,
analyzer,
minimum_should_match,
+ lenient,
} => QueryNode::Match {
field: strip(field),
query: query.clone(),
@@ -43256,6 +43340,7 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {
operator: *operator,
analyzer: analyzer.clone(),
minimum_should_match: minimum_should_match.clone(),
+ lenient: *lenient,
},
QueryNode::MatchPhrase {
field,
@@ -43263,12 +43348,14 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {
slop,
analyzer,
boost,
+ lenient,
} => QueryNode::MatchPhrase {
field: strip(field),
query: query.clone(),
slop: *slop,
analyzer: analyzer.clone(),
boost: *boost,
+ lenient: *lenient,
},
QueryNode::MatchPhrasePrefix {
field,
@@ -43307,6 +43394,7 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {
boost,
slop,
max_expansions,
+ lenient,
} => QueryNode::MultiMatch {
fields: fields.iter().map(|f| strip(f)).collect(),
query: query.clone(),
@@ -43316,6 +43404,7 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {
boost: *boost,
slop: *slop,
max_expansions: *max_expansions,
+ lenient: *lenient,
},
QueryNode::GeoDistance {
field,
@@ -50524,6 +50613,7 @@ fn query_node_to_fts_projected(
boost,
analyzer,
minimum_should_match,
+ lenient: _,
} => {
// Per-clause boost (ES `{"match": {"f": {"query": …, "boost": N}}}`)
// must reach the BM25 scorer — dropping it here made boosted and
@@ -50603,6 +50693,7 @@ fn query_node_to_fts_projected(
// prefix and nothing needs the bound.
max_expansions: _,
analyzer,
+ lenient: _,
} => {
// phrase is POSITIONAL (issue #230): it lowers to one positional
// clause per field, combined by dis_max — the
@@ -51138,6 +51229,7 @@ fn query_node_to_fts_projected(
slop,
analyzer,
boost,
+ lenient: _,
} => {
// match_phrase on a KEYWORD field: the field is not tokenized, so
// the query analyzes (keyword analyzer) to a single whole-value
@@ -52497,6 +52589,7 @@ fn scored_fast_plan(
lte,
lt,
boost,
+ lenient: _,
} if fs.num.contains(field) && boost_ok(boost) => {
let (min, min_inc) = match (gte, gt) {
(Some(v), None) => (v.as_f64()?, true),
@@ -54533,6 +54626,7 @@ mod fts_projection_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
};
let text_fields = vec!["body".to_string()];
let fq = query_node_to_fts(&q, &text_fields, &kw(&["status"])).expect("projects");
@@ -54563,6 +54657,7 @@ mod fts_projection_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
};
let fq = query_node_to_fts(&q, &["body".to_string()], &kw(&[])).expect("projects");
match fq {
@@ -54586,6 +54681,7 @@ mod fts_projection_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
};
let fq = query_node_to_fts(&q, &[], &kw(&["model"])).expect("projects");
match fq {
@@ -54627,6 +54723,7 @@ mod fts_projection_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
let fq = query_node_to_fts(&q, &[], &kw(&["model", "top_doc"])).expect("projects");
match fq {
@@ -54657,6 +54754,7 @@ mod fts_projection_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
let fq = query_node_to_fts(&q, &["title".to_string()], &kw(&["model"])).expect("projects");
match fq {
@@ -54709,6 +54807,7 @@ mod fts_projection_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
}
}
@@ -54822,6 +54921,7 @@ mod fts_projection_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
let fq = query_node_to_fts(&q, &["body".to_string(), "title".to_string()], &kw(&[]))
.expect("projects");
@@ -54857,6 +54957,7 @@ mod fts_projection_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
assert!(
query_node_to_fts(&q, &["body".to_string(), "title".to_string()], &kw(&[]),).is_none()
@@ -54878,6 +54979,7 @@ mod fts_projection_tests {
boost: None,
slop: 2,
max_expansions: 50,
+ lenient: false,
};
let text = vec!["body".to_string(), "title".to_string()];
match query_node_to_fts(&q, &text, &kw(&[])).expect("phrase projects") {
@@ -54915,6 +55017,7 @@ mod fts_projection_tests {
boost: None,
slop: 0,
max_expansions: 7,
+ lenient: false,
};
assert!(
query_node_to_fts(&q, &text, &kw(&[])).is_none(),
@@ -54941,6 +55044,7 @@ mod fts_projection_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
assert!(
query_node_to_fts(&keyword_field, &text, &kw(&["tags"])).is_none(),
@@ -54956,6 +55060,7 @@ mod fts_projection_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
assert!(
query_node_to_fts(&dotted, &text, &kw(&[])).is_none(),
@@ -54975,6 +55080,7 @@ mod fts_projection_tests {
slop: 0,
analyzer: None,
boost: None,
+ lenient: false,
};
let fq = query_node_to_fts(&q, &[], &kw(&["top_doc"])).expect("keyword phrase projects");
match fq {
@@ -54999,6 +55105,7 @@ mod fts_projection_tests {
slop: 0,
analyzer: None,
boost: None,
+ lenient: false,
};
let fq =
query_node_to_fts(&q, &["body".to_string()], &kw(&[])).expect("text phrase projects");
@@ -55023,6 +55130,7 @@ mod fts_projection_tests {
slop: 2,
analyzer: None,
boost: None,
+ lenient: false,
};
assert!(
query_node_to_fts(&q, &["body".to_string()], &kw(&[])).is_none(),
@@ -55133,6 +55241,7 @@ mod fts_projection_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
QueryNode::Match {
field: "status".into(),
@@ -55141,6 +55250,7 @@ mod fts_projection_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
],
should: vec![],
@@ -55853,6 +55963,7 @@ mod lexical_passage_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
let hit = Hit {
id: "doc-1".into(),
@@ -55911,6 +56022,7 @@ mod lexical_passage_tests {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient: false,
};
let hit = Hit {
id: "doc-1".into(),
@@ -55948,6 +56060,7 @@ mod lexical_passage_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
};
let hit = Hit {
id: "doc-1".into(),
@@ -55991,6 +56104,7 @@ mod lexical_passage_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
};
let plain_hit = Hit {
id: "doc-1".into(),
@@ -56045,6 +56159,7 @@ mod lexical_passage_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
};
let semantic = PassageMatch {
field: "body".into(),
@@ -57198,6 +57313,7 @@ mod write_publication_integration_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
..SearchRequest::default()
};
@@ -57268,6 +57384,7 @@ mod write_publication_integration_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
..SearchRequest::default()
};
@@ -57320,6 +57437,7 @@ mod write_publication_integration_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
..SearchRequest::default()
};
@@ -57375,6 +57493,7 @@ mod write_publication_integration_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
..SearchRequest::default()
};
@@ -57386,6 +57505,7 @@ mod write_publication_integration_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
..SearchRequest::default()
};
@@ -57693,6 +57813,7 @@ mod unwrap_single_clause_643_wrapper_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
}
}
@@ -58730,6 +58851,7 @@ mod exact_scan_hydration_tests {
lte: None,
lt: Some(Value::from(800)),
boost: None,
+ lenient: false,
};
vec![
("none", None),
@@ -58770,6 +58892,7 @@ mod exact_scan_hydration_tests {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
}),
),
]
@@ -59294,6 +59417,7 @@ mod exact_scan_hydration_tests {
lte: None,
lt: Some(Value::from(800)),
boost: None,
+ lenient: false,
},
QueryNode::Prefix {
field: "tag".into(),
diff --git a/engine/crates/xerj-engine/src/sql.rs b/engine/crates/xerj-engine/src/sql.rs
index 54aac6327..9e76ff970 100644
--- a/engine/crates/xerj-engine/src/sql.rs
+++ b/engine/crates/xerj-engine/src/sql.rs
@@ -603,6 +603,7 @@ fn parse_condition(tokens: &[Token], pos: &mut usize, depth: usize) -> Result<Qu
boost: None,
minimum_should_match: None,
analyzer: None,
+ lenient: false,
}
} else {
QueryNode::Term {
@@ -681,6 +682,7 @@ fn make_range(
lt,
lte,
boost: None,
+ lenient: false,
}
}
diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rs
index e031622a4..e7d2be0cc 100644
--- a/engine/crates/xerj-engine/tests/integration.rs
+++ b/engine/crates/xerj-engine/tests/integration.rs
@@ -5931,6 +5931,7 @@ async fn test_rescore_changes_ranking() {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
query_weight: 0.1,
rescore_query_weight: 10.0,
@@ -6091,6 +6092,7 @@ async fn rescore_keyword_query_is_mapping_aware() {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
},
query_weight: 0.1,
rescore_query_weight: 10.0,
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index 3c48aea56..da3cbafbe 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -471,6 +471,11 @@ pub enum QueryNode {
lt: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
boost: Option<f32>,
+ /// #1284: true only for a range lowered from a `query_string` that is
+ /// effectively lenient (see `Match::lenient`). The `range` DSL clause
+ /// has no `lenient` option in ES, so a parsed `range` is always strict.
+ #[serde(default, skip_serializing_if = "is_false")]
+ lenient: bool,
},
/// Prefix match on a keyword field.
@@ -538,6 +543,12 @@ pub enum QueryNode {
boost: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
minimum_should_match: Option<MinShouldMatch>,
+ /// ES `lenient` (#1284): when true, a value this field's type cannot
+ /// parse (`"abc"` on a `long`) matches nothing instead of failing the
+ /// query. The parser sets the *effective* value: explicit `lenient`, or
+ /// `true` when the clause targeted every field (`*`), as ES defaults it.
+ #[serde(default, skip_serializing_if = "is_false")]
+ lenient: bool,
},
/// Ordered phrase match with optional slop. `slop` is Lucene
@@ -553,6 +564,12 @@ pub enum QueryNode {
analyzer: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
boost: Option<f32>,
+ /// ES `lenient` (#1284): when true, a value this field's type cannot
+ /// parse (`"abc"` on a `long`) matches nothing instead of failing the
+ /// query. The parser sets the *effective* value: explicit `lenient`, or
+ /// `true` when the clause targeted every field (`*`), as ES defaults it.
+ #[serde(default, skip_serializing_if = "is_false")]
+ lenient: bool,
},
/// Full-text query across multiple fields simultaneously.
@@ -577,6 +594,12 @@ pub enum QueryNode {
/// terms the trailing prefix may expand to (ES default 50).
#[serde(default = "default_max_expansions")]
max_expansions: u32,
+ /// ES `lenient` (#1284): when true, a value this field's type cannot
+ /// parse (`"abc"` on a `long`) matches nothing instead of failing the
+ /// query. The parser sets the *effective* value: explicit `lenient`, or
+ /// `true` when the clause targeted every field (`*`), as ES defaults it.
+ #[serde(default, skip_serializing_if = "is_false")]
+ lenient: bool,
},
/// Lucene query-string syntax (e.g. `"title:(foo AND bar) OR body:baz"`).
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index e67d78040..a534952b8 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -523,6 +523,7 @@ fn parse_match(params: &Value) -> Result<QueryNode> {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient: false,
});
}
@@ -561,10 +562,35 @@ fn parse_match(params: &Value) -> Result<QueryNode> {
analyzer,
boost,
minimum_should_match,
+ lenient: parse_lenient(vobj.get("lenient")).unwrap_or(false),
};
Ok(maybe_named(node, name))
}
+/// Read a clause's `lenient` flag (#1284). ES takes a JSON boolean or the
+/// strings `"true"` / `"false"`; `None` means the caller applies its default.
+fn parse_lenient(v: Option<&Value>) -> Option<bool> {
+ match v? {
+ Value::Bool(b) => Some(*b),
+ Value::String(s) if s == "true" => Some(true),
+ Value::String(s) if s == "false" => Some(false),
+ _ => None,
+ }
+}
+
+/// Whether a field list means "every field", the case where ES defaults
+/// `lenient` to true for `multi_match`, `query_string` and
+/// `simple_query_string` (#1284). An empty list counts: with no fields those
+/// clauses fall back to `index.query.default_field`, whose default is `*`.
+/// A narrower pattern (`b*`) does not, and ES stays strict for it.
+fn targets_every_field(fields: &[String]) -> bool {
+ match fields {
+ [] => true,
+ [only] => qs_split_boost(only).0 == "*",
+ _ => false,
+ }
+}
+
/// Best-effort conversion of a JSON scalar to a string for query
/// shorthand (ES accepts `match: {foo: 42}` same as `match: {foo: "42"}`).
fn scalar_to_string(v: &Value) -> Option<String> {
@@ -602,6 +628,7 @@ fn parse_match_phrase(params: &Value) -> Result<QueryNode> {
slop: 0,
analyzer: None,
boost: None,
+ lenient: false,
});
}
@@ -632,12 +659,15 @@ fn parse_match_phrase(params: &Value) -> Result<QueryNode> {
.and_then(|v| v.as_str())
.map(str::to_string);
+ // `match_phrase` has no `lenient` option in ES (it answers `[match_phrase]
+ // query does not support [lenient]`), so a parsed phrase is always strict.
let node = QueryNode::MatchPhrase {
field,
query,
slop,
analyzer,
boost,
+ lenient: false,
};
Ok(maybe_named(node, name))
}
@@ -775,6 +805,9 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
if fields.is_empty() {
return invalid("`multi_match.fields` must not be empty");
}
+ // #1284: explicit `lenient`, else ES's default: lenient only when the
+ // clause targets every field (`["*"]`).
+ let lenient = parse_lenient(obj.get("lenient")).unwrap_or_else(|| targets_every_field(&fields));
let type_str = obj
.get("type")
@@ -909,6 +942,7 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
boost: None,
analyzer: analyzer_opt.clone(),
minimum_should_match: None,
+ lenient,
}
}
};
@@ -983,6 +1017,7 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
boost,
slop,
max_expansions,
+ lenient,
})
}
@@ -1229,6 +1264,7 @@ fn parse_range(params: &Value) -> Result<QueryNode> {
lte,
lt,
boost,
+ lenient: false,
})
}
@@ -1436,6 +1472,14 @@ fn parse_query_string(params: &Value) -> Result<QueryNode> {
.unwrap_or_default();
let default_operator = parse_bool_operator(obj.get("default_operator")).ok();
let boost = obj.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
+ // #1284: explicit `lenient`, else ES's default, which is lenient only when
+ // the clause's field set is every field. That is why `bytes:abc` with no
+ // `default_field` is a 200 on ES even though the clause names `bytes`.
+ let lenient =
+ parse_lenient(obj.get("lenient")).unwrap_or_else(|| match default_field.as_deref() {
+ Some(df) if fields.is_empty() => df == "*",
+ _ => targets_every_field(&fields),
+ });
// Try to lower the query string into a Bool tree so downstream matchers
// can honor `field:value` + OR/AND syntax. Fall back to the opaque
@@ -1447,6 +1491,7 @@ fn parse_query_string(params: &Value) -> Result<QueryNode> {
QsFields {
default_field: default_field.as_deref(),
fields: &fields,
+ lenient,
},
default_operator,
)? {
@@ -1502,6 +1547,10 @@ fn qs_split_boost(spec: &str) -> (String, Option<f32>) {
struct QsFields<'a> {
default_field: Option<&'a str>,
fields: &'a [String],
+ /// The clause's effective `lenient` (#1284), stamped on every leaf the
+ /// string lowers to, field-qualified ones included: ES decides leniency
+ /// once per `query_string`, not per clause.
+ lenient: bool,
}
impl QsFields<'_> {
@@ -2143,6 +2192,7 @@ fn parse_qs_unary(toks: &[QsTok], pos: &mut usize, ctx: QsFields<'_>) -> Option<
analyzer: None,
boost,
minimum_should_match: None,
+ lenient: ctx.lenient,
})
.collect(),
)
@@ -2158,6 +2208,7 @@ fn parse_qs_unary(toks: &[QsTok], pos: &mut usize, ctx: QsFields<'_>) -> Option<
slop: 0,
analyzer: None,
boost,
+ lenient: ctx.lenient,
})
.collect(),
)
@@ -2188,6 +2239,7 @@ fn parse_qs_unary(toks: &[QsTok], pos: &mut usize, ctx: QsFields<'_>) -> Option<
lte: lte.clone(),
lt: lt.clone(),
boost,
+ lenient: ctx.lenient,
})
.collect(),
)
@@ -2394,6 +2446,8 @@ fn parse_simple_query_string(params: &Value) -> Result<QueryNode> {
let mm: Option<MinShouldMatch> = obj
.get("minimum_should_match")
.and_then(|v| parse_min_should_match(v).ok());
+ // #1284: same default as `multi_match` / `query_string`.
+ let lenient = parse_lenient(obj.get("lenient")).unwrap_or_else(|| targets_every_field(&fields));
// Tokenize the query: split on whitespace; leading +/-/| signal per-term operators.
let mut must: Vec<QueryNode> = Vec::new();
@@ -2414,7 +2468,7 @@ fn parse_simple_query_string(params: &Value) -> Result<QueryNode> {
if term_text.is_empty() {
continue;
}
- let node = make_simple_query_node(term_text, &fields);
+ let node = make_simple_query_node(term_text, &fields, lenient);
match sign {
'+' => must.push(node),
'-' => must_not.push(node),
@@ -2432,7 +2486,7 @@ fn parse_simple_query_string(params: &Value) -> Result<QueryNode> {
// No tokens parsed (rare empty query): treat query as a literal term.
if must.is_empty() && should.is_empty() && must_not.is_empty() {
- let node = make_simple_query_node(&query, &fields);
+ let node = make_simple_query_node(&query, &fields, lenient);
return Ok(node);
}
@@ -2453,7 +2507,7 @@ fn parse_simple_query_string(params: &Value) -> Result<QueryNode> {
}
/// Build a Match or MultiMatch node for a term in a simple_query_string.
-fn make_simple_query_node(term: &str, fields: &[String]) -> QueryNode {
+fn make_simple_query_node(term: &str, fields: &[String], lenient: bool) -> QueryNode {
if fields.len() == 1 {
QueryNode::Match {
field: fields[0].clone(),
@@ -2462,6 +2516,7 @@ fn make_simple_query_node(term: &str, fields: &[String]) -> QueryNode {
analyzer: None,
boost: None,
minimum_should_match: None,
+ lenient,
}
} else if fields.is_empty() {
// No fields specified — use a match_all-like placeholder.
@@ -2481,6 +2536,7 @@ fn make_simple_query_node(term: &str, fields: &[String]) -> QueryNode {
boost: None,
slop: 0,
max_expansions: 50,
+ lenient,
}
}
}
@@ -3685,6 +3741,10 @@ fn parse_more_like_this(params: &Value) -> Result<QueryNode> {
analyzer: None,
boost: None,
minimum_should_match: None,
+ // ES refuses a numeric `more_like_this` field with a
+ // different error (`only supports text/keyword fields`);
+ // #1284's type check is not that refusal, so stay out.
+ lenient: true,
});
}
}
@@ -4360,6 +4420,8 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {
boost: None,
analyzer: analyzer.clone(),
minimum_should_match: None,
+ // `match_bool_prefix` has no `lenient` option in ES.
+ lenient: false,
}
}
};
diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rs
index 339b516bb..cd366523e 100644
--- a/engine/crates/xerj-query/src/planner.rs
+++ b/engine/crates/xerj-query/src/planner.rs
@@ -729,6 +729,7 @@ mod tests {
lte: None,
lt: Some(serde_json::json!(65)),
boost: None,
+ lenient: false,
};
let plan = plan_query(q, &empty_schema()).unwrap();
assert!(matches!(plan, ExecutionPlan::RangeScan { .. }));
diff --git a/engine/tests/es-compat-yaml/yaml/search/720_non_numeric_term_value.yml b/engine/tests/es-compat-yaml/yaml/search/720_non_numeric_term_value.yml
index 4d18dea74..e91fa4107 100644
--- a/engine/tests/es-compat-yaml/yaml/search/720_non_numeric_term_value.yml
+++ b/engine/tests/es-compat-yaml/yaml/search/720_non_numeric_term_value.yml
@@ -47,3 +47,54 @@ setup:
index: non_numeric_term
body: { query: { term: { h: "abc" } } }
- match: { hits.total.value: 0 }
+
+---
+"non-lenient full-text value on a numeric field is a 400 (#1284)":
+ - do:
+ catch: bad_request
+ search:
+ index: non_numeric_term
+ body: { query: { match: { bytes: "abc" } } }
+ - match: { status: 400 }
+
+ - do:
+ catch: bad_request
+ search:
+ index: non_numeric_term
+ body: { query: { query_string: { query: "bytes:abc", lenient: false } } }
+ - match: { status: 400 }
+
+ - do:
+ catch: bad_request
+ search:
+ index: non_numeric_term
+ body: { query: { multi_match: { query: "abc", fields: [ "bytes", "h" ] } } }
+ - match: { status: 400 }
+
+---
+"lenient full-text value on a numeric field stays a 200 (#1284)":
+ - do:
+ search:
+ index: non_numeric_term
+ body: { query: { match: { bytes: { query: "abc", lenient: true } } } }
+ - match: { hits.total.value: 0 }
+
+ # No `default_field`: the field set is every field, so ES defaults
+ # `lenient` to true even though the clause names `bytes` itself.
+ - do:
+ search:
+ index: non_numeric_term
+ body: { query: { query_string: { query: "bytes:abc" } } }
+ - match: { hits.total.value: 0 }
+
+ - do:
+ search:
+ index: non_numeric_term
+ body: { query: { query_string: { query: "bytes:>abc" } } }
+ - match: { hits.total.value: 0 }
+
+ - do:
+ search:
+ index: non_numeric_term
+ body: { query: { match: { bytes: "1" } } }
+ - match: { hits.total.value: 1 }