Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(es-compat): reject the silent-wrong-query class on _search (RC4 W3 item 4)

xerj-orgc2bb2afbfc73
diff --git a/engine/crates/xerj-api/src/auth.rs b/engine/crates/xerj-api/src/auth.rs
index 2cd87ae17..0060b2593 100644
--- a/engine/crates/xerj-api/src/auth.rs
+++ b/engine/crates/xerj-api/src/auth.rs
@@ -169,10 +169,17 @@ fn unauthorized_response() -> Response {
root_cause: vec![EsRootCause {
error_type: error_type.clone(),
reason: reason.clone(),
+ resource_type: None,
+ resource_id: None,
+ index_uuid: None,
index: None,
}],
error_type,
reason,
+ resource_type: None,
+ resource_id: None,
+ index_uuid: None,
+ index: None,
request_id: None,
},
status: 401,
diff --git a/engine/crates/xerj-api/src/error.rs b/engine/crates/xerj-api/src/error.rs
index 59b4558dd..6c172be5e 100644
--- a/engine/crates/xerj-api/src/error.rs
+++ b/engine/crates/xerj-api/src/error.rs
@@ -28,11 +28,20 @@ use xerj_common::XerjError;
// ─────────────────────────────────────────────────────────────────────────────
/// A single cause entry inside the ES `root_cause` array.
+///
+/// Field order matches ES byte-for-byte for `index_not_found_exception`:
+/// `type, reason, resource.type, resource.id, index_uuid, index`.
#[derive(Debug, Serialize)]
pub struct EsRootCause {
#[serde(rename = "type")]
pub error_type: String,
pub reason: String,
+ #[serde(rename = "resource.type", skip_serializing_if = "Option::is_none")]
+ pub resource_type: Option<String>,
+ #[serde(rename = "resource.id", skip_serializing_if = "Option::is_none")]
+ pub resource_id: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub index_uuid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub index: Option<String>,
}
@@ -44,6 +53,14 @@ pub struct EsErrorBody {
#[serde(rename = "type")]
pub error_type: String,
pub reason: String,
+ #[serde(rename = "resource.type", skip_serializing_if = "Option::is_none")]
+ pub resource_type: Option<String>,
+ #[serde(rename = "resource.id", skip_serializing_if = "Option::is_none")]
+ pub resource_id: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub index_uuid: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub index: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub request_id: Option<String>,
}
@@ -150,11 +167,39 @@ impl IntoResponse for ApiError {
error_type.clone()
};
+ // ES decorates `index_not_found_exception` with
+ // `resource.type`/`resource.id`/`index_uuid` and repeats `index` +
+ // `reason` at the top level, phrasing the reason as
+ // `no such index [name]`. Only apply this when the name is a real
+ // index name — several other 404s reuse this variant to carry a
+ // sentence (e.g. "index template [x] missing"), which must keep its
+ // existing shape. (RC4 Wave-3 item 4f.)
+ let (res_type, res_id, res_uuid, top_index) =
+ if matches!(self.inner, XerjError::IndexNotFound { .. }) {
+ match index_name.as_deref() {
+ Some(n) if is_plain_index_name(n) => {
+ reason = format!("no such index [{n}]");
+ (
+ Some("index_or_alias".to_string()),
+ Some(n.to_string()),
+ Some("_na_".to_string()),
+ Some(n.to_string()),
+ )
+ }
+ _ => (None, None, None, None),
+ }
+ } else {
+ (None, None, None, None)
+ };
+
let http_status =
StatusCode::from_u16(status_code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
let root_cause = vec![EsRootCause {
error_type: root_type,
reason: reason.clone(),
+ resource_type: res_type.clone(),
+ resource_id: res_id.clone(),
+ index_uuid: res_uuid.clone(),
index: index_name,
}];
@@ -163,6 +208,10 @@ impl IntoResponse for ApiError {
root_cause,
error_type,
reason,
+ resource_type: res_type,
+ resource_id: res_id,
+ index_uuid: res_uuid,
+ index: top_index,
request_id: self.request_id,
},
status: status_code,
@@ -253,6 +302,15 @@ fn extract_index_name(e: &XerjError) -> Option<String> {
}
}
+/// Whether a string is a plausible concrete index/alias name rather than one
+/// of the sentence-style messages some 404s smuggle through `IndexNotFound`
+/// (e.g. "index template [x] missing"). ES index names cannot contain spaces
+/// or brackets, so those characters reliably distinguish the two — only real
+/// names receive the `resource.*` / `no such index [name]` ES treatment.
+fn is_plain_index_name(name: &str) -> bool {
+ !name.is_empty() && !name.contains([' ', '[', ']'])
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Native API error response (with request_id, timing)
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index a3c1cb970..591f7d0b3 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -3,9 +3,11 @@
use std::time::Instant;
use axum::{
- extract::{Path, Query, State},
- http::StatusCode,
- response::IntoResponse,
+ async_trait,
+ body::Bytes,
+ extract::{FromRequest, Path, Query, Request, State},
+ http::{header, StatusCode},
+ response::{IntoResponse, Response},
Json,
};
use chrono::{Datelike, TimeZone, Timelike, Utc};
@@ -2142,6 +2144,547 @@ impl Default for EsSearchBody {
}
}
+// ─────────────────────────────────────────────────────────────────────────────
+// Request-validation bundle (RC4 Wave-3 item 4) — the silent-wrong-query class.
+//
+// Each shape below reproduces live ES 8.13.4 status + JSON so an existing ES
+// client sees the same 400 it would from a real cluster instead of a
+// misleading 200 with default (match-all) semantics.
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Full set of top-level keys ES 8.13.4 accepts on the `_search` request
+/// source (live-probed against the reference cluster), NOT merely the subset
+/// `EsSearchBody` models. Keys ES accepts but xerj does not act on
+/// (`terminate_after`, `post_filter`, `stats`, `ext`) stay accepted-and-ignored
+/// exactly as before — only genuinely-unknown keys are rejected.
+const ES_SEARCH_TOP_LEVEL_KEYS: &[&str] = &[
+ "query",
+ "from",
+ "size",
+ "timeout",
+ "terminate_after",
+ "min_score",
+ "post_filter",
+ "sort",
+ "track_scores",
+ "track_total_hits",
+ "indices_boost",
+ "_source",
+ "fields",
+ "docvalue_fields",
+ "stored_fields",
+ "script_fields",
+ "runtime_mappings",
+ "highlight",
+ "suggest",
+ "rescore",
+ "explain",
+ "version",
+ "seq_no_primary_term",
+ "aggs",
+ "aggregations",
+ "collapse",
+ "slice",
+ "search_after",
+ "pit",
+ "knn",
+ "profile",
+ "stats",
+ "ext",
+];
+
+/// A minimal position-tracking JSON scanner used solely to locate the first
+/// unknown top-level key in a `_search` body and report the ES-style
+/// (token-type, 1-based line, 1-based col) of its *value*. Deliberately
+/// conservative: on anything it does not understand it stops and reports "no
+/// unknown key", so genuinely malformed JSON falls through to the normal
+/// serde parse error rather than a misleading unknown-key message.
+struct SearchKeyScanner<'a> {
+ b: &'a [u8],
+ i: usize,
+ line: usize,
+ col: usize,
+}
+
+impl<'a> SearchKeyScanner<'a> {
+ fn new(b: &'a [u8]) -> Self {
+ Self {
+ b,
+ i: 0,
+ line: 1,
+ col: 1,
+ }
+ }
+ #[inline]
+ fn cur(&self) -> Option<u8> {
+ self.b.get(self.i).copied()
+ }
+ /// Advance one byte, keeping `line`/`col` pointed at the byte now under
+ /// the cursor (1-based, matching Jackson's reporting).
+ #[inline]
+ fn adv(&mut self) {
+ if let Some(c) = self.cur() {
+ if c == b'\n' {
+ self.line += 1;
+ self.col = 1;
+ } else {
+ self.col += 1;
+ }
+ self.i += 1;
+ }
+ }
+ fn skip_ws(&mut self) {
+ while let Some(c) = self.cur() {
+ if matches!(c, b' ' | b'\t' | b'\r' | b'\n') {
+ self.adv();
+ } else {
+ break;
+ }
+ }
+ }
+ /// At an opening quote, consume a whole JSON string and return its bytes as
+ /// a lossy String (escape decoding is irrelevant here — no allow-listed key
+ /// contains an escape). Advances past the closing quote; `None` if
+ /// unterminated.
+ fn read_string(&mut self) -> Option<String> {
+ if self.cur() != Some(b'"') {
+ return None;
+ }
+ self.adv(); // opening quote
+ let mut s = String::new();
+ while let Some(c) = self.cur() {
+ match c {
+ b'"' => {
+ self.adv();
+ return Some(s);
+ }
+ b'\\' => {
+ self.adv();
+ if let Some(e) = self.cur() {
+ s.push(e as char);
+ self.adv();
+ }
+ }
+ _ => {
+ s.push(c as char);
+ self.adv();
+ }
+ }
+ }
+ None
+ }
+ /// Skip a complete JSON value (object/array/string/primitive). Returns
+ /// false if it runs off the end.
+ fn skip_value(&mut self) -> bool {
+ self.skip_ws();
+ match self.cur() {
+ Some(b'"') => self.read_string().is_some(),
+ Some(b'{') | Some(b'[') => {
+ let mut depth = 0isize;
+ loop {
+ match self.cur() {
+ Some(b'"') => {
+ if self.read_string().is_none() {
+ return false;
+ }
+ }
+ Some(b'{') | Some(b'[') => {
+ depth += 1;
+ self.adv();
+ }
+ Some(b'}') | Some(b']') => {
+ depth -= 1;
+ self.adv();
+ if depth == 0 {
+ return true;
+ }
+ }
+ Some(_) => self.adv(),
+ None => return false,
+ }
+ }
+ }
+ Some(_) => {
+ while let Some(c) = self.cur() {
+ if matches!(c, b',' | b'}' | b']' | b' ' | b'\t' | b'\r' | b'\n') {
+ break;
+ }
+ self.adv();
+ }
+ true
+ }
+ None => false,
+ }
+ }
+}
+
+/// Classify a JSON value's first byte into ES's XContent token name.
+fn es_value_token(first: u8) -> &'static str {
+ match first {
+ b'"' => "VALUE_STRING",
+ b'{' => "START_OBJECT",
+ b'[' => "START_ARRAY",
+ b't' | b'f' => "VALUE_BOOLEAN",
+ b'n' => "VALUE_NULL",
+ _ => "VALUE_NUMBER",
+ }
+}
+
+/// Scan a raw `_search` body for the first top-level key ES would reject as
+/// unknown. Returns `(key, token_type, line, col)` — line/col are the 1-based
+/// position of the offending *value*, matching ES 8.13.4. `None` when every
+/// root key is allow-listed, or when the body is not a plain object / is too
+/// malformed to scan (serde then owns the error).
+fn first_unknown_search_key(bytes: &[u8]) -> Option<(String, &'static str, usize, usize)> {
+ let mut s = SearchKeyScanner::new(bytes);
+ s.skip_ws();
+ if s.cur() != Some(b'{') {
+ return None;
+ }
+ s.adv(); // consume '{'
+ loop {
+ s.skip_ws();
+ match s.cur() {
+ Some(b'}') => return None,
+ Some(b',') => {
+ s.adv();
+ continue;
+ }
+ Some(b'"') => {}
+ _ => return None,
+ }
+ let key = s.read_string()?;
+ s.skip_ws();
+ if s.cur() != Some(b':') {
+ return None;
+ }
+ s.adv(); // consume ':'
+ s.skip_ws();
+ let vc = s.cur()?;
+ if !ES_SEARCH_TOP_LEVEL_KEYS.contains(&key.as_str()) {
+ return Some((key, es_value_token(vc), s.line, s.col));
+ }
+ if !s.skip_value() {
+ return None;
+ }
+ }
+}
+
+/// Build ES's `parsing_exception` body for an unknown top-level search key
+/// (item 4a): `Unknown key for a VALUE_STRING in [foo].` with line/col.
+fn unknown_search_key_body(key: &str, token: &str, line: usize, col: usize) -> Value {
+ let reason = format!("Unknown key for a {token} in [{key}].");
+ json!({
+ "error": {
+ "root_cause": [{ "type": "parsing_exception", "reason": reason, "line": line, "col": col }],
+ "type": "parsing_exception",
+ "reason": reason,
+ "line": line,
+ "col": col,
+ },
+ "status": 400,
+ })
+}
+
+fn unknown_search_key_error(key: &str, token: &str, line: usize, col: usize) -> Response {
+ (
+ StatusCode::BAD_REQUEST,
+ Json(unknown_search_key_body(key, token, line, col)),
+ )
+ .into_response()
+}
+
+/// ES's shard-level validation envelope: a 400 `search_phase_execution_exception`
+/// ("all shards failed") wrapping an `illegal_argument_exception` root cause.
+/// Matches the house style xerj already uses for `_seq_no` sort rejections and
+/// ES's own search_after / rescore validation errors (items 4b, 4c).
+fn search_shard_validation_body(reason: &str) -> Value {
+ json!({
+ "error": {
+ "root_cause": [{ "type": "illegal_argument_exception", "reason": reason }],
+ "type": "search_phase_execution_exception",
+ "reason": "all shards failed",
+ },
+ "status": 400,
+ })
+}
+
+fn search_shard_validation_error(reason: &str) -> Response {
+ (
+ StatusCode::BAD_REQUEST,
+ Json(search_shard_validation_body(reason)),
+ )
+ .into_response()
+}
+
+/// Count `(total, field)` sort clauses in a raw ES `sort` value the way ES does
+/// for search_after: `total` counts every clause; `field` counts only clauses
+/// that are not `_score` (a `_score`-only sort produces no field for the
+/// FieldDoc, so search_after over it is rejected with "Sort must contain at
+/// least one field.").
+fn count_sort_clauses(sort: &Value) -> (usize, usize) {
+ fn clause_field(v: &Value) -> Option<&str> {
+ match v {
+ Value::String(s) => Some(s.as_str()),
+ Value::Object(o) => o.keys().next().map(String::as_str),
+ _ => None,
+ }
+ }
+ match sort {
+ Value::String(_) | Value::Object(_) => {
+ (1, usize::from(clause_field(sort) != Some("_score")))
+ }
+ Value::Array(arr) => (
+ arr.len(),
+ arr.iter()
+ .filter(|e| clause_field(e) != Some("_score"))
+ .count(),
+ ),
+ _ => (0, 0),
+ }
+}
+
+/// True when `sort` is effectively "sort by relevance descending" — a single
+/// `_score` clause with default/DESC order, or the bare string `"_score"`. ES
+/// permits `rescore` alongside such a sort but rejects any other sort
+/// ("Cannot use [sort] option in conjunction with [rescore].", item 4c).
+fn is_pure_score_desc_sort(sort: &Value) -> bool {
+ fn clause_is_score_desc(v: &Value) -> bool {
+ match v {
+ Value::String(s) => s == "_score",
+ Value::Object(o) => {
+ if o.len() != 1 {
+ return false;
+ }
+ let (k, ord) = o.iter().next().unwrap();
+ if k != "_score" {
+ return false;
+ }
+ match ord {
+ // `{"_score": "desc"}` — default order is DESC.
+ Value::String(s) => s.eq_ignore_ascii_case("desc"),
+ // `{"_score": {"order": "desc"}}`; absent order → DESC.
+ Value::Object(oo) => oo
+ .get("order")
+ .and_then(Value::as_str)
+ .map(|s| s.eq_ignore_ascii_case("desc"))
+ .unwrap_or(true),
+ _ => false,
+ }
+ }
+ _ => false,
+ }
+ }
+ match sort {
+ Value::String(s) => s == "_score",
+ Value::Object(_) => clause_is_score_desc(sort),
+ Value::Array(arr) => arr.len() == 1 && clause_is_score_desc(&arr[0]),
+ _ => false,
+ }
+}
+
+/// Reject the search-body semantic combinations ES rejects at query-build time
+/// (items 4b + 4c). Returns `Some(response)` with the ES-shaped 400 when
+/// invalid; `None` when the body is acceptable.
+fn validate_search_after_rescore(body: &EsSearchBody) -> Option<Response> {
+ // rescore is incompatible with any sort other than relevance (_score DESC).
+ if body.rescore.is_some() {
+ if let Some(sort) = body.sort.as_ref() {
+ if !is_pure_score_desc_sort(sort) {
+ return Some(search_shard_validation_error(
+ "Cannot use [sort] option in conjunction with [rescore].",
+ ));
+ }
+ }
+ }
+ // search_after arity / field validation. Non-array search_after is left to
+ // the engine (a documented minor gap — ES reports it as a token-level
+ // parsing_exception which needs the raw byte offset we no longer hold here).
+ if let Some(Value::Array(arr)) = body.search_after.as_ref() {
+ let (total_sort, field_sort) = body.sort.as_ref().map(count_sort_clauses).unwrap_or((0, 0));
+ if field_sort == 0 {
+ return Some(search_shard_validation_error(
+ "Sort must contain at least one field.",
+ ));
+ }
+ if arr.is_empty() {
+ return Some(search_shard_validation_error(
+ "Values must contains at least one value.",
+ ));
+ }
+ if arr.len() != total_sort {
+ return Some(search_shard_validation_error(&format!(
+ "search_after has {} value(s) but sort has {}.",
+ arr.len(),
+ total_sort
+ )));
+ }
+ }
+ None
+}
+
+/// Build ES's `illegal_argument_exception` for an unparseable integer URL
+/// parameter (item 4d): `?size=abc` → a JSON 400 with a `number_format_exception`
+/// cause, instead of axum's default `text/plain` 400.
+fn int_param_parse_body(param: &str, value: &str) -> Value {
+ let reason = format!("Failed to parse int parameter [{param}] with value [{value}]");
+ json!({
+ "error": {
+ "root_cause": [{ "type": "illegal_argument_exception", "reason": reason }],
+ "type": "illegal_argument_exception",
+ "reason": reason,
+ "caused_by": {
+ "type": "number_format_exception",
+ "reason": format!("For input string: \"{value}\""),
+ },
+ },
+ "status": 400,
+ })
+}
+
+fn int_param_parse_error(param: &str, value: &str) -> Response {
+ (
+ StatusCode::BAD_REQUEST,
+ Json(int_param_parse_body(param, value)),
+ )
+ .into_response()
+}
+
+/// ES's `illegal_argument_exception` for a scroll id that cannot be decoded
+/// (item 4e): `Cannot parse scroll id`, 400. Previously xerj answered 404
+/// `index_not_found_exception` with the whole "No search context found for id
+/// […]" sentence mis-stuffed into `root_cause[].index`.
+fn cannot_parse_scroll_id_body() -> Value {
+ json!({
+ "error": {
+ "root_cause": [{ "type": "illegal_argument_exception", "reason": "Cannot parse scroll id" }],
+ "type": "illegal_argument_exception",
+ "reason": "Cannot parse scroll id",
+ "caused_by": { "type": "e_o_f_exception", "reason": Value::Null },
+ },
+ "status": 400,
+ })
+}
+
+fn cannot_parse_scroll_id() -> Response {
+ (StatusCode::BAD_REQUEST, Json(cannot_parse_scroll_id_body())).into_response()
+}
+
+/// ES's 404 for a well-formed scroll id whose search context is gone
+/// (expired/cleared/unknown): `search_context_missing_exception` inside the
+/// shard-failure envelope — NOT an `index_not_found_exception` carrying the
+/// sentence as an index name.
+fn search_context_missing_body(scroll_id: &str) -> Value {
+ let reason = format!("No search context found for id [{scroll_id}]");
+ json!({
+ "error": {
+ "root_cause": [{ "type": "search_context_missing_exception", "reason": reason }],
+ "type": "search_phase_execution_exception",
+ "reason": "all shards failed",
+ },
+ "status": 404,
+ })
+}
+
+fn search_context_missing(scroll_id: &str) -> Response {
+ (
+ StatusCode::NOT_FOUND,
+ Json(search_context_missing_body(scroll_id)),
+ )
+ .into_response()
+}
+
+/// Body extractor for `_search` that layers ES 8.13.4's deny-unknown-key
+/// behaviour on top of `OptionalJson<EsSearchBody>` (item 4a): a bogus
+/// top-level key is rejected with the exact ES `parsing_exception` instead of
+/// being silently dropped and answered with a default match-all 200. Empty
+/// body → `None` (match-all default); malformed JSON and non-JSON
+/// `Content-Type` behave exactly like `OptionalJson`.
+pub struct EsSearchJson(pub Option<EsSearchBody>);
+
+impl EsSearchJson {
+ #[inline]
+ pub fn into_or_default(self) -> EsSearchBody {
+ self.0.unwrap_or_default()
+ }
+}
+
+/// Rejection for [`EsSearchJson`]: either a stock `OptionalJson` failure
+/// (body-read / malformed-JSON / unsupported media type) or an unknown
+/// top-level search key.
+pub enum EsSearchJsonRejection {
+ Optional(crate::extract::OptionalJsonRejection),
+ UnknownKey {
+ key: String,
+ token: &'static str,
+ line: usize,
+ col: usize,
+ },
+}
+
+impl IntoResponse for EsSearchJsonRejection {
+ fn into_response(self) -> Response {
+ match self {
+ EsSearchJsonRejection::Optional(r) => r.into_response(),
+ EsSearchJsonRejection::UnknownKey {
+ key,
+ token,
+ line,
+ col,
+ } => unknown_search_key_error(&key, token, line, col),
+ }
+ }
+}
+
+#[async_trait]
+impl<S: Send + Sync> FromRequest<S> for EsSearchJson {
+ type Rejection = EsSearchJsonRejection;
+
+ async fn from_request(req: Request, state: &S) -> Result<Self, Self::Rejection> {
+ use crate::extract::OptionalJsonRejection;
+ let ct = req
+ .headers()
+ .get(header::CONTENT_TYPE)
+ .and_then(|v| v.to_str().ok())
+ .map(str::to_owned);
+
+ let bytes = Bytes::from_request(req, state)
+ .await
+ .map_err(|e| EsSearchJsonRejection::Optional(OptionalJsonRejection::BodyRead(e)))?;
+
+ if bytes.is_empty() {
+ return Ok(EsSearchJson(None));
+ }
+
+ if let Some(ct_str) = ct {
+ let lower = ct_str.to_ascii_lowercase();
+ let is_json = lower.contains("json") || lower.starts_with("text/plain");
+ if !is_json {
+ return Err(EsSearchJsonRejection::Optional(
+ OptionalJsonRejection::UnsupportedMediaType(ct_str),
+ ));
+ }
+ }
+
+ // Deny unknown top-level keys (ES parity) before the typed parse, so
+ // the client sees the offending key rather than a downstream match-all
+ // 200. The scanner is conservative and bails to the serde error on
+ // malformed input.
+ if let Some((key, token, line, col)) = first_unknown_search_key(&bytes) {
+ return Err(EsSearchJsonRejection::UnknownKey {
+ key,
+ token,
+ line,
+ col,
+ });
+ }
+
+ let value: EsSearchBody = serde_json::from_slice(&bytes)
+ .map_err(|e| EsSearchJsonRejection::Optional(OptionalJsonRejection::JsonParse(e)))?;
+ Ok(EsSearchJson(Some(value)))
+ }
+}
+
/// Format a sort value according to an ES sort `format` string. When
/// `format` is `None` (or not a recognized date format), return the raw
/// value unchanged. Handles `strict_date_optional_time_nanos` (and its
@@ -4490,10 +5033,12 @@ fn parse_highlight(hl_val: &Value) -> Option<xerj_query::ast::HighlightRequest>
pub struct EsSearchQueryParams {
/// Simple query string — `?q=title:rust`.
pub q: Option<String>,
- /// Maximum hits to return.
- pub size: Option<usize>,
- /// Start offset.
- pub from: Option<usize>,
+ /// Maximum hits to return. Kept as a raw string so `?size=abc` reaches the
+ /// handler (which returns an ES-shaped 400) instead of failing at the axum
+ /// `Query` layer with a `text/plain` 400 (item 4d).
+ pub size: Option<String>,
+ /// Start offset. Raw string for the same reason as `size`.
+ pub from: Option<String>,
/// Sort field — `price:desc` or `_score`.
pub sort: Option<String>,
/// Comma-separated list of source fields to include.
@@ -4569,7 +5114,7 @@ pub struct EsSearchQueryParams {
pub async fn search_all(
State(state): State<AppState>,
Query(params): Query<EsSearchQueryParams>,
- body: OptionalJson<EsSearchBody>,
+ body: EsSearchJson,
) -> impl IntoResponse {
search(State(state), Path("*".to_string()), Query(params), body).await
}
@@ -4578,7 +5123,7 @@ pub async fn search(
State(state): State<AppState>,
Path(index): Path<String>,
Query(params): Query<EsSearchQueryParams>,
- body: OptionalJson<EsSearchBody>,
+ body: EsSearchJson,
) -> impl IntoResponse {
let started = Instant::now();
// Closed-index handling is deferred to after index resolution below so
@@ -4679,11 +5224,20 @@ pub async fn search(
}
body.query = Some(json!({ "query_string": qs }));
}
- if let Some(size) = params.size {
- body.size = size;
+ // Parse `?size=` / `?from=` here (not at the axum `Query` layer) so a
+ // non-numeric value yields ES's JSON `illegal_argument_exception` rather
+ // than axum's `text/plain` 400 (item 4d).
+ if let Some(ref size_s) = params.size {
+ match size_s.parse::<usize>() {
+ Ok(n) => body.size = n,
+ Err(_) => return int_param_parse_error("size", size_s),
+ }
}
- if let Some(from) = params.from {
- body.from = from;
+ if let Some(ref from_s) = params.from {
+ match from_s.parse::<usize>() {
+ Ok(n) => body.from = n,
+ Err(_) => return int_param_parse_error("from", from_s),
+ }
}
// URL-level `seq_no_primary_term=true` / `version=true` promote the
// corresponding body flag when the caller didn't already set one.
@@ -4738,6 +5292,18 @@ pub async fn search(
body.sort = Some(sort_val);
}
}
+
+ // Reject the search-body semantic combinations ES rejects at query-build
+ // time (items 4b + 4c): search_after without/against the wrong sort arity,
+ // and rescore alongside a non-relevance sort. Silently ignored before this
+ // (200 with wrong results); now the ES-shaped 400. Placed after the URL
+ // `?sort=` merge so a URL-provided sort counts toward the arity/field rules
+ // exactly as ES does (URL sort + body search_after is valid; URL sort +
+ // body rescore is rejected — both live-verified).
+ if let Some(err) = validate_search_after_rescore(&body) {
+ return err;
+ }
+
// `?_source=field1,field2`
if let Some(ref src_str) = params.source {
if body.source.is_none() {
@@ -13672,6 +14238,12 @@ pub async fn next_scroll(
}
};
+ // An id xerj never issued (its scroll ids are v4 UUIDs) cannot name a
+ // context: ES answers 400 `Cannot parse scroll id`, not a 404 (item 4e).
+ if Uuid::parse_str(&scroll_id).is_err() {
+ return cannot_parse_scroll_id();
+ }
+
match state.engine.scrolls.get_mut(&scroll_id) {
Some(mut ctx) => {
// Keep-alive enforcement (RC4 blocker 11): an expired context
@@ -13681,10 +14253,7 @@ pub async fn next_scroll(
if ctx.expires_at <= now {
drop(ctx); // release the DashMap shard guard before remove
state.engine.scrolls.remove(&scroll_id);
- let e = xerj_common::XerjError::index_not_found(format!(
- "No search context found for id [{scroll_id}]"
- ));
- return ApiError::new(e).into_response();
+ return search_context_missing(&scroll_id);
}
// Re-arm the deadline: an explicit `scroll` on the continuation
// (body wins over query param, like scroll_id) sets a new
@@ -13795,12 +14364,7 @@ pub async fn next_scroll(
}
Json(resp).into_response()
}
- None => {
- let e = xerj_common::XerjError::index_not_found(format!(
- "No search context found for id [{scroll_id}]"
- ));
- ApiError::new(e).into_response()
- }
+ None => search_context_missing(&scroll_id),
}
}
@@ -26880,3 +27444,298 @@ mod reindex_keyset_tests {
);
}
}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// RC4 Wave-3 item 4 — request-validation bundle tests.
+//
+// Every golden below was captured live from the ES 8.13.4 reference cluster
+// (`da95df1`) so each assertion is a byte-for-byte xerj==ES check on the
+// error shape, not a xerj-invented expectation.
+// ─────────────────────────────────────────────────────────────────────────────
+#[cfg(test)]
+mod request_validation_tests {
+ use super::*;
+ use serde_json::json;
+
+ // ── item 4a: deny_unknown_fields (scanner: token type + 1-based line/col) ──
+ #[test]
+ fn unknown_key_number_value_reports_es_token_and_col() {
+ // ES: "Unknown key for a VALUE_NUMBER in [bogus_key]." line 1 col 39.
+ let body = br#"{"query":{"match_all":{}},"bogus_key":123}"#;
+ assert_eq!(
+ first_unknown_search_key(body),
+ Some(("bogus_key".to_string(), "VALUE_NUMBER", 1, 39))
+ );
+ }
+
+ #[test]
+ fn unknown_key_string_value_col() {
+ // ES: "Unknown key for a VALUE_STRING in [foobar]." line 1 col 20.
+ let body = br#"{"size":5,"foobar":"hello"}"#;
+ assert_eq!(
+ first_unknown_search_key(body),
+ Some(("foobar".to_string(), "VALUE_STRING", 1, 20))
+ );
+ }
+
+ #[test]
+ fn unknown_key_all_value_tokens() {
+ assert_eq!(
+ first_unknown_search_key(br#"{"badobj":{"a":1}}"#),
+ Some(("badobj".to_string(), "START_OBJECT", 1, 11))
+ );
+ assert_eq!(
+ first_unknown_search_key(br#"{"badarr":[1,2]}"#),
+ Some(("badarr".to_string(), "START_ARRAY", 1, 11))
+ );
+ assert_eq!(
+ first_unknown_search_key(br#"{"badbool":true}"#),
+ Some(("badbool".to_string(), "VALUE_BOOLEAN", 1, 12))
+ );
+ assert_eq!(
+ first_unknown_search_key(br#"{"badnull":null}"#),
+ Some(("badnull".to_string(), "VALUE_NULL", 1, 12))
+ );
+ }
+
+ #[test]
+ fn top_level_match_is_unknown() {
+ // The gate's "Search body without query element" case: a bare `match`
+ // at the search-source top level is not a query wrapper → rejected.
+ assert_eq!(
+ first_unknown_search_key(br#"{"match":{"foo":"bar"}}"#),
+ Some(("match".to_string(), "START_OBJECT", 1, 10))
+ );
+ }
+
+ #[test]
+ fn multiline_body_reports_line_and_col() {
+ let body = b"{\n \"bogus\": 1\n}";
+ assert_eq!(
+ first_unknown_search_key(body),
+ Some(("bogus".to_string(), "VALUE_NUMBER", 2, 12))
+ );
+ }
+
+ #[test]
+ fn all_allow_listed_keys_pass() {
+ // Every key ES accepts on the search source must NOT be flagged.
+ let body = br#"{"query":{"match_all":{}},"size":5,"from":0,"sort":["name"],
+ "_source":true,"aggs":{},"post_filter":{},"terminate_after":10,
+ "stats":["a"],"ext":{},"track_total_hits":true,"min_score":0.5,
+ "search_after":["x"],"collapse":{"field":"f"},"pit":{"id":"z"}}"#;
+ assert_eq!(first_unknown_search_key(body), None);
+ }
+
+ #[test]
+ fn braces_inside_string_values_do_not_desync_scanner() {
+ // A '{' or '"' inside a string value must not be mistaken for
+ // structure and surface an inner key as top-level.
+ let body = br#"{"query":{"match":{"msg":"a {curly} brace and \"quoted\" text"}}}"#;
+ assert_eq!(first_unknown_search_key(body), None);
+ }
+
+ #[test]
+ fn non_object_body_defers_to_serde() {
+ assert_eq!(first_unknown_search_key(br#"[1,2,3]"#), None);
+ assert_eq!(first_unknown_search_key(br#""just a string""#), None);
+ assert_eq!(first_unknown_search_key(br#"not json at all"#), None);
+ }
+
+ #[test]
+ fn unknown_key_body_matches_es_golden() {
+ // Live ES 8.13.4 for {"foobar":"hello"} at col 20.
+ let es = json!({
+ "error": {
+ "root_cause": [{
+ "type": "parsing_exception",
+ "reason": "Unknown key for a VALUE_STRING in [foobar].",
+ "line": 1, "col": 20
+ }],
+ "type": "parsing_exception",
+ "reason": "Unknown key for a VALUE_STRING in [foobar].",
+ "line": 1, "col": 20
+ },
+ "status": 400
+ });
+ assert_eq!(unknown_search_key_body("foobar", "VALUE_STRING", 1, 20), es);
+ }
+
+ // ── item 4b: search_after sort counting ──
+ #[test]
+ fn sort_clause_counting_matches_es_semantics() {
+ assert_eq!(count_sort_clauses(&json!(["name"])), (1, 1));
+ assert_eq!(count_sort_clauses(&json!(["_score"])), (1, 0));
+ assert_eq!(count_sort_clauses(&json!(["name", "_score"])), (2, 1));
+ assert_eq!(count_sort_clauses(&json!(["name", "age"])), (2, 2));
+ assert_eq!(count_sort_clauses(&json!("name")), (1, 1));
+ assert_eq!(count_sort_clauses(&json!({"name": "desc"})), (1, 1));
+ assert_eq!(count_sort_clauses(&json!({"_score": "desc"})), (1, 0));
+ }
+
+ #[test]
+ fn search_after_without_field_sort_rejected() {
+ // No sort at all, and a _score-only sort, both → "Sort must contain
+ // at least one field." (ES fires this before arity).
+ for sort in [None, Some(json!(["_score"]))] {
+ let body = EsSearchBody {
+ sort,
+ search_after: Some(json!(["alice"])),
+ ..Default::default()
+ };
+ let v = validate_search_after_rescore(&body);
+ assert!(v.is_some());
+ }
+ }
+
+ #[test]
+ fn search_after_arity_mismatch_message_matches_es() {
+ let body = EsSearchBody {
+ sort: Some(json!(["name"])),
+ search_after: Some(json!(["alice", "extra"])),
+ ..Default::default()
+ };
+ assert!(validate_search_after_rescore(&body).is_some());
+ // Golden reason string, per live ES.
+ assert_eq!(
+ search_shard_validation_body("search_after has 2 value(s) but sort has 1."),
+ json!({
+ "error": {
+ "root_cause": [{
+ "type": "illegal_argument_exception",
+ "reason": "search_after has 2 value(s) but sort has 1."
+ }],
+ "type": "search_phase_execution_exception",
+ "reason": "all shards failed"
+ },
+ "status": 400
+ })
+ );
+ }
+
+ #[test]
+ fn search_after_matching_arity_accepted() {
+ // Includes the mixed field+_score case (arity counts _score).
+ for (sort, sa) in [
+ (json!(["name"]), json!(["alice"])),
+ (json!(["name", "_score"]), json!(["alice", 1.5])),
+ (json!({"name": "desc"}), json!(["alice"])),
+ ] {
+ let body = EsSearchBody {
+ sort: Some(sort.clone()),
+ search_after: Some(sa.clone()),
+ ..Default::default()
+ };
+ assert!(
+ validate_search_after_rescore(&body).is_none(),
+ "sort={sort} search_after={sa} should be valid"
+ );
+ }
+ }
+
+ // ── item 4c: rescore + sort ──
+ #[test]
+ fn pure_score_desc_sort_detection() {
+ assert!(is_pure_score_desc_sort(&json!(["_score"])));
+ assert!(is_pure_score_desc_sort(&json!([{"_score": "desc"}])));
+ assert!(is_pure_score_desc_sort(&json!("_score")));
+ assert!(is_pure_score_desc_sort(&json!({"_score": "desc"})));
+ assert!(!is_pure_score_desc_sort(&json!([{"_score": "asc"}])));
+ assert!(!is_pure_score_desc_sort(&json!(["_score", "name"])));
+ assert!(!is_pure_score_desc_sort(&json!(["name"])));
+ assert!(!is_pure_score_desc_sort(
+ &json!({"_score": {"order": "asc"}})
+ ));
+ }
+
+ #[test]
+ fn rescore_with_field_sort_rejected_but_score_sort_ok() {
+ let rescore = Some(json!({"query": {"rescore_query": {"match_all": {}}}}));
+ // field sort + rescore → reject
+ let bad = EsSearchBody {
+ sort: Some(json!(["name"])),
+ rescore: rescore.clone(),
+ ..Default::default()
+ };
+ assert!(validate_search_after_rescore(&bad).is_some());
+ // _score sort (or no sort) + rescore → allow
+ for sort in [
+ None,
+ Some(json!(["_score"])),
+ Some(json!([{"_score": "desc"}])),
+ ] {
+ let ok = EsSearchBody {
+ sort,
+ rescore: rescore.clone(),
+ ..Default::default()
+ };
+ assert!(validate_search_after_rescore(&ok).is_none());
+ }
+ }
+
+ // ── item 4d: size=abc ──
+ #[test]
+ fn int_param_parse_body_matches_es_golden() {
+ let es = json!({
+ "error": {
+ "root_cause": [{
+ "type": "illegal_argument_exception",
+ "reason": "Failed to parse int parameter [size] with value [abc]"
+ }],
+ "type": "illegal_argument_exception",
+ "reason": "Failed to parse int parameter [size] with value [abc]",
+ "caused_by": {
+ "type": "number_format_exception",
+ "reason": "For input string: \"abc\""
+ }
+ },
+ "status": 400
+ });
+ assert_eq!(int_param_parse_body("size", "abc"), es);
+ }
+
+ // ── item 4e: scroll_id ──
+ #[test]
+ fn cannot_parse_scroll_id_matches_es_golden() {
+ let es = json!({
+ "error": {
+ "root_cause": [{
+ "type": "illegal_argument_exception",
+ "reason": "Cannot parse scroll id"
+ }],
+ "type": "illegal_argument_exception",
+ "reason": "Cannot parse scroll id",
+ "caused_by": { "type": "e_o_f_exception", "reason": null }
+ },
+ "status": 400
+ });
+ assert_eq!(cannot_parse_scroll_id_body(), es);
+ }
+
+ #[test]
+ fn search_context_missing_shape() {
+ let v = search_context_missing_body("2937355");
+ assert_eq!(v["status"], json!(404));
+ assert_eq!(
+ v["error"]["root_cause"][0]["type"],
+ json!("search_context_missing_exception")
+ );
+ assert_eq!(
+ v["error"]["root_cause"][0]["reason"],
+ json!("No search context found for id [2937355]")
+ );
+ assert_eq!(
+ v["error"]["type"],
+ json!("search_phase_execution_exception")
+ );
+ // The sentence must NOT be smuggled into an `index` field.
+ assert!(v["error"]["root_cause"][0].get("index").is_none());
+ }
+
+ #[test]
+ fn valid_uuid_is_not_undecodable() {
+ // xerj issues v4 UUIDs; those must reach the context lookup, not 400.
+ assert!(uuid::Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").is_ok());
+ assert!(uuid::Uuid::parse_str("not-a-real-scroll-id").is_err());
+ }
+}
diff --git a/engine/crates/xerj-api/src/memory_api.rs b/engine/crates/xerj-api/src/memory_api.rs
index b7f7a9537..3381fc757 100644
--- a/engine/crates/xerj-api/src/memory_api.rs
+++ b/engine/crates/xerj-api/src/memory_api.rs
@@ -36,7 +36,8 @@ use serde_json::{json, Value};
use uuid::Uuid;
use crate::es_compat::{
- self, DeleteDocParams, DeleteIndexParams, EsSearchBody, EsSearchQueryParams, IndexDocParams,
+ self, DeleteDocParams, DeleteIndexParams, EsSearchBody, EsSearchJson, EsSearchQueryParams,
+ IndexDocParams,
};
use crate::extract::OptionalJson;
use crate::state::AppState;
@@ -250,7 +251,7 @@ async fn dedup_probe(
State(state.clone()),
Path(index.to_string()),
axum::extract::Query(EsSearchQueryParams::default()),
- OptionalJson(Some(search_body)),
+ EsSearchJson(Some(search_body)),
)
.await
.into_response();
@@ -525,7 +526,7 @@ pub async fn recall(
State(state.clone()),
Path(index.clone()),
axum::extract::Query(EsSearchQueryParams::default()),
- OptionalJson(Some(search_body)),
+ EsSearchJson(Some(search_body)),
)
.await
.into_response();
@@ -655,7 +656,7 @@ pub async fn list(State(state): State<AppState>, Path(namespace): Path<String>)
State(state.clone()),
Path(index.clone()),
axum::extract::Query(EsSearchQueryParams::default()),
- OptionalJson(Some(search_body)),
+ EsSearchJson(Some(search_body)),
)
.await
.into_response();