AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(search): reject sort on an unresolvable field, but honour unmapped_type (#437)
xerj-orgbf90446c8c41
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex d4552b188..6d1173020 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -6475,18 +6475,20 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {mode: SortMode::default(),missing: SortMissing::default(),format: None,+ unmapped_type: None,},};fields.push(sf);}Value::Object(obj) => {for (field_name, spec) in obj {- let (order, mode, missing, format) = match spec {+ let (order, mode, missing, format, unmapped_type) = match spec {Value::String(ord) => (parse_sort_order(ord),SortMode::default(),SortMissing::default(),None,+ None,),Value::Object(opts) => {let order = opts@@ -6509,13 +6511,18 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {.unwrap_or_default();let format =opts.get("format").and_then(Value::as_str).map(String::from);- (order, mode, missing, format)+ let unmapped_type = opts+ .get("unmapped_type")+ .and_then(Value::as_str)+ .map(String::from);+ (order, mode, missing, format, unmapped_type)}_ => (SortOrder::Asc,SortMode::default(),SortMissing::default(),None,+ None,),};let sf = match field_name.as_str() {@@ -6525,6 +6532,7 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {mode,missing,format,+ unmapped_type,},"_doc" => SortField {field: "_doc".to_string(),@@ -6532,6 +6540,7 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {mode,missing,format,+ unmapped_type,},other => SortField {field: other.to_string(),@@ -6539,6 +6548,7 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {mode,missing,format,+ unmapped_type,},};fields.push(sf);diff --git a/engine/crates/xerj-api/tests/sort_on_unresolvable_field_http.rs b/engine/crates/xerj-api/tests/sort_on_unresolvable_field_http.rsindex bc5ecce4f..5741a7c55 100644--- a/engine/crates/xerj-api/tests/sort_on_unresolvable_field_http.rs+++ b/engine/crates/xerj-api/tests/sort_on_unresolvable_field_http.rs@@ -310,3 +310,59 @@ async fn sort_on_a_keyword_multi_field_and_a_nested_property_still_works() {assert_eq!(body["hits"]["hits"].as_array().map(Vec::len), Some(2));}}++/// The `#437` gate must NOT reject a sort on an unmapped field that carries+/// ES's `unmapped_type` escape hatch: real Elasticsearch answers 200 and+/// treats every document as missing, and XERJ's own memory-list endpoint+/// issues exactly `{"stored_at":{"order":"desc","unmapped_type":"long"}}` as a+/// guard for namespaces created before `stored_at` existed. Rejecting it (the+/// #504 review's blocking finding) would 400 those requests. Both directions+/// are pinned here so the exemption cannot silently regress in either sense.+#[tokio::test]+async fn sort_on_an_unmapped_field_with_unmapped_type_is_not_rejected() {+ let (app, _dir) = app().await;+ seed(&app, "umt", 4).await; // docs carry only `n`; `stored_at` is unmapped++ // With unmapped_type: ES treats every doc as missing — 200, not an error.+ let (status, body) = send(+ &app,+ Request::builder()+ .method("POST")+ .uri("/umt/_search")+ .header("content-type", "application/json")+ .body(Body::from(+ json!({+ "query": {"match_all": {}},+ "sort": [{"stored_at": {"order": "desc", "unmapped_type": "long"}}]+ })+ .to_string(),+ ))+ .unwrap(),+ )+ .await;+ assert_eq!(+ status,+ StatusCode::OK,+ "sort on an unmapped field carrying unmapped_type must be answered (ES \+ treats it as all-missing), not rejected: {body}"+ );+ assert_eq!(+ body["hits"]["total"]["value"], 4,+ "all four docs must come back, sorted as missing: {body}"+ );++ // Without unmapped_type the same field is still rejected (the #437 fix).+ let (status, body) = send(+ &app,+ Request::builder()+ .method("POST")+ .uri("/umt/_search")+ .header("content-type", "application/json")+ .body(Body::from(+ json!({"query": {"match_all": {}}, "sort": [{"stored_at": "desc"}]}).to_string(),+ ))+ .unwrap(),+ )+ .await;+ assert_no_mapping_found_error(status, &body, "stored_at");+}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex faddbe226..78ff05511 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -13768,6 +13768,7 @@ impl Index {mode: xerj_query::sort::SortMode::default(),missing: xerj_query::sort::SortMissing::default(),format: None,+ unmapped_type: None,}];r.leaf_ts_field = None;Some(r)@@ -13855,7 +13856,13 @@ impl Index {{continue;}- if declared_field(&schema.schema, &sf.field).is_none() {+ // `unmapped_type` is ES's escape hatch: a sort on an unmapped+ // field carrying it is NOT an error — every doc is treated as+ // missing. Skip the guard so those requests keep working (real+ // ES answers 200, and XERJ's own memory-list issues exactly+ // this against a legacy namespace). #437 / #504 review.+ if sf.unmapped_type.is_none() && declared_field(&schema.schema, &sf.field).is_none()+ {return Err(EngineError::Common(xerj_common::XerjError::invalid_query(format!("No mapping found for [{}] in order to sort on", sf.field),)));diff --git a/engine/crates/xerj-engine/src/sql.rs b/engine/crates/xerj-engine/src/sql.rsindex fb27797ae..a8323e518 100644--- a/engine/crates/xerj-engine/src/sql.rs+++ b/engine/crates/xerj-engine/src/sql.rs@@ -376,6 +376,7 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {mode: xerj_query::sort::SortMode::default(),missing: xerj_query::sort::SortMissing::default(),format: None,+ unmapped_type: None,});if let Some(Token::Comma) = peek(pos) {pos += 1;diff --git a/engine/crates/xerj-query/src/executor.rs b/engine/crates/xerj-query/src/executor.rsindex 2ae835208..c46690a0b 100644--- a/engine/crates/xerj-query/src/executor.rs+++ b/engine/crates/xerj-query/src/executor.rs@@ -618,6 +618,7 @@ mod tests {mode: SortMode::default(),missing: SortMissing::Last,format: None,+ unmapped_type: None,}];let merged = merge_hits(vec![vec![h1, h2, h3]], 0, 10, &sort_fields);diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rsindex 8f8b0737d..2b6f78b2a 100644--- a/engine/crates/xerj-query/src/parser.rs+++ b/engine/crates/xerj-query/src/parser.rs@@ -3197,6 +3197,7 @@ fn parse_sort_field_str(s: &str) -> SortField {mode: SortMode::default(),missing: SortMissing::default(),format: None,+ unmapped_type: None,},"_doc" => SortField {field: "_doc".to_string(),@@ -3204,6 +3205,7 @@ fn parse_sort_field_str(s: &str) -> SortField {mode: SortMode::default(),missing: SortMissing::default(),format: None,+ unmapped_type: None,},other => SortField {field: other.to_string(),@@ -3211,6 +3213,7 @@ fn parse_sort_field_str(s: &str) -> SortField {mode: SortMode::default(),missing: SortMissing::default(),format: None,+ unmapped_type: None,},}}@@ -3224,6 +3227,7 @@ fn parse_sort_field_spec(field: &str, spec: &Value) -> Result<SortField> {mode: SortMode::default(),missing: SortMissing::default(),format: None,+ unmapped_type: None,});}@@ -3253,6 +3257,10 @@ fn parse_sort_field_spec(field: &str, spec: &Value) -> Result<SortField> {};let format = obj.get("format").and_then(|v| v.as_str()).map(String::from);+ let unmapped_type = obj+ .get("unmapped_type")+ .and_then(|v| v.as_str())+ .map(String::from);Ok(SortField {field: field.to_string(),@@ -3260,6 +3268,7 @@ fn parse_sort_field_spec(field: &str, spec: &Value) -> Result<SortField> {mode,missing,format,+ unmapped_type,})}diff --git a/engine/crates/xerj-query/src/sort.rs b/engine/crates/xerj-query/src/sort.rsindex a595cec13..2193482d2 100644--- a/engine/crates/xerj-query/src/sort.rs+++ b/engine/crates/xerj-query/src/sort.rs@@ -34,6 +34,13 @@ pub struct SortField {/// than the raw numeric epoch that the engine produces by default.#[serde(default, skip_serializing_if = "Option::is_none")]pub format: Option<String>,+ /// ES `unmapped_type`: when set, sorting a field that has no mapping is not+ /// an error — every document is treated as missing (ES semantics). The+ /// unresolvable-field guard skips a field carrying this, so a client that+ /// asks for `unmapped_type` (e.g. a cross-index sort, or a query written to+ /// survive a schema that predates the field) is not rejected.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub unmapped_type: Option<String>,}impl SortField {@@ -45,6 +52,7 @@ impl SortField {mode: SortMode::default(),missing: SortMissing::default(),format: None,+ unmapped_type: None,}}@@ -56,6 +64,7 @@ impl SortField {mode: SortMode::default(),missing: SortMissing::default(),format: None,+ unmapped_type: None,}}@@ -279,6 +288,7 @@ mod tests {mode: SortMode::default(),missing: SortMissing::Last,format: None,+ unmapped_type: None,}];let a = vec![json!(1)];let b = vec![json!(2)];@@ -293,6 +303,7 @@ mod tests {mode: SortMode::default(),missing: SortMissing::Last,format: None,+ unmapped_type: None,}];let a = vec![json!(2)];let b = vec![json!(1)];@@ -307,6 +318,7 @@ mod tests {mode: SortMode::default(),missing: SortMissing::Last,format: None,+ unmapped_type: None,}];// null sorts after non-nullassert_eq!(@@ -323,6 +335,7 @@ mod tests {mode: SortMode::default(),missing: SortMissing::First,format: None,+ unmapped_type: None,}];assert_eq!(compare_sort_keys(&[json!(null)], &[json!(1)], &fields),@@ -339,6 +352,7 @@ mod tests {mode: SortMode::default(),missing: SortMissing::Last,format: None,+ unmapped_type: None,},SortField {field: "name".to_string(),@@ -346,6 +360,7 @@ mod tests {mode: SortMode::default(),missing: SortMissing::Last,format: None,+ unmapped_type: None,},];// Same date, "alice" < "bob"