Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(search): reject sort on an unresolvable field, but honour unmapped_type (#437)

xerj-orgbf90446c8c41
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index d4552b188..6d1173020 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -6475,18 +6475,20 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {
mode: SortMode::default(),
missing: SortMissing::default(),
format: None,
+ unmapped_type: None,
},
};
fields.push(sf);
}
Value::Object(obj) => {
for (field_name, spec) in obj {
- let (order, mode, missing, format) = match spec {
+ let (order, mode, missing, format, unmapped_type) = match spec {
Value::String(ord) => (
parse_sort_order(ord),
SortMode::default(),
SortMissing::default(),
None,
+ None,
),
Value::Object(opts) => {
let order = opts
@@ -6509,13 +6511,18 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {
.unwrap_or_default();
let format =
opts.get("format").and_then(Value::as_str).map(String::from);
- (order, mode, missing, format)
+ let unmapped_type = opts
+ .get("unmapped_type")
+ .and_then(Value::as_str)
+ .map(String::from);
+ (order, mode, missing, format, unmapped_type)
}
_ => (
SortOrder::Asc,
SortMode::default(),
SortMissing::default(),
None,
+ None,
),
};
let sf = match field_name.as_str() {
@@ -6525,6 +6532,7 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {
mode,
missing,
format,
+ unmapped_type,
},
"_doc" => SortField {
field: "_doc".to_string(),
@@ -6532,6 +6540,7 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {
mode,
missing,
format,
+ unmapped_type,
},
other => SortField {
field: other.to_string(),
@@ -6539,6 +6548,7 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {
mode,
missing,
format,
+ unmapped_type,
},
};
fields.push(sf);
diff --git a/engine/crates/xerj-api/tests/sort_on_unresolvable_field_http.rs b/engine/crates/xerj-api/tests/sort_on_unresolvable_field_http.rs
index bc5ecce4f..5741a7c55 100644
--- a/engine/crates/xerj-api/tests/sort_on_unresolvable_field_http.rs
+++ b/engine/crates/xerj-api/tests/sort_on_unresolvable_field_http.rs
@@ -310,3 +310,59 @@ async fn sort_on_a_keyword_multi_field_and_a_nested_property_still_works() {
assert_eq!(body["hits"]["hits"].as_array().map(Vec::len), Some(2));
}
}
+
+/// The `#437` gate must NOT reject a sort on an unmapped field that carries
+/// ES's `unmapped_type` escape hatch: real Elasticsearch answers 200 and
+/// treats every document as missing, and XERJ's own memory-list endpoint
+/// issues exactly `{"stored_at":{"order":"desc","unmapped_type":"long"}}` as a
+/// guard for namespaces created before `stored_at` existed. Rejecting it (the
+/// #504 review's blocking finding) would 400 those requests. Both directions
+/// are pinned here so the exemption cannot silently regress in either sense.
+#[tokio::test]
+async fn sort_on_an_unmapped_field_with_unmapped_type_is_not_rejected() {
+ let (app, _dir) = app().await;
+ seed(&app, "umt", 4).await; // docs carry only `n`; `stored_at` is unmapped
+
+ // With unmapped_type: ES treats every doc as missing — 200, not an error.
+ let (status, body) = send(
+ &app,
+ Request::builder()
+ .method("POST")
+ .uri("/umt/_search")
+ .header("content-type", "application/json")
+ .body(Body::from(
+ json!({
+ "query": {"match_all": {}},
+ "sort": [{"stored_at": {"order": "desc", "unmapped_type": "long"}}]
+ })
+ .to_string(),
+ ))
+ .unwrap(),
+ )
+ .await;
+ assert_eq!(
+ status,
+ StatusCode::OK,
+ "sort on an unmapped field carrying unmapped_type must be answered (ES \
+ treats it as all-missing), not rejected: {body}"
+ );
+ assert_eq!(
+ body["hits"]["total"]["value"], 4,
+ "all four docs must come back, sorted as missing: {body}"
+ );
+
+ // Without unmapped_type the same field is still rejected (the #437 fix).
+ let (status, body) = send(
+ &app,
+ Request::builder()
+ .method("POST")
+ .uri("/umt/_search")
+ .header("content-type", "application/json")
+ .body(Body::from(
+ json!({"query": {"match_all": {}}, "sort": [{"stored_at": "desc"}]}).to_string(),
+ ))
+ .unwrap(),
+ )
+ .await;
+ assert_no_mapping_found_error(status, &body, "stored_at");
+}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index faddbe226..78ff05511 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -13768,6 +13768,7 @@ impl Index {
mode: xerj_query::sort::SortMode::default(),
missing: xerj_query::sort::SortMissing::default(),
format: None,
+ unmapped_type: None,
}];
r.leaf_ts_field = None;
Some(r)
@@ -13855,7 +13856,13 @@ impl Index {
{
continue;
}
- if declared_field(&schema.schema, &sf.field).is_none() {
+ // `unmapped_type` is ES's escape hatch: a sort on an unmapped
+ // field carrying it is NOT an error — every doc is treated as
+ // missing. Skip the guard so those requests keep working (real
+ // ES answers 200, and XERJ's own memory-list issues exactly
+ // this against a legacy namespace). #437 / #504 review.
+ if sf.unmapped_type.is_none() && declared_field(&schema.schema, &sf.field).is_none()
+ {
return Err(EngineError::Common(xerj_common::XerjError::invalid_query(
format!("No mapping found for [{}] in order to sort on", sf.field),
)));
diff --git a/engine/crates/xerj-engine/src/sql.rs b/engine/crates/xerj-engine/src/sql.rs
index fb27797ae..a8323e518 100644
--- a/engine/crates/xerj-engine/src/sql.rs
+++ b/engine/crates/xerj-engine/src/sql.rs
@@ -376,6 +376,7 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {
mode: xerj_query::sort::SortMode::default(),
missing: xerj_query::sort::SortMissing::default(),
format: None,
+ unmapped_type: None,
});
if let Some(Token::Comma) = peek(pos) {
pos += 1;
diff --git a/engine/crates/xerj-query/src/executor.rs b/engine/crates/xerj-query/src/executor.rs
index 2ae835208..c46690a0b 100644
--- a/engine/crates/xerj-query/src/executor.rs
+++ b/engine/crates/xerj-query/src/executor.rs
@@ -618,6 +618,7 @@ mod tests {
mode: SortMode::default(),
missing: SortMissing::Last,
format: None,
+ unmapped_type: None,
}];
let merged = merge_hits(vec![vec![h1, h2, h3]], 0, 10, &sort_fields);
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index 8f8b0737d..2b6f78b2a 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -3197,6 +3197,7 @@ fn parse_sort_field_str(s: &str) -> SortField {
mode: SortMode::default(),
missing: SortMissing::default(),
format: None,
+ unmapped_type: None,
},
"_doc" => SortField {
field: "_doc".to_string(),
@@ -3204,6 +3205,7 @@ fn parse_sort_field_str(s: &str) -> SortField {
mode: SortMode::default(),
missing: SortMissing::default(),
format: None,
+ unmapped_type: None,
},
other => SortField {
field: other.to_string(),
@@ -3211,6 +3213,7 @@ fn parse_sort_field_str(s: &str) -> SortField {
mode: SortMode::default(),
missing: SortMissing::default(),
format: None,
+ unmapped_type: None,
},
}
}
@@ -3224,6 +3227,7 @@ fn parse_sort_field_spec(field: &str, spec: &Value) -> Result<SortField> {
mode: SortMode::default(),
missing: SortMissing::default(),
format: None,
+ unmapped_type: None,
});
}
@@ -3253,6 +3257,10 @@ fn parse_sort_field_spec(field: &str, spec: &Value) -> Result<SortField> {
};
let format = obj.get("format").and_then(|v| v.as_str()).map(String::from);
+ let unmapped_type = obj
+ .get("unmapped_type")
+ .and_then(|v| v.as_str())
+ .map(String::from);
Ok(SortField {
field: field.to_string(),
@@ -3260,6 +3268,7 @@ fn parse_sort_field_spec(field: &str, spec: &Value) -> Result<SortField> {
mode,
missing,
format,
+ unmapped_type,
})
}
diff --git a/engine/crates/xerj-query/src/sort.rs b/engine/crates/xerj-query/src/sort.rs
index a595cec13..2193482d2 100644
--- a/engine/crates/xerj-query/src/sort.rs
+++ b/engine/crates/xerj-query/src/sort.rs
@@ -34,6 +34,13 @@ pub struct SortField {
/// than the raw numeric epoch that the engine produces by default.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub format: Option<String>,
+ /// ES `unmapped_type`: when set, sorting a field that has no mapping is not
+ /// an error — every document is treated as missing (ES semantics). The
+ /// unresolvable-field guard skips a field carrying this, so a client that
+ /// asks for `unmapped_type` (e.g. a cross-index sort, or a query written to
+ /// survive a schema that predates the field) is not rejected.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub unmapped_type: Option<String>,
}
impl SortField {
@@ -45,6 +52,7 @@ impl SortField {
mode: SortMode::default(),
missing: SortMissing::default(),
format: None,
+ unmapped_type: None,
}
}
@@ -56,6 +64,7 @@ impl SortField {
mode: SortMode::default(),
missing: SortMissing::default(),
format: None,
+ unmapped_type: None,
}
}
@@ -279,6 +288,7 @@ mod tests {
mode: SortMode::default(),
missing: SortMissing::Last,
format: None,
+ unmapped_type: None,
}];
let a = vec![json!(1)];
let b = vec![json!(2)];
@@ -293,6 +303,7 @@ mod tests {
mode: SortMode::default(),
missing: SortMissing::Last,
format: None,
+ unmapped_type: None,
}];
let a = vec![json!(2)];
let b = vec![json!(1)];
@@ -307,6 +318,7 @@ mod tests {
mode: SortMode::default(),
missing: SortMissing::Last,
format: None,
+ unmapped_type: None,
}];
// null sorts after non-null
assert_eq!(
@@ -323,6 +335,7 @@ mod tests {
mode: SortMode::default(),
missing: SortMissing::First,
format: None,
+ unmapped_type: None,
}];
assert_eq!(
compare_sort_keys(&[json!(null)], &[json!(1)], &fields),
@@ -339,6 +352,7 @@ mod tests {
mode: SortMode::default(),
missing: SortMissing::Last,
format: None,
+ unmapped_type: None,
},
SortField {
field: "name".to_string(),
@@ -346,6 +360,7 @@ mod tests {
mode: SortMode::default(),
missing: SortMissing::Last,
format: None,
+ unmapped_type: None,
},
];
// Same date, "alice" < "bob"