AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(aggs): keep composite missing order consistent across pages (#2944)
Parijat Biswas9892820aa9ad
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex 852aab45f..10da320fd 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -8786,44 +8786,60 @@ fn run_composite(parts.next().unwrap_or(0),)}- let mut sorted_keys: Vec<Vec<String>> = bucket_map.keys().cloned().collect();- sorted_keys.sort_by(|a, b| {+ // Use exactly the same ordering for the initial sort and cursor filtering.+ // Explicit missing_order is absolute; only the default follows direction.+ let cmp_parts = |a: &[String], b: &[String]| -> std::cmp::Ordering {use std::cmp::Ordering;for i in 0..a.len().min(b.len()) {let (av, bv) = (&a[i], &b[i]);- let null_cmp = match missing_orders.get(i).copied().unwrap_or("default") {- "first" => (Ordering::Less, Ordering::Greater),- "last" => (Ordering::Greater, Ordering::Less),- _ => (Ordering::Less, Ordering::Greater), // "default" — first+ let asc = orders.get(i).copied().unwrap_or(true);+ let null_first = match missing_orders.get(i).copied().unwrap_or("default") {+ "first" => true,+ "last" => false,+ _ => asc,};- let cmp = if av == "__null__" && bv == "__null__" {- Ordering::Equal- } else if av == "__null__" {- null_cmp.0- } else if bv == "__null__" {- null_cmp.1- } else if geotile_src.get(i).copied().unwrap_or(false) {- // Geotile keys sort by (zoom, x, y) numerically.- geotile_parse(av).cmp(&geotile_parse(bv))- } else {- composite_part_cmp(- kinds.get(i).copied().unwrap_or(CompositeKeyKind::Heuristic),- numeric_src.get(i).copied().unwrap_or(false),- av,- bv,- )- };- let cmp = if orders.get(i).copied().unwrap_or(true) {- cmp- } else {- cmp.reverse()+ let cmp = match (av == "__null__", bv == "__null__") {+ (true, true) => Ordering::Equal,+ (true, false) => {+ if null_first {+ Ordering::Less+ } else {+ Ordering::Greater+ }+ }+ (false, true) => {+ if null_first {+ Ordering::Greater+ } else {+ Ordering::Less+ }+ }+ (false, false) => {+ let cmp = if geotile_src.get(i).copied().unwrap_or(false) {+ geotile_parse(av).cmp(&geotile_parse(bv))+ } else {+ composite_part_cmp(+ kinds.get(i).copied().unwrap_or(CompositeKeyKind::Heuristic),+ numeric_src.get(i).copied().unwrap_or(false),+ av,+ bv,+ )+ };+ if asc {+ cmp+ } else {+ cmp.reverse()+ }+ }};if cmp != Ordering::Equal {return cmp;}}a.len().cmp(&b.len())- });+ };+ let mut sorted_keys: Vec<Vec<String>> = bucket_map.keys().cloned().collect();+ sorted_keys.sort_by(|a, b| cmp_parts(a, b));// Filter by `after` param for pagination: drop every key <= after-key.if let Some(after_obj) = params.get("after").and_then(Value::as_object) {@@ -8901,40 +8917,6 @@ fn run_composite(.unwrap_or_default()}).collect();- // Compare keys against `after` using the same per-source- // numeric/lex semantics as the sort — otherwise `after: {long: 20}`- // would keep keys like `"100"` before `"20"` lexicographically.- let cmp_parts = |a: &[String], b: &[String]| -> std::cmp::Ordering {- use std::cmp::Ordering;- for i in 0..a.len().min(b.len()) {- let (av, bv) = (&a[i], &b[i]);- let c = if av == "__null__" && bv == "__null__" {- Ordering::Equal- } else if av == "__null__" {- Ordering::Less- } else if bv == "__null__" {- Ordering::Greater- } else if geotile_src.get(i).copied().unwrap_or(false) {- geotile_parse(av).cmp(&geotile_parse(bv))- } else {- composite_part_cmp(- kinds.get(i).copied().unwrap_or(CompositeKeyKind::Heuristic),- numeric_src.get(i).copied().unwrap_or(false),- av,- bv,- )- };- let c = if orders.get(i).copied().unwrap_or(true) {- c- } else {- c.reverse()- };- if c != Ordering::Equal {- return c;- }- }- a.len().cmp(&b.len())- };sorted_keys.retain(|k| cmp_parts(k, &after_key) == std::cmp::Ordering::Greater);}sorted_keys.truncate(size);@@ -15594,6 +15576,112 @@ mod tests {// ── RC4-W2 item 7: composite key typing from the field mapping ──────+ #[test]+ fn composite_missing_order_is_absolute_and_pages_match_full_result() {+ let docs = vec![json!({"tag":"1"}), json!({"tag":"2"}), json!({})];+ for order in ["asc", "desc"] {+ for missing_order in ["first", "last", "default"] {+ let mut spec = json!({"size":10,"sources":[{"tag":{"terms":{+ "field":"tag","order":order,"missing_bucket":true,"missing_order":missing_order+ }}}],"__xy_field_types__":{"tag":"keyword"}});+ let result = run_aggs(&json!({"c":{"composite":spec}}), &docs);+ let full: Vec<Value> = result["c"]["buckets"]+ .as_array()+ .unwrap()+ .iter()+ .map(|b| b["key"]["tag"].clone())+ .collect();+ let mut expected = if order == "asc" {+ vec![json!("1"), json!("2")]+ } else {+ vec![json!("2"), json!("1")]+ };+ if missing_order == "first" || (missing_order == "default" && order == "asc") {+ expected.insert(0, Value::Null);+ } else {+ expected.push(Value::Null);+ }+ assert_eq!(full, expected, "{order}/{missing_order}");+ spec["size"] = json!(1);+ let mut paged = Vec::new();+ for _ in 0..4 {+ let page = run_aggs(&json!({"c":{"composite":spec}}), &docs);+ let buckets = page["c"]["buckets"].as_array().unwrap();+ if buckets.is_empty() {+ break;+ }+ paged.push(buckets[0]["key"]["tag"].clone());+ spec["after"] = page["c"]["after_key"].clone();+ }+ assert_eq!(paged, expected, "pagination {order}/{missing_order}");+ }+ }+ }++ #[test]+ fn composite_missing_order_multisource_pages_preserve_lexicographic_order() {+ let values = [Value::Null, json!(2), json!(10)];+ let mut docs = Vec::new();+ for a in &values {+ for b in &values {+ let mut doc = json!({});+ if !a.is_null() {+ doc["a"] = a.clone();+ }+ if !b.is_null() {+ doc["b"] = b.clone();+ }+ docs.push(doc);+ }+ }+ fn ordered(order: &str, missing: &str) -> Vec<Value> {+ let mut v = if order == "asc" {+ vec![json!(2), json!(10)]+ } else {+ vec![json!(10), json!(2)]+ };+ if missing == "first" || (missing == "default" && order == "asc") {+ v.insert(0, Value::Null);+ } else {+ v.push(Value::Null);+ }+ v+ }+ for oa in ["asc", "desc"] {+ for ob in ["asc", "desc"] {+ for ma in ["first", "last", "default"] {+ for mb in ["first", "last", "default"] {+ let mut expected = Vec::new();+ for a in ordered(oa, ma) {+ for b in ordered(ob, mb) {+ expected.push(json!({"a": a, "b": b}));+ }+ }+ for size in [1, 2, 4, 20] {+ let mut spec = json!({"size": size, "sources": [+ {"a": {"terms": {"field":"a", "order":oa,+ "missing_bucket":true, "missing_order":ma}}},+ {"b": {"terms": {"field":"b", "order":ob,+ "missing_bucket":true, "missing_order":mb}}}+ ], "__xy_field_types__": {"a":"long", "b":"long"}});+ let mut found = Vec::new();+ for _ in 0..11 {+ let page = run_aggs(&json!({"c":{"composite":spec}}), &docs);+ let buckets = page["c"]["buckets"].as_array().unwrap();+ if buckets.is_empty() {+ break;+ }+ found.extend(buckets.iter().map(|b| b["key"].clone()));+ spec["after"] = page["c"]["after_key"].clone();+ }+ assert_eq!(found, expected, "{oa}/{ma}, {ob}/{mb}, size {size}");+ }+ }+ }+ }+ }+ }+#[test]fn composite_keyword_keys_stay_strings_and_sort_lexicographically() {let docs = vec![diff --git a/engine/tests/es-compat-yaml/yaml/aggregations/composite_missing_order_pagination.yml b/engine/tests/es-compat-yaml/yaml/aggregations/composite_missing_order_pagination.ymlnew file mode 100644index 000000000..6592f6b2f--- /dev/null+++ b/engine/tests/es-compat-yaml/yaml/aggregations/composite_missing_order_pagination.yml@@ -0,0 +1,70 @@+---+setup:+ - do:+ indices.create:+ index: missing-order-test+ body:+ mappings:+ properties:+ tag: { type: keyword }+ - do:+ index:+ index: missing-order-test+ id: "1"+ body: { tag: "1" }+ - do:+ index:+ index: missing-order-test+ id: "2"+ body: { tag: "2" }+ - do:+ index:+ index: missing-order-test+ id: "3"+ body: {}+ - do:+ indices.refresh:+ index: missing-order-test+---+"missing last survives ascending cursor pagination":+ - do:+ search:+ index: missing-order-test+ body:+ size: 0+ aggs:+ c:+ composite:+ size: 1+ after: { tag: "2" }+ sources:+ - tag:+ terms:+ field: tag+ missing_bucket: true+ missing_order: last+ - length: { aggregations.c.buckets: 1 }+ - match: { aggregations.c.buckets.0.key: { tag: null } }+ - match: { aggregations.c.buckets.0.doc_count: 1 }+---+"explicit missing first stays first under descending order":+ - do:+ search:+ index: missing-order-test+ body:+ size: 0+ aggs:+ c:+ composite:+ size: 10+ sources:+ - tag:+ terms:+ field: tag+ order: desc+ missing_bucket: true+ missing_order: first+ - length: { aggregations.c.buckets: 3 }+ - match: { aggregations.c.buckets.0.key: { tag: null } }+ - match: { aggregations.c.buckets.1.key.tag: "2" }+ - match: { aggregations.c.buckets.2.key.tag: "1" }