AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(search): sort _doc in arrival order, not lexicographic _id order
Saurav Kumar13470dcfd4b8
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 636df6251..f57ed00eb 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -65,6 +65,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0offsets and positions match on all 24 probed requests. Only the wording ofthe reason differs on 3 of the 400s.+- **`sort: [{"_doc": ...}]` is arrival order, not `_id` order** — the `_doc`+ sort key projected the document id string, so `_doc` pages ranked+ lexicographically by `_id` (matching neither ES index order nor XERJ's own+ stored order). The key is now the document's `seq_no`: numeric arrival+ order, unique per live doc (no ties), and a working `search_after` cursor.+ `_id` sort still echoes the id string.+## [1.0.0-rc.85] - 2026-10-06The cluster-ring-works window. One defect fix and one docs correction,diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 467d6679e..698df1066 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -48001,8 +48001,26 @@ fn compute_sort_values(// `448.39999_f32 as f64` prints 448.3999938964844 on the wire.let wire: f64 = score.to_string().parse().unwrap_or(score as f64);Value::Number(serde_json::Number::from_f64(wire).unwrap_or(serde_json::Number::from(0)))- } else if sf.is_doc_order() || sf.field == "_id" {- // `_id` (and `_doc`) is doc metadata, NOT part of `_source`, so a+ } else if sf.is_doc_order() {+ // `_doc` is physical index order (arrival order), NOT the doc+ // id: ES defines it as Lucene internal doc-id order. It used to+ // share `_id`'s arm below and project the id STRING, so+ // `sort:["_doc"]` ranked lexicographically by `_id` — matching+ // neither ES nor xerj's own stored order. Project the doc's+ // `seq_no` (arrival sequence, unique per live doc) as a Number+ // instead: ordering is numeric arrival order, ties are+ // impossible, and the emitted value pages as a `search_after`+ // cursor through the existing numeric path unchanged. A doc+ // unresolvable in the version map (concurrent tombstone between+ // collect and key computation) yields Null so the request's+ // `missing` policy governs it, the same convention+ // `meta_sort_value` uses.+ match idx.lookup_seq_no(id) {+ Some(seq) => Value::Number(seq.into()),+ None => Value::Null,+ }+ } else if sf.field == "_id" {+ // `_id` is doc metadata, NOT part of `_source`, so a// plain `get_field_value` returns Null — which breaks `_id`-sorted// `search_after` cursors (the cursor `[id]` has nothing to compare// against). Project the doc id as the sort value so `sort: [_id]`diff --git a/engine/crates/xerj-engine/tests/sort_doc_is_arrival_order.rs b/engine/crates/xerj-engine/tests/sort_doc_is_arrival_order.rsnew file mode 100644index 000000000..e1862ee2f--- /dev/null+++ b/engine/crates/xerj-engine/tests/sort_doc_is_arrival_order.rs@@ -0,0 +1,143 @@+//! `sort: [{"_doc": ...}]` is physical index (arrival) order, not `_id` order.+//!+//! ES defines `_doc` as index order — the order docs were written (Lucene+//! internal doc-id order), "no real use-case besides being the most efficient+//! sort order" (`sort-search-results`, ES 8.13 docs). `compute_sort_values`+//! (`engine/crates/xerj-engine/src/index.rs`) projected the `_id` STRING as+//! the `_doc` sort value, so `compare_sort_keys` ranked lexicographically by+//! id: `sort:["_doc"]` returned `alpha, mike, zeta` for docs inserted as+//! `zeta, alpha, mike` — matching neither ES nor xerj's own stored order+//! (tracked in `demo/playbooks/LOSS_BATTLE_PLAN.md` as the `sort:["_doc"]`+//! `_id`-string defect, explicitly split out as its own ticket).+//!+//! The fix projects the doc's `seq_no` (arrival sequence) as the `_doc` sort+//! value — a Number, so ordering is numeric arrival order, ties are+//! impossible (seq is unique per live doc), and the emitted `sort` array+//! doubles as a working `search_after` cursor through the existing numeric+//! cursor path (`normalize_search_after_value` passes numbers through+//! unchanged). `_id` sort keeps echoing the id string; only `_doc` changes.++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::Schema;+use xerj_engine::Engine;+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++fn doc_req(order: &str, size: usize) -> xerj_query::ast::SearchRequest {+ parse_request(&json!({+ "query": {"match_all": {}},+ "size": size,+ "sort": [{"_doc": order}],+ }))+ .expect("parse_request")+}++/// Ids chosen so lexicographic order differs from insertion order: inserted+/// `zeta, alpha, mike`, lexicographic ascending would be `alpha, mike, zeta`.+/// FAIL-BEFORE: the page comes back in `_id` string order.+#[tokio::test]+async fn doc_sort_is_arrival_order_not_id_order() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine.create_index("doc_sort", Schema::empty()).unwrap();+ let idx = engine.get_index("doc_sort").unwrap();++ for id in ["zeta", "alpha", "mike"] {+ idx.index_document(Some(id.to_string()), json!({ "n": id }))+ .await+ .unwrap();+ }++ let res = idx.search(&doc_req("asc", 10)).await.unwrap();+ let ids: Vec<&str> = res.hits.iter().map(|h| h.id.as_str()).collect();+ assert_eq!(+ ids,+ ["zeta", "alpha", "mike"],+ "_doc asc must be insertion order, not lexicographic _id order"+ );++ // The emitted sort value is the doc's arrival sequence number — a Number+ // that agrees with `lookup_seq_no`, so it pages as a cursor.+ for hit in &res.hits {+ assert_eq!(+ hit.sort.first().and_then(Value::as_u64),+ idx.lookup_seq_no(&hit.id),+ "hit {} must carry its seq_no as the _doc sort value, got {:?}",+ hit.id,+ hit.sort+ );+ }++ // Desc is the exact reverse.+ let res = idx.search(&doc_req("desc", 10)).await.unwrap();+ let ids: Vec<&str> = res.hits.iter().map(|h| h.id.as_str()).collect();+ assert_eq!(ids, ["mike", "alpha", "zeta"], "_doc desc page");++ // Keyset paging across the whole corpus: no dupes, no drops.+ let mut collected: Vec<String> = Vec::new();+ let mut after: Option<Vec<Value>> = None;+ for _ in 0..5 {+ let mut body = json!({+ "query": {"match_all": {}},+ "size": 2,+ "sort": [{"_doc": "asc"}],+ });+ if let Some(ref a) = after {+ body["search_after"] = Value::Array(a.clone());+ }+ let page = idx.search(&parse_request(&body).unwrap()).await.unwrap();+ if page.hits.is_empty() {+ break;+ }+ after = page.hits.last().map(|h| h.sort.clone());+ collected.extend(page.hits.iter().map(|h| h.id.clone()));+ }+ assert_eq!(+ collected,+ ["zeta", "alpha", "mike"],+ "search_after on _doc must walk insertion order exactly once"+ );+}++/// Same guarantee once the docs live in a segment: the segment scan path+/// resolves `_doc` through the version map too, not from `_source`.+#[tokio::test]+async fn doc_sort_is_arrival_order_after_flush() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine+ .create_index("doc_sort_seg", Schema::empty())+ .unwrap();+ let idx = engine.get_index("doc_sort_seg").unwrap();++ for id in ["doc-9", "doc-10", "doc-2"] {+ idx.index_document(Some(id.to_string()), json!({ "n": id }))+ .await+ .unwrap();+ }+ idx.flush().await.unwrap();++ let res = idx.search(&doc_req("asc", 10)).await.unwrap();+ let ids: Vec<&str> = res.hits.iter().map(|h| h.id.as_str()).collect();+ assert_eq!(+ ids,+ ["doc-9", "doc-10", "doc-2"],+ "flushed _doc asc must be insertion order (lexicographic would be doc-10, doc-2, doc-9)"+ );+ for hit in &res.hits {+ assert_eq!(+ hit.sort.first().and_then(Value::as_u64),+ idx.lookup_seq_no(&hit.id),+ "flushed hit {} must carry its seq_no, got {:?}",+ hit.id,+ hit.sort+ );+ }+}