Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): _search_scroll first page carries _seq_no/_version when requested (#630)

xerj-org39aa4093a825
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index f562dcecc..389954272 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -20204,6 +20204,53 @@ pub async fn search_with_scroll(
};
state.engine.scrolls.insert(scroll_id.clone(), ctx);
+ // #630: emit `_version`/`_seq_no`/`_primary_term` when requested. The
+ // `EsHit` carried them (gated on the request flags), but this hand-built
+ // first-page map previously dropped them, unlike the continuation page
+ // (`scroll_page_response`, #428) and the `_search?scroll=` first page
+ // (`search_impl`). `_source` emission is unchanged (always present, null
+ // when absent). The `disable_sequence_numbers` sentinel is then applied
+ // exactly as the continuation does (#440), so both pages agree.
+ let mut hits_json: Vec<Value> = first_page
+ .iter()
+ .map(|h| {
+ let mut o = serde_json::Map::new();
+ o.insert("_index".to_string(), Value::String(h.index.clone()));
+ o.insert("_id".to_string(), Value::String(h.id.clone()));
+ o.insert(
+ "_score".to_string(),
+ match h.score {
+ Some(s) => json!(s),
+ None => Value::Null,
+ },
+ );
+ if let Some(v) = h.version {
+ o.insert("_version".to_string(), json!(v));
+ }
+ if let Some(sn) = h.seq_no {
+ o.insert("_seq_no".to_string(), json!(sn));
+ }
+ if let Some(pt) = h.primary_term {
+ o.insert("_primary_term".to_string(), json!(pt));
+ }
+ o.insert(
+ "_source".to_string(),
+ match &h.source {
+ Some(src) => src.clone(),
+ None => Value::Null,
+ },
+ );
+ if let Some(fields) = &h.fields {
+ o.insert(
+ "fields".to_string(),
+ serde_json::to_value(fields).unwrap_or(Value::Null),
+ );
+ }
+ Value::Object(o)
+ })
+ .collect();
+ apply_disable_seqno_sentinel(&state, &mut hits_json);
+
let resp = json!({
"_scroll_id": scroll_id,
"took": took_ms,
@@ -20212,18 +20259,7 @@ pub async fn search_with_scroll(
"hits": {
"total": { "value": total_count, "relation": "eq" },
"max_score": first_page.first().and_then(|h| h.score),
- "hits": first_page.iter().map(|h| {
- let mut hit = json!({
- "_index": h.index,
- "_id": h.id,
- "_score": h.score,
- "_source": h.source,
- });
- if let Some(fields) = &h.fields {
- hit["fields"] = serde_json::to_value(fields).unwrap_or(Value::Null);
- }
- hit
- }).collect::<Vec<_>>()
+ "hits": hits_json
}
});
return Json(resp).into_response();
diff --git a/engine/crates/xerj-api/tests/search_scroll_firstpage_versioning.rs b/engine/crates/xerj-api/tests/search_scroll_firstpage_versioning.rs
new file mode 100644
index 000000000..ed559f753
--- /dev/null
+++ b/engine/crates/xerj-api/tests/search_scroll_firstpage_versioning.rs
@@ -0,0 +1,102 @@
+//! Issue #630: the `POST /{index}/_search_scroll` (alias-route scroll) FIRST page
+//! must carry `_seq_no`/`_version`/`_primary_term` when the request asks for them.
+//!
+//! The continuation path (`scroll_page_response`) emits them (#428), and the
+//! `_search?scroll=` first page (rendered by `search_impl`) emits them too, but
+//! `search_with_scroll`'s first-page response is hand-built and only serialized
+//! `_index`/`_id`/`_score`/`_source`(+`fields`) — the `EsHit` carried the
+//! versioning pair, but the map never read it out. So a client that opens a
+//! scroll via this route with `seq_no_primary_term`/`version` got them on every
+//! CONTINUATION page but not the first — a wire divergence from ES.
+//!
+//! ES is referenced for wire semantics only; no ES code is here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+/// A scroll opened via `_search_scroll` must carry the requested versioning pair
+/// on its FIRST page, matching the continuation page and the `_search?scroll=`
+/// first page (#630).
+#[tokio::test]
+async fn search_scroll_first_page_carries_seq_no_and_version_when_requested() {
+ let (app, _dir) = app().await;
+ let (st, _) = call(&app, "POST", "/docs/_doc/d1", json!({ "v": 1 })).await;
+ assert_eq!(st, StatusCode::CREATED, "index d1");
+ let _ = call(&app, "POST", "/docs/_refresh", Value::Null).await;
+
+ // Open a scroll via the alias route, requesting the versioning pair.
+ let (st, first) = call(
+ &app,
+ "POST",
+ "/docs/_search_scroll?scroll=1m",
+ json!({
+ "query": { "match_all": {} },
+ "size": 10,
+ "seq_no_primary_term": true,
+ "version": true
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "search_scroll first page: {first}");
+ let hit = &first["hits"]["hits"][0];
+ assert_eq!(hit["_id"], "d1", "first page returns d1: {first}");
+
+ // #630: the first page must carry the requested versioning pair — and the
+ // REAL engine values, not fabricated ones. Cross-check against a plain GET,
+ // the ground truth (the same read the engine resolves `_source` from). This
+ // catches both the omission bug AND any future value-fabrication regression.
+ assert!(
+ hit.get("_seq_no").is_some()
+ && hit.get("_version").is_some()
+ && hit.get("_primary_term").is_some(),
+ "first page must carry the full versioning triple when requested (#630): {hit}"
+ );
+ let (st, got) = call(&app, "GET", "/docs/_doc/d1", Value::Null).await;
+ assert_eq!(st, StatusCode::OK, "GET d1 ground truth: {got}");
+ assert_eq!(
+ hit["_seq_no"], got["_seq_no"],
+ "first-page _seq_no must be the real value, not fabricated (#630): first={hit} got={got}"
+ );
+ assert_eq!(
+ hit["_version"], got["_version"],
+ "first-page _version must be the real value, not fabricated (#630): first={hit} got={got}"
+ );
+ // `_primary_term` is the engine's placeholder alongside a real `_seq_no`.
+ assert_eq!(
+ hit["_primary_term"],
+ json!(1),
+ "first-page _primary_term must be 1 alongside a real _seq_no (#630): {hit}"
+ );
+}