Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(search): echo pit_id in PIT responses + ES-exact pinned sentinel scores

xerj-org096c743526de
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 7254f6185..d232f039e 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -9459,6 +9459,30 @@ pub async fn search(
})
};
+ // PIT searches echo the (still-valid) `pit_id` back in the response —
+ // ES 8.x places it as the FIRST key, before `took` (live-verified on
+ // 8.13.4). Clients' PIT + search_after loops read the id off every
+ // page and thread it into the next request, so omitting it breaks the
+ // canonical pagination pattern (ES helpers fail with a missing-pit_id
+ // error). xerj PIT ids are stable, so the echoed id is the request id.
+ if pit_context.is_some() {
+ if let Some(pit_id) = body
+ .pit
+ .as_ref()
+ .and_then(|p| p.get("id"))
+ .and_then(Value::as_str)
+ {
+ if let Some(obj) = response_body.as_object_mut() {
+ let mut rebuilt = serde_json::Map::with_capacity(obj.len() + 1);
+ rebuilt.insert("pit_id".to_string(), Value::String(pit_id.to_string()));
+ for (k, v) in obj.iter() {
+ rebuilt.insert(k.clone(), v.clone());
+ }
+ *obj = rebuilt;
+ }
+ }
+ }
+
// ES 8.x emits `"terminated_early": false` (immediately after
// `timed_out`) on the hits-skipping exact-count path: `size: 0` +
// explicit `track_total_hits: true`, without aggregations. Live-probed
@@ -14136,6 +14160,25 @@ async fn msearch_impl(
if let Some(aggs) = merged_aggs {
resp["aggregations"] = aggs;
}
+ // PIT sub-searches echo the pit_id first, exactly like `_search`
+ // (ES 8.x msearch items carry it too — same search_after loop
+ // contract per item).
+ if pit_context.is_some() {
+ if let Some(pit_id) = search_body_val
+ .get("pit")
+ .and_then(|p| p.get("id"))
+ .and_then(Value::as_str)
+ {
+ if let Some(obj) = resp.as_object_mut() {
+ let mut rebuilt = serde_json::Map::with_capacity(obj.len() + 1);
+ rebuilt.insert("pit_id".to_string(), Value::String(pit_id.to_string()));
+ for (k, v) in obj.iter() {
+ rebuilt.insert(k.clone(), v.clone());
+ }
+ *obj = rebuilt;
+ }
+ }
+ }
responses.push(resp);
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 160363ed3..4cde04893 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -7990,17 +7990,20 @@ impl Index {
if scored_fast_applied {
// exact pin scores already in place
} else if let QueryNode::Pinned { ids, .. } = query {
- let max_organic_score = final_hits
- .iter()
- .map(|h| h.score)
- .fold(f32::NEG_INFINITY, f32::max);
- let pin_score = (max_organic_score + 1.0).max(1.0);
+ // ES pin scores are sentinel constants descending one float ULP
+ // per rank from just above 2^127 (1.7014124e38 for the first pin
+ // with two ids — live-verified vs ES 8.13.4). The columnar
+ // Pinned fast path already emits exactly these
+ // (`f32::from_bits(0x7F00_0001 + (p - rank))`); use the same
+ // formula here so `max_score` (population max, reported even on
+ // empty beyond-end pages) and per-hit `_score` values match ES
+ // regardless of which execution path served the query. The old
+ // `max_organic + 1 + (p - rank)` fakes leaked small integers
+ // (4.0/3.0) into `max_score` where ES reports the sentinel.
+ let p = ids.len() as u32;
for hit in &mut final_hits {
- if ids.iter().any(|pid| pid == &hit.id) {
- // Give pinned docs a very high score so they sort first.
- // Use index in the ids list to preserve pinned ordering.
- let rank = ids.iter().position(|pid| pid == &hit.id).unwrap_or(0);
- hit.score = pin_score + (ids.len() - rank) as f32;
+ if let Some(rank) = ids.iter().position(|pid| pid == &hit.id) {
+ hit.score = f32::from_bits(0x7F00_0001_u32 + (p - rank as u32));
}
}
}