AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(search): echo pit_id in PIT responses + ES-exact pinned sentinel scores
xerj-org096c743526de
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 7254f6185..d232f039e 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -9459,6 +9459,30 @@ pub async fn search(})};+ // PIT searches echo the (still-valid) `pit_id` back in the response —+ // ES 8.x places it as the FIRST key, before `took` (live-verified on+ // 8.13.4). Clients' PIT + search_after loops read the id off every+ // page and thread it into the next request, so omitting it breaks the+ // canonical pagination pattern (ES helpers fail with a missing-pit_id+ // error). xerj PIT ids are stable, so the echoed id is the request id.+ if pit_context.is_some() {+ if let Some(pit_id) = body+ .pit+ .as_ref()+ .and_then(|p| p.get("id"))+ .and_then(Value::as_str)+ {+ if let Some(obj) = response_body.as_object_mut() {+ let mut rebuilt = serde_json::Map::with_capacity(obj.len() + 1);+ rebuilt.insert("pit_id".to_string(), Value::String(pit_id.to_string()));+ for (k, v) in obj.iter() {+ rebuilt.insert(k.clone(), v.clone());+ }+ *obj = rebuilt;+ }+ }+ }+// ES 8.x emits `"terminated_early": false` (immediately after// `timed_out`) on the hits-skipping exact-count path: `size: 0` +// explicit `track_total_hits: true`, without aggregations. Live-probed@@ -14136,6 +14160,25 @@ async fn msearch_impl(if let Some(aggs) = merged_aggs {resp["aggregations"] = aggs;}+ // PIT sub-searches echo the pit_id first, exactly like `_search`+ // (ES 8.x msearch items carry it too — same search_after loop+ // contract per item).+ if pit_context.is_some() {+ if let Some(pit_id) = search_body_val+ .get("pit")+ .and_then(|p| p.get("id"))+ .and_then(Value::as_str)+ {+ if let Some(obj) = resp.as_object_mut() {+ let mut rebuilt = serde_json::Map::with_capacity(obj.len() + 1);+ rebuilt.insert("pit_id".to_string(), Value::String(pit_id.to_string()));+ for (k, v) in obj.iter() {+ rebuilt.insert(k.clone(), v.clone());+ }+ *obj = rebuilt;+ }+ }+ }responses.push(resp);}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 160363ed3..4cde04893 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -7990,17 +7990,20 @@ impl Index {if scored_fast_applied {// exact pin scores already in place} else if let QueryNode::Pinned { ids, .. } = query {- let max_organic_score = final_hits- .iter()- .map(|h| h.score)- .fold(f32::NEG_INFINITY, f32::max);- let pin_score = (max_organic_score + 1.0).max(1.0);+ // ES pin scores are sentinel constants descending one float ULP+ // per rank from just above 2^127 (1.7014124e38 for the first pin+ // with two ids — live-verified vs ES 8.13.4). The columnar+ // Pinned fast path already emits exactly these+ // (`f32::from_bits(0x7F00_0001 + (p - rank))`); use the same+ // formula here so `max_score` (population max, reported even on+ // empty beyond-end pages) and per-hit `_score` values match ES+ // regardless of which execution path served the query. The old+ // `max_organic + 1 + (p - rank)` fakes leaked small integers+ // (4.0/3.0) into `max_score` where ES reports the sentinel.+ let p = ids.len() as u32;for hit in &mut final_hits {- if ids.iter().any(|pid| pid == &hit.id) {- // Give pinned docs a very high score so they sort first.- // Use index in the ids list to preserve pinned ordering.- let rank = ids.iter().position(|pid| pid == &hit.id).unwrap_or(0);- hit.score = pin_score + (ids.len() - rank) as f32;+ if let Some(rank) = ids.iter().position(|pid| pid == &hit.id) {+ hit.score = f32::from_bits(0x7F00_0001_u32 + (p - rank as u32));}}}