AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
es_compat: skip the zero-hit unknown_value sample for size:0 probes (#2451) (#2457)
xerj-orge00e2f914f22
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 776236042..027cd99b6 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -10410,6 +10410,22 @@ fn referenced_or_recurse(v: &Value, out: &mut Vec<(String, String)>) {/// reasonable guess — and got a silent zero, because this corpus classifies/// Markdown as `txt-prose`/`txt-lines`. Discovering that cost the agent two/// round trips and a terms aggregation it had to think to write.+/// Whether a response qualifies for the zero-hit `unknown_value` sample+/// (#2451). Only a request that asked for hits (`size > 0`) and whose whole+/// match set is empty: an empty page with a non-zero tracked total is a page+/// past the end, not a wrong value, and a `size: 0` count/existence probe must+/// stay as cheap as its own query — the sample is a 256-row full-source read.+/// With `track_total_hits: false` the total is not authoritative, so the+/// empty page (with `size > 0`) is the only signal, as before.+fn zero_hit_value_sample_applies(+ size: usize,+ page_empty: bool,+ total_count: u64,+ track_total_disabled: bool,+) -> bool {+ size > 0 && page_empty && (track_total_disabled || total_count == 0)+}+fn unknown_value_hint(index: &str, field: &str, value: &str, present: &[String]) -> Value {json!({"hints": [{@@ -11069,6 +11085,69 @@ mod search_response_hint_tests {(json, text)}+ fn has_unknown_value_hint(json: &Value) -> bool {+ json["_xerj"]["hints"].as_array().is_some_and(|hints| {+ hints+ .iter()+ .any(|h| h["code"].as_str() == Some("unknown_value"))+ })+ }++ #[test]+ fn zero_hit_value_sample_gates_on_the_match_set_not_the_page() {+ // #2451: a `size: 0` probe never samples, hit or miss.+ assert!(!super::zero_hit_value_sample_applies(0, true, 0, false));+ assert!(!super::zero_hit_value_sample_applies(0, true, 7, false));+ assert!(!super::zero_hit_value_sample_applies(0, true, 0, true));+ // A page past the end of a non-empty match set is not a wrong value.+ assert!(!super::zero_hit_value_sample_applies(10, true, 7, false));+ // The case the hint exists for, tracked total or not.+ assert!(super::zero_hit_value_sample_applies(10, true, 0, false));+ assert!(super::zero_hit_value_sample_applies(10, true, 0, true));+ assert!(!super::zero_hit_value_sample_applies(10, false, 3, false));+ }++ #[tokio::test]+ async fn size_zero_term_probe_skips_the_unknown_value_sample() {+ let state = test_state();+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("kind", FieldType::Keyword));+ state.engine.create_index("hint-probe", schema).unwrap();+ let idx = state.engine.get_index("hint-probe").unwrap();+ for (id, kind) in [("1", "alpha"), ("2", "beta")] {+ idx.index_document(Some(id.into()), json!({ "kind": kind }))+ .await+ .unwrap();+ }+ idx.refresh().await.unwrap();+ let router = crate::router::build_es_compat_router(state);++ // A search that asked for hits and matched nothing keeps the hint.+ let (json, _) = run_search(+ &router,+ "hint-probe",+ json!({"size": 10, "query": {"term": {"kind": "gamma"}}}),+ )+ .await;+ assert!(has_unknown_value_hint(&json), "hint expected: {json}");++ // The autoindex existence-probe shape, miss and hit: an exact total,+ // no sample (#2451).+ for (value, total) in [("gamma", 0), ("alpha", 1)] {+ let (json, _) = run_search(+ &router,+ "hint-probe",+ json!({"size": 0, "track_total_hits": true,+ "query": {"term": {"kind": value}}}),+ )+ .await;+ assert_eq!(json["hits"]["total"]["value"], total, "{json}");+ assert!(!has_unknown_value_hint(&json), "no hint for size 0: {json}");+ }+ }+#[tokio::test]async fn hint_appears_for_a_large_unprojected_response() {let state = test_state();@@ -18987,7 +19066,22 @@ async fn search_impl(// and name the values that do exist. Only on zero hits, only for// exact clauses, and capped — this must never cost anything on a// query that worked.- if all_hints.is_empty() && hits.is_empty() {+ //+ // #2451: "zero hits" means the MATCH SET is empty, not the page. A+ // `size: 0` request (a count or existence probe — autoindex's+ // per-upsert `index_holds_content` is one) always has an empty+ // page, so gating on `hits.is_empty()` alone ran the 256-row+ // `match_all` sample — a full-source decode of ~356 rows, 1.4 s on+ // a 234k-doc code index — after EVERY probe, hit or miss. The+ // sample is for a caller who asked for hits and got none.+ if all_hints.is_empty()+ && zero_hit_value_sample_applies(+ body.size,+ hits.is_empty(),+ total_count,+ track_total_disabled,+ )+ {let mut clauses = Vec::new();if let Some(q) = body.query.as_ref() {exact_term_clauses(q, &mut clauses);