Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Merge origin/main into perf/concurrent-scan-clone-and-single-flight (resolve index.rs conflict: keep both query_needs_id_injection and geo_coord)

xerj-org6921e88db6fe
diff --cc engine/crates/xerj-engine/src/index.rs
index 3d2a8331d,22b1dada1..2674c13e4
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@@ -20548,97 -20637,17 +20684,108 @@@ fn postings_union_expand
(count, positions)
}
+/// Whether `_id` must be injected into a doc's `_source` before
+/// `doc_matches_query` evaluates `q` against it — required only when an
+/// `Ids` clause is nested somewhere inside (a *top-level* `Ids` is already
+/// handled by the caller without needing this at all).
+///
+/// `_source` never contains `_id` itself, so `scan_stored_section_into`
+/// previously deep-cloned EVERY scanned doc's `_source` unconditionally just
+/// to splice `_id` in — for a query with no nested `Ids` anywhere (the
+/// overwhelming majority: any `term`/`match`/`match_phrase`/`query_string`/
+/// `bool` filter that isn't specifically an ids lookup), that clone is pure
+/// waste, paid on every doc regardless of whether it even matches. Measured
+/// as the dominant cost of a concurrent dashboard's wildcard `query_string`
+/// filters (~200-300µs/doc — the simd_json parse itself costs ~3µs/doc per
+/// this file's own comment).
+///
+/// This match is intentionally exhaustive (no wildcard arm): adding a new
+/// `QueryNode` variant that can nest a sub-query MUST update this function
+/// or the compiler rejects the build — so a missed variant can only ever
+/// cost the (already-paid-before-this-fix) clone, never a silently wrong
+/// match.
+fn query_needs_id_injection(q: &QueryNode) -> bool {
+ match q {
+ QueryNode::Ids { .. } => true,
+ QueryNode::MatchAll
+ | QueryNode::MatchNone
+ | QueryNode::Term { .. }
+ | QueryNode::Terms { .. }
+ | QueryNode::Range { .. }
+ | QueryNode::Prefix { .. }
+ | QueryNode::Wildcard { .. }
+ | QueryNode::Exists { .. }
+ | QueryNode::Match { .. }
+ | QueryNode::MatchPhrase { .. }
+ | QueryNode::MultiMatch { .. }
+ | QueryNode::QueryString { .. }
+ | QueryNode::Fuzzy { .. }
+ | QueryNode::Regexp { .. }
+ | QueryNode::Intervals { .. }
+ | QueryNode::MatchPhrasePrefix { .. }
+ | QueryNode::SimpleQueryString { .. }
+ | QueryNode::GeoDistance { .. }
+ | QueryNode::GeoBoundingBox { .. }
+ | QueryNode::GeoPolygon { .. }
+ | QueryNode::GeoShape { .. }
+ | QueryNode::SpanTerm { .. }
+ | QueryNode::MoreLikeThis { .. }
+ | QueryNode::Percolate { .. } => false,
+ QueryNode::Bool {
+ must,
+ should,
+ must_not,
+ filter,
+ ..
+ } => must
+ .iter()
+ .chain(should)
+ .chain(must_not)
+ .chain(filter)
+ .any(query_needs_id_injection),
+ QueryNode::Constant { query, .. }
+ | QueryNode::Boosted { query, .. }
+ | QueryNode::Named { query, .. }
+ | QueryNode::FunctionScore { query, .. }
+ | QueryNode::Nested { query, .. } => query_needs_id_injection(query),
+ QueryNode::Boosting {
+ positive, negative, ..
+ } => query_needs_id_injection(positive) || query_needs_id_injection(negative),
+ QueryNode::DisMax { queries, .. } => queries.iter().any(query_needs_id_injection),
+ QueryNode::Pinned { organic, .. } => query_needs_id_injection(organic),
+ QueryNode::HasChild { query, .. } | QueryNode::HasParent { query, .. } => {
+ query_needs_id_injection(query)
+ }
+ QueryNode::Knn { filter, .. } | QueryNode::SemanticSearch { filter, .. } => {
+ filter.as_deref().is_some_and(query_needs_id_injection)
+ }
+ QueryNode::Hybrid { queries, .. } => {
+ queries.iter().any(|wq| query_needs_id_injection(&wq.query))
+ }
+ QueryNode::SpanNear { clauses, .. } | QueryNode::SpanOr { clauses } => {
+ clauses.iter().any(query_needs_id_injection)
+ }
+ QueryNode::SpanNot { include, exclude } => {
+ query_needs_id_injection(include) || query_needs_id_injection(exclude)
+ }
+ QueryNode::SpanFirst { match_query, .. } => query_needs_id_injection(match_query),
+ QueryNode::SpanContaining { little, big } | QueryNode::SpanWithin { little, big } => {
+ query_needs_id_injection(little) || query_needs_id_injection(big)
+ }
+ }
+}
+
+ /// Coerce a `lat`/`lon` object member to `f64`. Real ES accepts numeric OR
+ /// string-encoded lat/lon inside a `{"lat": .., "lon": ..}` geo_point object
+ /// (normalised to numbers at index time) — Kibana's own flights sample data
+ /// ships exactly this shape (`{"lat": "50.033333", ...}`), which `.as_f64()`
+ /// alone silently rejects, turning every `geo_distance`/`geo_bounding_box`
+ /// query against that field into an unconditional non-match.
+ fn geo_coord(v: &Value) -> Option<f64> {
+ v.as_f64()
+ .or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
+ }
+
/// Evaluate a query against a single stored document source value.
///
/// Returns true if the document matches the query.