AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(knn): exact scan ranks addresses and hydrates only the winners (#939)
xerj-org61966df3cc29
diff --git a/benchmarks/hybrid-scan-939/common.py b/benchmarks/hybrid-scan-939/common.pynew file mode 100755index 000000000..02820bc74--- /dev/null+++ b/benchmarks/hybrid-scan-939/common.py@@ -0,0 +1,89 @@+"""Shared helpers for the #939 exact-scan benchmark. Standard library only."""+import hashlib+import json+import os+import urllib.error+import urllib.request++API_KEY = os.environ.get("XERJ_API_KEY", "")+++def http(url, method, path, body=None, content_type="application/json", timeout=600, raw=False):+ data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode()+ headers = {"content-type": content_type}+ if API_KEY:+ headers["authorization"] = "ApiKey " + API_KEY+ req = urllib.request.Request(url + path, data=data, method=method, headers=headers)+ try:+ payload = urllib.request.urlopen(req, timeout=timeout).read()+ except urllib.error.HTTPError as e:+ return {"_err": e.code, "body": e.read().decode()[:400]}+ return payload.decode() if raw else json.loads(payload)+++def loadavg():+ with open("/proc/loadavg") as f:+ return f.read().split()[0]+++def cache_counters(url):+ """(hits, misses) of the result cache, scraped from the node's own gauges.++ A run that claims to have measured the engine must show hits did not move.+ """+ text = http(url, "GET", "/v1/metrics", raw=True)+ if isinstance(text, dict):+ raise SystemExit(f"cannot read /v1/metrics: {text}")+ found = {}+ for line in text.splitlines():+ for name in ("xerj_query_cache_hits", "xerj_query_cache_misses"):+ if line.startswith(name + " ") or line.startswith(name + "{"):+ found[name] = found.get(name, 0) + int(float(line.rsplit(" ", 1)[1]))+ if len(found) != 2:+ raise SystemExit(f"result-cache gauges missing from /v1/metrics: {found}")+ return found["xerj_query_cache_hits"], found["xerj_query_cache_misses"]+++def distinct_queries(data_dir, n):+ """First n DISTINCT query texts, in file order."""+ seen, out = set(), []+ with open(data_dir + "/queries.jsonl") as f:+ for line in f:+ text = json.loads(line)["text"]+ if text not in seen:+ seen.add(text)+ out.append(text)+ if len(out) == n:+ break+ if len(out) < n:+ raise SystemExit(f"only {len(out)} distinct queries in {data_dir}, need {n}")+ return out+++def bucket_of(doc_id):+ """Deterministic 10-way keyword partition of the corpus, for the filter arms."""+ return "b%d" % (int(hashlib.md5(doc_id.encode()).hexdigest(), 16) % 10)+++def n_of(doc_id):+ """Deterministic integer in [0, 1000), for the range-filter arm."""+ return int(hashlib.md5(("n" + doc_id).encode()).hexdigest(), 16) % 1000+++def bm25_query(q):+ return {"multi_match": {"query": q, "fields": ["title", "text"]}}+++def semantic_query(q, k=10, flt=None):+ body = {"field": "body", "query": q, "k": k}+ if flt is not None:+ body["filter"] = flt+ return {"semantic": body}+++def hybrid_query(q, k=10):+ return {"hybrid": {"queries": [{"query": bm25_query(q)}, {"query": semantic_query(q, k)}], "fusion": "rrf"}}+++def pct(sorted_values, p):+ return sorted_values[min(len(sorted_values) - 1, int(len(sorted_values) * p))]diff --git a/benchmarks/hybrid-scan-939/compare.py b/benchmarks/hybrid-scan-939/compare.pynew file mode 100755index 000000000..e730de04b--- /dev/null+++ b/benchmarks/hybrid-scan-939/compare.py@@ -0,0 +1,39 @@+#!/usr/bin/env python3+"""Compare two identity.py dumps. Exit 1 on ANY difference.++ python3 compare.py before.json after.json++Tolerance is ZERO: same ids, same order, same `_score` (the exact JSON number),+same `_source`, same totals and relations, same passages. The scan rewrite in+#939 is meant to be bit-identical, so a tolerance would only hide a defect.+"""+import json+import sys++a, b = (json.load(open(p)) for p in sys.argv[1:3])+bad = 0+if a.keys() != b.keys():+ print("ARMS DIFFER:", sorted(a.keys() ^ b.keys()))+ bad += 1+for arm in sorted(a.keys() & b.keys()):+ rows_a, rows_b = a[arm], b[arm]+ diffs, hits = 0, 0+ if len(rows_a) != len(rows_b):+ print(f"{arm}: query count differs {len(rows_a)} vs {len(rows_b)}")+ bad += 1+ continue+ for ra, rb in zip(rows_a, rows_b):+ hits += len(ra["response"].get("hits", {}).get("hits", []))+ if ra != rb:+ diffs += 1+ if diffs <= 3:+ ha = [(h["_id"], h.get("_score")) for h in ra["response"].get("hits", {}).get("hits", [])]+ hb = [(h["_id"], h.get("_score")) for h in rb["response"].get("hits", {}).get("hits", [])]+ first = next((i for i, (x, y) in enumerate(zip(ha, hb)) if x != y), None)+ print(f" {arm}: DIFF q={ra['q'][:60]!r} first_differing_rank={first}"+ f" before={ha[first] if first is not None else None} after={hb[first] if first is not None else None}"+ f" total {ra['response'].get('hits', {}).get('total')} vs {rb['response'].get('hits', {}).get('total')}")+ print(f"{arm:32s} queries={len(rows_a)} hits_compared={hits} differing_responses={diffs}")+ bad += diffs+print("IDENTICAL" if bad == 0 else f"NOT IDENTICAL: {bad} differing responses")+sys.exit(0 if bad == 0 else 1)diff --git a/benchmarks/hybrid-scan-939/identity.py b/benchmarks/hybrid-scan-939/identity.pynew file mode 100755index 000000000..88b526319--- /dev/null+++ b/benchmarks/hybrid-scan-939/identity.py@@ -0,0 +1,63 @@+#!/usr/bin/env python3+"""Dump full responses for a fixed query set, to compare two binaries.++ XERJ_API_KEY=... python3 identity.py http://localhost:10840 scifact /path/to/beir/scifact 100 out.json++Read-only. Deeper and heavier than latency.py on purpose: `size`/`k` 100 so a+reordering far from the top still shows, and `_source` ON for the vector arms+so the hydrated documents are compared too, not only ids and scores. `took` is+dropped; everything else in the response is kept verbatim.++Run it against the OLD binary and the NEW binary over the SAME data directory+(same segments, same document order), then feed both dumps to compare.py.+"""+import json+import sys++from common import bm25_query, distinct_queries, http, hybrid_query, semantic_query++url, index, data_dir, n, out = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5]+qs = distinct_queries(data_dir, n)+K = 100+++def search(query, size, source):+ r = http(url, "POST", f"/{index}/_search", {"size": size, "_source": source, "query": query})+ if "_err" in r:+ raise SystemExit(f"search failed: {r}")+ r.pop("took", None)+ return r+++top30 = {q: [h["_id"] for h in search(bm25_query(q), 30, False)["hits"]["hits"]] for q in qs}+TERM = {"term": {"bucket": "b3"}}+TERM_AND_RANGE = {"bool": {"filter": [{"term": {"bucket": "b3"}}, {"range": {"n": {"gte": 100, "lt": 900}}}]}}+# Filters the projection fast path must DECLINE (not on its whitelist), so the+# fallback path is compared too.+MATCH_FILTER = {"match": {"title": "cell"}}++arms = {+ "semantic": lambda q: search(semantic_query(q, K), K, True),+ "semantic_ids": lambda q: search(semantic_query(q, K, {"ids": {"values": top30[q]}}), K, True),+ "semantic_term": lambda q: search(semantic_query(q, K, TERM), K, True),+ "semantic_term_range": lambda q: search(semantic_query(q, K, TERM_AND_RANGE), K, True),+ "semantic_match_filter": lambda q: search(semantic_query(q, K, MATCH_FILTER), K, True),+ "semantic_passage": lambda q: http(url, "POST", f"/{index}/_search", {+ "size": 10, "_source": False, "fields": ["_passage"], "query": semantic_query(q, 10)}),+ "hybrid": lambda q: search(hybrid_query(q, K), K, True),+ "bool_must_semantic_filter_ids": lambda q: search(+ {"bool": {"must": [semantic_query(q, K)], "filter": [{"ids": {"values": top30[q]}}]}}, K, False),+}+dump = {}+for name, fn in arms.items():+ rows = []+ for q in qs:+ r = fn(q)+ r.pop("took", None)+ rows.append({"q": q, "response": r})+ dump[name] = rows+ hits = sum(len(row["response"].get("hits", {}).get("hits", [])) for row in rows)+ print(f"{name:32s} queries={len(rows)} hits_total={hits}", flush=True)+with open(out, "w") as f:+ json.dump(dump, f, sort_keys=True)+print("wrote", out)diff --git a/benchmarks/hybrid-scan-939/latency.py b/benchmarks/hybrid-scan-939/latency.pynew file mode 100755index 000000000..6a027c062--- /dev/null+++ b/benchmarks/hybrid-scan-939/latency.py@@ -0,0 +1,75 @@+#!/usr/bin/env python3+"""Per-arm search latency, closed loop, one client.++ XERJ_API_KEY=... python3 latency.py http://localhost:10840 scifact /path/to/beir/scifact 300++Read-only (`_search`, `/v1/metrics`). Every arm sends N DISTINCT query texts,+so no request inside an arm repeats. Arms differ in request body, and the+result cache is keyed on the whole body, so arms cannot serve each other. That+is not taken on trust: the node's own result-cache hit gauge is scraped around+every arm and the delta printed. A row whose `cache_hits` is not 0 measured+the cache, not the engine.++The last arm repeats ONE request on purpose and is expected to show hits: it+is there so the trap stays visible to whoever runs this next.++Machine load (1-minute loadavg) is printed per arm. A latency taken while+other jobs saturate the box is an upper bound, not a measurement of the engine.+"""+import sys+import time++from common import (bm25_query, cache_counters, distinct_queries, http, hybrid_query, loadavg, pct,+ semantic_query)++url, index, data_dir, n = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])+qs = distinct_queries(data_dir, n)+++def search(query, size=10):+ r = http(url, "POST", f"/{index}/_search", {"size": size, "_source": False, "query": query})+ if "_err" in r:+ raise SystemExit(f"search failed: {r}")+ return r+++def timed(query):+ t = time.perf_counter()+ r = search(query)+ return time.perf_counter() - t, r+++# The ids filter names the BM25 top-30 of the same text. Those lookups are+# made up front so they are not inside any timed region.+print("preparing ids filters (untimed) ...", flush=True)+top30 = {q: [h["_id"] for h in search(bm25_query(q), 30)["hits"]["hits"]] for q in qs}++TERM = {"term": {"bucket": "b3"}}+TERM_AND_RANGE = {"bool": {"filter": [{"term": {"bucket": "b3"}}, {"range": {"n": {"gte": 100, "lt": 900}}}]}}++arms = [+ ("bm25", lambda q: timed(bm25_query(q))),+ ("semantic (unfiltered)", lambda q: timed(semantic_query(q))),+ ("semantic + ids filter (30 docs)", lambda q: timed(semantic_query(q, flt={"ids": {"values": top30[q]}}))),+ ("semantic + term filter (~10%)", lambda q: timed(semantic_query(q, flt=TERM))),+ ("semantic + term AND range filter", lambda q: timed(semantic_query(q, flt=TERM_AND_RANGE))),+ ("hybrid rrf", lambda q: timed(hybrid_query(q))),+ ("semantic, SAME request repeated", lambda q: timed(semantic_query(qs[0], k=11))),+]+print(f"index={index} docs={http(url, 'GET', f'/{index}/_count').get('count')} distinct_queries={len(qs)}")+for name, fn in arms:+ hits0, _ = cache_counters(url)+ load0 = loadavg()+ print("ARM_START", name, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()), flush=True)+ lat, took = [], []+ for q in qs:+ dt, r = fn(q)+ lat.append(dt * 1000)+ took.append(r.get("took", 0))+ print("ARM_END", name, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()), flush=True)+ hits1, _ = cache_counters(url)+ lat.sort()+ took.sort()+ print(f"{name:34s} n={len(lat)} p50={pct(lat, .5):7.1f}ms p95={pct(lat, .95):7.1f}ms"+ f" server_took p50={pct(took, .5)}ms p95={pct(took, .95)}ms"+ f" cache_hits={hits1 - hits0} loadavg={load0}->{loadavg()}", flush=True)diff --git a/benchmarks/hybrid-scan-939/load.py b/benchmarks/hybrid-scan-939/load.pynew file mode 100755index 000000000..27660838a--- /dev/null+++ b/benchmarks/hybrid-scan-939/load.py@@ -0,0 +1,62 @@+#!/usr/bin/env python3+"""Load a BEIR corpus into a throwaway XERJ node.++ XERJ_API_KEY=... python3 load.py http://localhost:10840 scifact /path/to/beir/scifact [clients]++WRITES: it deletes and recreates the index named on the command line. Point it+only at a node you started yourself, on a port you own.++`body` is `semantic_text` (title + ". " + text, as the BEIR harness this was+copied from built it), so every document longer than 512 characters is stored+as several passage vectors — the shape issue #939 is about. `bucket` and `n`+are deterministic functions of `_id` that give the filter arms something to+select on. `clients` > 1 sends `_bulk` requests concurrently (useful under+`--embed-mode neural`, see #938); document order inside the index then depends+on arrival order, so build the index ONCE and run both binaries over that same+data directory when comparing results.+"""+import json+import sys+import time+from concurrent.futures import ThreadPoolExecutor++from common import bucket_of, http, loadavg, n_of++url, index, data_dir = sys.argv[1], sys.argv[2], sys.argv[3]+clients = int(sys.argv[4]) if len(sys.argv) > 4 else 1+BULK = 100++print("delete:", http(url, "DELETE", "/" + index))+print("create:", http(url, "PUT", "/" + index, {"mappings": {"properties": {+ "title": {"type": "text"}, "text": {"type": "text"},+ "bucket": {"type": "keyword"}, "n": {"type": "integer"},+ "body": {"type": "semantic_text"}}}}))++docs = [json.loads(line) for line in open(data_dir + "/corpus.jsonl")]+++def send(start):+ lines = []+ for d in docs[start:start + BULK]:+ lines.append(json.dumps({"index": {"_index": index, "_id": d["_id"]}}))+ lines.append(json.dumps({+ "title": d["title"], "text": d["text"],+ "bucket": bucket_of(d["_id"]), "n": n_of(d["_id"]),+ "body": d["title"] + ". " + d["text"]}))+ r = http(url, "POST", "/_bulk", ("\n".join(lines) + "\n").encode(), "application/x-ndjson", timeout=3600)+ if r.get("errors") or "_err" in r:+ raise SystemExit("bulk problem: " + str(r)[:400])+ return len(lines) // 2+++t0 = time.time()+done = 0+with ThreadPoolExecutor(max_workers=clients) as pool:+ for n in pool.map(send, range(0, len(docs), BULK)):+ done += n+ if done % 1000 < BULK:+ print(f"indexed {done}/{len(docs)} {done / (time.time() - t0):.1f} docs/s loadavg={loadavg()}", flush=True)+print("refresh:", http(url, "POST", f"/{index}/_refresh"))+print("flush:", http(url, "POST", f"/{index}/_flush"))+print("count:", http(url, "GET", f"/{index}/_count"))+print(f"wall={time.time() - t0:.1f}s clients={clients} loadavg={loadavg()}")diff --git a/benchmarks/hybrid-scan-939/phases.py b/benchmarks/hybrid-scan-939/phases.pynew file mode 100755index 000000000..086682306--- /dev/null+++ b/benchmarks/hybrid-scan-939/phases.py@@ -0,0 +1,68 @@+#!/usr/bin/env python3+"""Split traced semantic requests into phases, per latency.py arm.++ python3 phases.py server.log latency.log++Start the node with XERJ_TRACE_SEMANTIC_PHASES=1 NO_COLOR=1. latency.py prints+ARM_START/ARM_END stamps (UTC, 1 s resolution); this buckets the node's+`semantic_phase=` lines by those windows. One client, closed loop, so lines of+one request are contiguous: embed_start → embed_complete → start_brute →+load_segment* → scored → complete.+"""+import re+import sys+from datetime import datetime, timezone++server_log, latency_log = sys.argv[1], sys.argv[2]+arms = []+start = {}+for line in open(latency_log):+ m = re.match(r"ARM_(START|END) (.+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)$", line.strip())+ if not m:+ continue+ t = datetime.strptime(m.group(3), "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc)+ if m.group(1) == "START":+ start[m.group(2)] = t+ else:+ arms.append((m.group(2), start[m.group(2)], t))++ts_re = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)\.\d+Z")+kv = lambda line, key: (lambda m: int(m.group(1)) if m else None)(re.search(key + r"=(\d+)", line))+requests, cur = [], None+for line in open(server_log, errors="replace"):+ if "semantic_phase=" not in line:+ continue+ m = ts_re.match(line)+ if not m:+ continue+ t = datetime.strptime(m.group(1), "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc)+ if "semantic_phase=embed_start" in line:+ cur = {"t": t}+ requests.append(cur)+ elif cur is None:+ continue+ elif "semantic_phase=embed_complete" in line:+ cur["embed"] = kv(line, "elapsed_ms")+ elif "semantic_phase=scored" in line:+ cur["collect"] = kv(line, "collect_ms")+ cur["scored_at"] = kv(line, "elapsed_ms")+ cur["scored_docs"] = kv(line, "scored")+ elif "semantic_phase=complete" in line:+ cur["total"] = kv(line, "elapsed_ms")+++def p50(xs):+ xs = sorted(x for x in xs if x is not None)+ return xs[len(xs) // 2] if xs else float("nan")+++for name, t0, t1 in arms:+ rs = [r for r in requests if t0 <= r["t"] <= t1 and "total" in r]+ if not rs:+ continue+ score = [r["scored_at"] - r["collect"] for r in rs if r.get("scored_at") is not None and r.get("collect") is not None]+ tail = [r["total"] - r["scored_at"] for r in rs if r.get("scored_at") is not None]+ print(f"{name:34s} n={len(rs):4d} embed p50={p50([r.get('embed') for r in rs]):6.1f}"+ f" collect p50={p50([r.get('collect') for r in rs]):6.1f} score p50={p50(score):6.1f}"+ f" topk+hits p50={p50(tail):6.1f} scan_total p50={p50([r['total'] for r in rs]):6.1f}"+ f" scored_docs p50={p50([r.get('scored_docs') for r in rs])}")diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 1b4c97c4d..0044a617b 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -6867,6 +6867,7 @@ dependencies = ["chrono-tz","criterion","dashmap",+ "futures-util","libc","memchr","parking_lot",diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.tomlindex df749b25b..93261f4e8 100644--- a/engine/crates/xerj-engine/Cargo.toml+++ b/engine/crates/xerj-engine/Cargo.toml@@ -19,6 +19,9 @@ xerj-wasm = { path = "../xerj-wasm" }xerj-cluster = { path = "../xerj-cluster" }tokio = { workspace = true }+# `join_all` for the hybrid legs (#939). Already in the lockfile via the+# workspace's `futures`; no new crate is pulled in.+futures-util = "0.3"serde = { workspace = true }serde_json = { workspace = true }simd-json = { workspace = true }diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 55a4ead28..446d4e819 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -6245,6 +6245,25 @@ mod semantic_deadline_regression_tests {Arc::new(reader),);}+ SegmentCacheCategory::VectorColumn => {+ let key = format!("{old_id}\u{1}embedding\u{1}c");+ let mut builder = crate::vector_column::ColumnBuilder::new(+ "embedding",+ crate::vector_column::ColumnMode::BestPassage,+ );+ builder.push_stored_doc(&serde_json::json!({+ "_id": "v1",+ "_source": {"embedding": [1.0, 0.0]}+ }));+ let _ = idx.publish_current(+ &idx.vector_column_cache,+ &old_id,+ key,+ category,+ 1,+ builder.finish(),+ );+ }}));}@@ -6288,6 +6307,10 @@ mod semantic_deadline_regression_tests {.sort_shadow_cache.iter().any(|entry| entry.key().starts_with(&prefix)));+ assert!(!idx+ .vector_column_cache+ .iter()+ .any(|entry| entry.key().starts_with(&prefix)));}#[tokio::test]@@ -8134,6 +8157,25 @@ pub struct Index {/// it is asked for; a reader opened for a narrower set must never satisfy/// a query needing a wider one.fts_reader_cache: Arc<dashmap::DashMap<String, Resident<Arc<FtsIndexReader>>>>,+ /// Flat `f32` vectors of one vector field over one segment, keyed+ /// `"<segment_id>\u{1}<field>\u{1}<mode>"` (#939).+ ///+ /// The exact kNN scan used to read every vector out of the parsed+ /// `_source` tree on every query — after deep-cloning that tree, vectors+ /// included. `stored_value_cache` saves the disk read and the parse; it+ /// cannot save a walk over ~10 M `Value::Number`s per query on a 5 k+ /// document multi-passage corpus. This holds the same numbers once, as+ /// `f32`, at ~1/8 of the bytes the parsed tree spends on them.+ ///+ /// Same lifecycle as every other cache here: segments are immutable so an+ /// entry never goes stale, it is evicted by segment-id prefix when a merge+ /// retires the segment, and it is charged to the hydration budget — over+ /// budget the scan builds the column per query and drops it, which is+ /// still cheaper than the clones it replaced. Liveness is NOT in the+ /// column: deletes and updates are resolved per query against the version+ /// map, exactly as before.+ vector_column_cache:+ Arc<dashmap::DashMap<String, Resident<crate::vector_column::SegmentVectorColumn>>>,/// Per-(segment, query-shape) match-count cache for the/// `try_shortcut_count` Bool intersection arm. The fused columnar/// walk is O(anchor-predicate matches) per segment PER QUERY — for a@@ -8299,7 +8341,8 @@ impl Index {!collect_dense_vector_fields(&schema.schema).is_empty()}- /// Aggregate DashMap table capacities for the seven hydration families.+ /// Aggregate DashMap table capacities for the hydration families, in+ /// `SegmentCacheCategory` order./// These are diagnostic table slots, separate from the refundable/// retained-payload/key budget.pub fn segment_hydration_cache_capacities(&self) -> [usize; CATEGORY_COUNT] {@@ -8312,6 +8355,7 @@ impl Index {self.row_seq_cache.capacity(),self.decoded_stored_cache.capacity(),self.fts_reader_cache.capacity(),+ self.vector_column_cache.capacity(),]}@@ -8623,6 +8667,7 @@ impl Index {stored_slices_cache: Arc::new(per_index_map()),decoded_stored_cache: Arc::new(per_index_map()),fts_reader_cache: Arc::new(per_index_map()),+ vector_column_cache: Arc::new(per_index_map()),shortcut_count_cache: Arc::new(per_index_map()),ghost_positions_cache: Arc::new(per_index_map()),regexp_expand_cache: Arc::new(per_index_map()),@@ -9019,6 +9064,7 @@ impl Index {stored_slices_cache: Arc::new(per_index_map()),decoded_stored_cache: Arc::new(per_index_map()),fts_reader_cache: Arc::new(per_index_map()),+ vector_column_cache: Arc::new(per_index_map()),shortcut_count_cache: Arc::new(per_index_map()),ghost_positions_cache: Arc::new(per_index_map()),regexp_expand_cache: Arc::new(per_index_map()),@@ -13770,6 +13816,8 @@ impl Index {boost: Option<f32>,min_similarity: Option<f32>,) -> Result<SearchResult> {+ use crate::vector_column::{ColumnBuilder, ColumnMode, DocVectorsView};+let started = std::time::Instant::now();let mut timed_out = false;let trace_phases = std::env::var_os("XERJ_TRACE_SEMANTIC_PHASES").is_some();@@ -13787,18 +13835,49 @@ impl Index {// this is the whole filtered-kNN path. Owned clone; the lock is released.let dmq_schema = self.schema().await;- // ── Collect all candidate (doc_id, source) pairs ──────────────- let mut candidates: Vec<(String, Value)> = Vec::new();- // Memtable first (newest writes).- {- let mem = &*self.memtable;- candidates.extend(mem.all_docs_with_sources());- }+ // ── Determine whether this field opts into SQ8 (scalar8) ──────+ // Default fields keep the exact f32 brute-force scan below,+ // byte-identical to before. A `scalar8` field takes the branch above+ // instead: it reads the same live f32 vector out of `_source`, but+ // rounds it through 1-byte-per-dimension SQ8 codes before scoring, so+ // the score carries `scalar8`'s precision loss. Nothing is cached —+ // neither the codes nor the codebook outlive the query (#371).+ let use_sq8 = {+ let schema = self.schema.read().await;+ lookup_vector_quantization(&schema.schema, field).as_deref() == Some("scalar8")+ };+ // `scalar8` reads only the pooled vector; the default scan prefers the+ // per-passage companion. The two shapes are cached under different keys.+ let column_mode = if use_sq8 {+ ColumnMode::PooledOnly+ } else {+ ColumnMode::BestPassage+ };++ // ── Collect candidate ADDRESSES, not documents (#939) ─────────+ // A candidate is where a live document sits — a memtable slot or a+ // (segment, position) pair — never a copy of it. The scan this+ // replaces deep-cloned every live `_source`, vectors included, into+ // this list before it had scored or filtered anything: ~10 M+ // `Value::Number`s per query on a 5 k-document multi-passage corpus,+ // and half of each request. Sources are now cloned for the documents+ // that are RETURNED, after ranking. Same shape as qdrant's+ // `ScoredPointOffset` → `process_search_result` and tantivy's+ // `DocAddress` → `Searcher::doc` (pointers in `vector_column.rs`).+ //+ // Candidate ORDER is unchanged — memtable first, then segments in+ // snapshot order, then position — because the rank below is a stable+ // sort and that order is what breaks score ties.+ // Memtable first (newest writes). `Arc` clones, not tree clones.+ let mem_docs: Vec<(String, Arc<Value>)> = self.memtable.all_docs_with_sources_arc();+ let mut candidates: Vec<KnnCandidateAt> =+ (0..mem_docs.len()).map(KnnCandidateAt::Memtable).collect();// Then every flushed segment's stored section.let snap = self.store.snapshot();// Track seen IDs so later-segment copies don't duplicate memtable entries.- let mut seen: HashSet<String> = candidates.iter().map(|(id, _)| id.clone()).collect();- 'segments: for meta in snap.segments.iter() {+ let mut seen: HashSet<String> = mem_docs.iter().map(|(id, _)| id.clone()).collect();+ let mut segment_views: Vec<KnnSegmentView> = Vec::with_capacity(snap.segments.len());+ for meta in snap.segments.iter() {let segment_started = std::time::Instant::now();// Cache-backed: first KNN against this segment pays the// I/O + decompress + serde_json parse, every subsequent@@ -13829,6 +13908,22 @@ impl Index {timed_out = true;break;}+ // The segment's vectors as flat `f32`s. Cached per immutable+ // segment; on a miss it is derived in the SAME pass that collects+ // candidates, so a cold query walks the stored documents once and+ // keeps the cooperative checkpoints it always had. Every position+ // is pushed — live or not — because liveness is this query's+ // business and the column outlives it.+ let column_key = format!("{}\u{1}{field}\u{1}{}", meta.id, column_mode.key_tag());+ let cached_column = self+ .vector_column_cache+ .get(&column_key)+ .map(|entry| Arc::clone(entry.value()));+ let mut builder = cached_column+ .is_none()+ .then(|| ColumnBuilder::new(field, column_mode));+ let view = segment_views.len();+ let mut segment_complete = true;// Cache-backed iteration: stored_values_for() handles the// I/O + decode + parse; subsequent KNN over the same segment// is an Arc clone. The underlying parse uses serde_json (not@@ -13838,13 +13933,16 @@ impl Index {for (position, doc) in docs_arc.iter().enumerate() {if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {timed_out = true;- break 'segments;+ segment_complete = false;+ break;}- let id = match doc.get("_id").and_then(Value::as_str) {- Some(s) => s.to_string(),- None => continue,+ if let Some(builder) = builder.as_mut() {+ builder.push_stored_doc(doc);+ }+ let Some(id) = doc.get("_id").and_then(Value::as_str) else {+ continue;};- if let Some(ver) = self.store.version_map.get(&id) {+ if let Some(ver) = self.store.version_map.get(id) {// Skip tombstoned (deleted) docs.if ver.deleted {continue;@@ -13864,46 +13962,65 @@ impl Index {}}}- if !seen.insert(id.clone()) {+ if seen.contains(id) {continue;}- // Reassembled segment docs have shape- // { "_id":..., "_seq_no":..., "_source": {...} }- // but `get_field_value(src, "embedding")` further down- // looks for the field at the top level (memtable path- // shape). Unwrap to `_source` so either layout matches;- // legacy pre-M7 segments without `_source` fall through- // to the wrapper.- let src = doc.get("_source").cloned().unwrap_or_else(|| {- let mut d = doc.clone();- if let Some(obj) = d.as_object_mut() {- obj.remove("_id");- }- d- });- candidates.push((id, src));- }+ seen.insert(id.to_string());+ candidates.push(KnnCandidateAt::Segment { view, position });+ }+ let column = match (cached_column, builder) {+ (Some(column), _) => column,+ // A column abandoned at the deadline covers a prefix of the+ // segment. It serves this request's partial answer and is+ // dropped — only a complete column is ever published.+ (None, Some(builder)) if !segment_complete => {+ CacheResident::uncached(builder.finish())+ }+ (None, Some(builder)) => {+ let column = builder.finish();+ let bytes = column+ .retained_bytes()+ .saturating_add(column_key.len() as u64);+ // Over budget this comes back uncached: the next query+ // derives it again, which is what every query did before.+ self.publish_current(+ &self.vector_column_cache,+ &meta.id,+ column_key,+ SegmentCacheCategory::VectorColumn,+ bytes,+ column,+ )+ }+ (None, None) => unreachable!("a builder exists whenever the column is not cached"),+ };+ segment_views.push(KnnSegmentView {+ docs: docs_arc,+ column,+ });if trace_phases {tracing::info!(index=%self.name, segment=%meta.id, elapsed_ms=segment_started.elapsed().as_millis() as u64, candidates=candidates.len(), "semantic_phase=load_segment");}+ if !segment_complete {+ break;+ }}let collect_elapsed = started.elapsed();- // ── Determine whether this field opts into SQ8 (scalar8) ──────- // Default fields keep the exact f32 brute-force scan below,- // byte-identical to before. A `scalar8` field takes the branch above- // instead: it reads the same live f32 vector out of `_source`, but- // rounds it through 1-byte-per-dimension SQ8 codes before scoring, so- // the score carries `scalar8`'s precision loss. Nothing is cached —- // neither the codes nor the codebook outlive the query (#371).- let use_sq8 = {- let schema = self.schema.read().await;- lookup_vector_quantization(&schema.schema, field).as_deref() == Some("scalar8")- };+ // The filter runs BEFORE scoring and without the vectors (#939): see+ // `KnnFilterPlan`. It used to clone the whole source a second time,+ // per candidate, to insert `_id` beside ~1,900 numbers it never read.+ let filter_plan = filter.as_deref().map(KnnFilterPlan::for_filter);+ // Memtable documents have no column (the memtable is mutable), so+ // their vectors are read out of the `Arc`'d source one document at a+ // time into this reused scratch — after the filter, never before.+ let mut memtable_scratch = ColumnBuilder::new(field, column_mode);// ── Score each candidate against the query vector ─────────────- let mut scored: Vec<(String, f32, Value, Option<u32>)> =- Vec::with_capacity(candidates.len());+ // `(candidate index, score, passage ordinal)`: 16 bytes a candidate,+ // where this used to carry every candidate's cloned source through the+ // sort and then drop all but `k` of them.+ let mut scored: Vec<(usize, f32, Option<u32>)> = Vec::with_capacity(candidates.len());if use_sq8 {// Cosine fields are L2-normalised before quantising so SQ8 fits// over bounded [-1,1] per-dim ranges (much better recall); cosine@@ -13912,40 +14029,41 @@ impl Index {let normalize = !matches!(similarity, "l2_norm" | "dot_product" | "max_inner_product");let dim = query_vec.len();- // Post-filter candidate vectors for this field: (id, src, doc_vec).- let mut cand: Vec<(String, Value, Vec<f32>)> = Vec::with_capacity(candidates.len());- for (position, (id, src)) in candidates.into_iter().enumerate() {+ // Post-filter candidate vectors for this field: (candidate, doc_vec).+ let mut cand: Vec<(usize, Vec<f32>)> = Vec::with_capacity(candidates.len());+ for (position, at) in candidates.iter().enumerate() {if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {timed_out = true;break;}- if let Some(ref f) = filter {- let mut src_with_id = src.clone();- if let Some(obj) = src_with_id.as_object_mut() {- obj.insert("_id".to_string(), Value::String(id.clone()));- }- if !doc_matches_query_typed(f, &src_with_id, &dmq_schema) {+ if let (Some(plan), Some(f)) = (filter_plan.as_ref(), filter.as_deref()) {+ let (id, source) = at.id_and_source(&mem_docs, &segment_views);+ if !plan.matches(f, id, source, &dmq_schema) {continue;}}- let vec_val = match get_field_value(&src, field) {- Some(v) => v,- None => continue,+ let vectors = match *at {+ KnnCandidateAt::Memtable(slot) => {+ memtable_scratch.clear();+ memtable_scratch.push_source(&mem_docs[slot].1);+ memtable_scratch.column().doc(0)+ }+ KnnCandidateAt::Segment {+ view,+ position: doc_position,+ } => segment_views[view].column.doc(doc_position),};- let mut doc_vec: Vec<f32> = match &vec_val {- Value::Array(arr) => arr- .iter()- .filter_map(|v| v.as_f64().map(|f| f as f32))- .collect(),- _ => continue,+ let DocVectorsView::Pooled(vector) = vectors else {+ continue;};- if doc_vec.len() != dim {+ if vector.values.len() != dim {continue;}+ let mut doc_vec = vector.values.to_vec();if normalize {l2_normalize_vec(&mut doc_vec);}- cand.push((id, src, doc_vec));+ cand.push((position, doc_vec));}// Fit this field's SQ8 codebook over the candidate vectors this@@ -13994,7 +14112,7 @@ impl Index {// INDEX time, so a Lucene score is a function of the index state// alone; here it is a function of the query's candidate set too.// #392 is what would close that gap.- let params = Sq8Params::fit_borrowed(cand.iter().map(|(_, _, v)| v.as_slice()), dim);+ let params = Sq8Params::fit_borrowed(cand.iter().map(|(_, v)| v.as_slice()), dim);debug!(field,dim,@@ -14011,7 +14129,7 @@ impl Index {// document. `v.len() == dim` was established when `cand` was built.let mut codes = vec![0u8; dim];let mut decoded = vec![0.0f32; dim];- for (position, (id, src, v)) in cand.into_iter().enumerate() {+ for (position, (candidate, v)) in cand.into_iter().enumerate() {if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {timed_out = true;break;@@ -14019,7 +14137,7 @@ impl Index {params.encode_into(&v, &mut codes);params.decode_into(&codes, &mut decoded);let score = compute_vector_similarity(similarity, query_vec, &decoded);- scored.push((id, score, src, None));+ scored.push((candidate, score, None));}} else {// Per-chunk (passage) companion: multi-chunk `semantic_text` docs@@ -14030,74 +14148,72 @@ impl Index {// that section, not on the whole-doc average. Plain `dense_vector`// kNN and short single-chunk docs have no such companion, so they// fall through to the exact single-vector scan below (unchanged).- let chunk_field = format!("{field}_chunks");- for (position, (id, src)) in candidates.into_iter().enumerate() {+ let scorer = QueryScorer::new(similarity, query_vec);+ for (position, at) in candidates.iter().enumerate() {if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {timed_out = true;break;}// Apply filter: if the filter doesn't match this doc, skip.- if let Some(ref f) = filter {- let mut src_with_id = src.clone();- if let Some(obj) = src_with_id.as_object_mut() {- obj.insert("_id".to_string(), Value::String(id.clone()));- }- if !doc_matches_query_typed(f, &src_with_id, &dmq_schema) {+ if let (Some(plan), Some(f)) = (filter_plan.as_ref(), filter.as_deref()) {+ let (id, source) = at.id_and_source(&mem_docs, &segment_views);+ if !plan.matches(f, id, source, &dmq_schema) {continue;}}- let (score, passage_ordinal) =- if let Some(Value::Array(chunks)) = get_field_value(&src, &chunk_field) {+ let vectors = match *at {+ KnnCandidateAt::Memtable(slot) => {+ memtable_scratch.clear();+ memtable_scratch.push_source(&mem_docs[slot].1);+ memtable_scratch.column().doc(0)+ }+ KnnCandidateAt::Segment {+ view,+ position: doc_position,+ } => segment_views[view].column.doc(doc_position),+ };+ let (score, passage_ordinal) = match vectors {+ DocVectorsView::Absent => continue,+ DocVectorsView::Chunks(passages) => {// Best-matching passage over the stored chunk vectors.let mut best: Option<(f32, u32)> = None;- for (chunk_position, cv) in chunks.iter().enumerate() {- if chunk_position & 31 == 0- && self.exact_scan_checkpoint(chunk_position, deadline).await+ for passage in passages {+ // Bounds ONE pathological document. The per-128-+ // documents checkpoint above is the scan's cadence;+ // this used to fire at passage 0 of every document+ // as well, which was one `yield_now` per document+ // per query for no bound the outer check lacks.+ if passage.ordinal > 0+ && passage.ordinal & 31 == 0+ && self+ .exact_scan_checkpoint(passage.ordinal as usize, deadline)+ .await{timed_out = true;break;}- let dv: Vec<f32> = match cv {- Value::Array(a) => a- .iter()- .filter_map(|v| v.as_f64().map(|f| f as f32))- .collect(),- _ => continue,- };- if dv.len() != query_vec.len() {+ if passage.values.len() != query_vec.len() {continue;}- let s = compute_vector_similarity(similarity, query_vec, &dv);- let Some(ordinal) = u32::try_from(chunk_position).ok() else {- continue;- };- choose_passage_winner(&mut best, s, ordinal);+ choose_passage_winner(+ &mut best,+ scorer.score(passage),+ passage.ordinal,+ );}match best {Some((score, ordinal)) => (score, Some(ordinal)),None => continue,}- } else {- let vec_val = match get_field_value(&src, field) {- Some(v) => v,- None => continue,- };- let doc_vec: Vec<f32> = match &vec_val {- Value::Array(arr) => arr- .iter()- .filter_map(|v| v.as_f64().map(|f| f as f32))- .collect(),- _ => continue,- };- if doc_vec.len() != query_vec.len() {+ }+ DocVectorsView::Pooled(vector) => {+ if vector.values.len() != query_vec.len() {continue;}- (- compute_vector_similarity(similarity, query_vec, &doc_vec),- Some(0),- )- };- scored.push((id, score, src, passage_ordinal));+ (scorer.score(vector), Some(0))+ }+ };+ scored.push((position, score, passage_ordinal));}}if trace_phases {@@ -14113,32 +14229,45 @@ impl Index {// total (live-verified 2026-07-12).if let Some(raw) = min_similarity {let cut = raw_similarity_to_score(similarity, raw);- scored.retain(|(_, score, _, _)| *score >= cut);+ scored.retain(|(_, score, _)| *score >= cut);}// ES applies `boost` to the final `_score` AFTER the similarity// cutoff (a boosted sub-threshold doc stays excluded; a boosted// passing doc scores `transform(sim) * boost`).if let Some(b) = boost {if (b - 1.0).abs() > f32::EPSILON {- for (_, score, _, _) in scored.iter_mut() {+ for (_, score, _) in scored.iter_mut() {*score *= b;}}}- // ── Rank, cap the candidate pool at k, then paginate ──────────- // (shared with the HNSW path so hits format / total semantics- // cannot drift between the exact and approximate executors)+ // ── Rank, cap the pool at k, THEN hydrate (#939) ──────────────+ // Same comparator, same stability and same `k.max(1)` cap as+ // `knn_result_from_scored`, applied to 16-byte tuples instead of to+ // every candidate's source. Only the survivors are cloned out of the+ // shared stored values, so a query pays for `k` sources, not for the+ // corpus.+ rank_knn_pool(&mut scored, k, |entry| entry.1);+ let ranked: Vec<(String, f32, Value, Option<u32>)> = scored+ .into_iter()+ .map(|(candidate, score, ordinal)| {+ let (id, source) = candidates[candidate].hydrate(&mem_docs, &segment_views);+ (id, score, source, ordinal)+ })+ .collect();++ // (result shaping is shared with the HNSW path so hits format / total+ // semantics cannot drift between the exact and approximate executors)let generated_companion_fields = {let schema = self.schema.read().await;generated_embedding_companion_fields(&schema.schema)};- let mut result = knn_result_from_scored(+ let mut result = knn_result_from_ranked(self,request,field,- scored,- k,+ ranked,started,&generated_companion_fields,);@@ -15027,11 +15156,22 @@ impl Index {// for a deeper page.let per_query_topk = request.size.saturating_add(request.from).max(50);- // Run each sub-query sequentially. (Parallel `join_all` is the- // obvious next step but means cloning Self into each future;- // sequential is correct, easy to reason about, and good enough- // for v0.7-P1 — the per-query latency is dominated by the kNN- // / FTS scan, not the await ordering.)+ // Run the legs CONCURRENTLY, not in parallel (#939). A search runs+ // inside `block_in_place(block_on(..))`, so these futures share one+ // thread and only overlap where a leg awaits. The await worth+ // overlapping is the query embedding of a `semantic` leg under an+ // active backend: a BERT forward pass on the blocking pool (~14 ms) or+ // a proxy round trip, during which this thread used to sit idle and+ // the lexical leg had not started. So legs that embed are POLLED+ // FIRST — they park on the embedder, the lexical legs run underneath+ // them — while results are still consumed in the caller's order, which+ // is what fusion weights and the timeout rule below are defined over.+ // Under the default lexical embedder the embedding is synchronous and+ // there is nothing to overlap; this costs nothing there.+ //+ // Spawning each leg as its own task would buy real parallelism, but+ // needs an `Arc<Self>` this `&self` chain does not have, next to the+ // `block_in_place` hand-off that produced #751. Not for ~10 ms.let mut sub_results: Vec<(Vec<Hit>, f32)> = Vec::with_capacity(sub_queries.len());let mut sub_savings: Vec<PayloadSavings> = Vec::new();let mut any_timed_out = false;@@ -15041,12 +15181,9 @@ impl Index {// the true match set, so the reported relation must be Gte, not a false// Eq. Stays false only when every sub-list was fully retrieved.let mut fused_count_capped = false;- for wq in sub_queries {- if std::time::Instant::now() >= deadline {- any_timed_out = true;- break;- }- let sub_request = SearchRequest {+ let sub_requests: Vec<SearchRequest> = sub_queries+ .iter()+ .map(|wq| SearchRequest {query: wq.query.clone(),from: 0,size: per_query_topk,@@ -15066,10 +15203,37 @@ impl Index {profile: false,leaf_ts_field: None,savings: request.savings,- };+ })+ .collect();+ let mut leg_outcomes: Vec<Option<Result<SearchResult>>> =+ sub_requests.iter().map(|_| None).collect();+ if std::time::Instant::now() >= deadline {+ any_timed_out = true;+ } else {+ let mut poll_order: Vec<usize> = (0..sub_requests.len()).collect();+ // Stable: legs that embed keep their relative order, then the rest.+ poll_order.sort_by_key(|&leg| peel_semantic_query(&sub_requests[leg].query).is_none());// Box::pin to break the type-recursion (search_inner ↔// run_hybrid both async fn).- let sub_result = Box::pin(self.search_inner(&sub_request, deadline)).await?;+ let legs = poll_order+ .iter()+ .map(|&leg| Box::pin(self.search_inner(&sub_requests[leg], deadline)));+ for (&leg, outcome) in poll_order+ .iter()+ .zip(futures_util::future::join_all(legs).await)+ {+ leg_outcomes[leg] = Some(outcome);+ }+ }+ // Consume in the CALLER'S order with the sequential loop's exact+ // rules: the first failing leg in that order is the error returned,+ // and a leg that timed out ends the list — the legs after it are+ // dropped, as they were when they had simply never been started.+ for (wq, outcome) in sub_queries.into_iter().zip(leg_outcomes) {+ let Some(outcome) = outcome else {+ break;+ };+ let sub_result = outcome?;any_timed_out |= sub_result.timed_out;// #569/#594: this sub-list under-counts the fused total only when the// leg genuinely has MORE than `per_query_topk` matches — signalled by@@ -22160,6 +22324,7 @@ impl Index {self.stored_slices_cache.clear();self.decoded_stored_cache.clear();self.fts_reader_cache.clear();+ self.vector_column_cache.clear();self.shortcut_count_cache.clear();self.ghost_positions_cache.clear();self.regexp_expand_cache.clear();@@ -22181,6 +22346,7 @@ impl Index {+ self.stored_slices_cache.len()+ self.decoded_stored_cache.len()+ self.fts_reader_cache.len()+ + self.vector_column_cache.len()+ self.shortcut_count_cache.len()+ self.ghost_positions_cache.len()+ self.regexp_expand_cache.len()@@ -24815,6 +24981,9 @@ impl Index {let fts_prefix = format!("{segment_id}\u{1}");self.fts_reader_cache.retain(|key, _| !key.starts_with(&fts_prefix));+ // Keyed by "<segment_id>\u{1}<field>\u{1}<mode>" — same prefix.+ self.vector_column_cache+ .retain(|key, _| !key.starts_with(&fts_prefix));let shadow_prefix = format!("{segment_id}\u{1}");self.sort_shadow_cache.retain(|key, _| !key.starts_with(&shadow_prefix));@@ -34823,11 +34992,7 @@ fn raw_similarity_to_score(sim: &str, raw: f32) -> f32 {fn compute_vector_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {debug_assert_eq!(a.len(), b.len());- let dot: f64 = a- .iter()- .zip(b.iter())- .map(|(x, y)| (*x as f64) * (*y as f64))- .sum();+ let dot = crate::vector_column::dot_f64(a, b);let result: f64 = match sim {"l2_norm" => {let sq: f64 = a@@ -34848,25 +35013,74 @@ fn compute_vector_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {dot + 1.0}}- _ => {- let na: f64 = a- .iter()- .map(|x| (*x as f64) * (*x as f64))- .sum::<f64>()- .sqrt();- let nb: f64 = b- .iter()- .map(|x| (*x as f64) * (*x as f64))- .sum::<f64>()- .sqrt();- let denom = na * nb;- let cos = if denom > 0.0 { dot / denom } else { 0.0 };- (1.0 + cos) / 2.0- }+ _ => cosine_score(+ dot,+ crate::vector_column::norm_f64(a),+ crate::vector_column::norm_f64(b),+ ),};result as f32}+/// The cosine arm of [`compute_vector_similarity`], given the dot product and+/// both norms. Split out so a scan can supply norms it already has.+#[inline]+fn cosine_score(dot: f64, na: f64, nb: f64) -> f64 {+ let denom = na * nb;+ let cos = if denom > 0.0 { dot / denom } else { 0.0 };+ (1.0 + cos) / 2.0+}++/// One query vector scored against many stored vectors (#939).+///+/// Bit-identical to calling [`compute_vector_similarity`] per vector — pinned+/// by `query_scorer_is_bit_identical_to_compute_vector_similarity`. For cosine+/// that function recomputes BOTH norms on every call; the query's is the same+/// number every time and the document's is a property of the stored vector,+/// so here the first is computed once per query and the second once per+/// segment. Every value still comes out of the same expression+/// (`vector_column::norm_f64`, `vector_column::dot_f64`, [`cosine_score`]),+/// which is what makes the result exact rather than close. The other metrics+/// have no norm to hoist and go through `compute_vector_similarity` unchanged.+struct QueryScorer<'a> {+ similarity: &'a str,+ query: &'a [f32],+ query_norm: f64,+ cosine: bool,+}++impl<'a> QueryScorer<'a> {+ fn new(similarity: &'a str, query: &'a [f32]) -> Self {+ // Mirrors the `_` arm of `compute_vector_similarity`: anything that is+ // not one of the three named metrics is cosine.+ let cosine = !matches!(similarity, "l2_norm" | "dot_product" | "max_inner_product");+ Self {+ similarity,+ query,+ query_norm: if cosine {+ crate::vector_column::norm_f64(query)+ } else {+ 0.0+ },+ cosine,+ }+ }++ /// `vector.values.len()` must equal the query's; callers skip the rest.+ #[inline]+ fn score(&self, vector: crate::vector_column::StoredVector<'_>) -> f32 {+ if self.cosine {+ cosine_score(+ crate::vector_column::dot_f64(self.query, vector.values),+ self.query_norm,+ vector.norm,+ ) as f32+ } else {+ compute_vector_similarity(self.similarity, self.query, vector.values)+ }+ }+}+/// Minimum live doc count before the HNSW query path may serve a kNN/// request. Below this the exact brute-force scan is already fast and — more/// importantly — every ES-YAML vector test (≤ a few hundred docs) keeps@@ -35489,13 +35703,49 @@ fn knn_result_from_scored(started: std::time::Instant,generated_companion_fields: &HashSet<String>,) -> SearchResult {- scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));+ rank_knn_pool(&mut scored, k, |entry| entry.1);+ knn_result_from_ranked(+ index,+ request,+ vector_field,+ scored,+ started,+ generated_companion_fields,+ )+}++/// Rank a kNN candidate pool by score, descending, and cap it at `k` — the one+/// definition of "top-k" every kNN executor shares.+///+/// The sort is STABLE and ties compare `Equal`, so documents with the same+/// score keep the order they were collected in (memtable, then segments in+/// snapshot order, then stored position). Generic over the entry so the exact+/// scan can rank 16-byte `(candidate, score, ordinal)` tuples and clone a+/// source only for what survives (#939); same comparator, so same order.+fn rank_knn_pool<T>(pool: &mut Vec<T>, k: usize, score: impl Fn(&T) -> f32) {+ pool.sort_by(|a, b| {+ score(b)+ .partial_cmp(&score(a))+ .unwrap_or(std::cmp::Ordering::Equal)+ });// `k` bounds the kNN candidate pool; `from`/`size` then window into// it, exactly like an ES top-level knn (returned hits are the// size-slice of the top-k, NOT k itself). Without this, a// `{"knn"/"semantic": {k: 5}}` with `"size": 3` returned 5 hits.// `size == 0` is a count-only request (total only, no hits).- scored.truncate(k.max(1));+ pool.truncate(k.max(1));+}++/// Shape an already ranked, already `k`-capped pool (see [`rank_knn_pool`])+/// into the response.+fn knn_result_from_ranked(+ index: &Index,+ request: &SearchRequest,+ vector_field: &str,+ scored: Vec<(String, f32, Value, Option<u32>)>,+ started: std::time::Instant,+ generated_companion_fields: &HashSet<String>,+) -> SearchResult {// ES reports `hits.total.value` for a knn/semantic query as the size// of the retrieved neighbor pool (min(k, matches)), NOT the number of// docs that merely have a vector — so compute it AFTER the truncate.@@ -35581,6 +35831,206 @@ fn knn_result_from_scored(}}+/// Where a live kNN candidate sits — never a copy of the document (#939).+#[derive(Clone, Copy, Debug)]+enum KnnCandidateAt {+ /// Slot in this query's memtable snapshot.+ Memtable(usize),+ /// `position` within the stored documents of `segment_views[view]`.+ Segment { view: usize, position: usize },+}++/// One segment as the exact scan sees it: the shared parsed documents and the+/// flat vectors derived from them, aligned by position.+struct KnnSegmentView {+ docs: Resident<Vec<Value>>,+ column: Resident<crate::vector_column::SegmentVectorColumn>,+}++impl KnnCandidateAt {+ /// The candidate's id and source, BORROWED — what the filter reads.+ fn id_and_source<'a>(+ &self,+ mem_docs: &'a [(String, Arc<Value>)],+ segment_views: &'a [KnnSegmentView],+ ) -> (&'a str, &'a Value) {+ match *self {+ Self::Memtable(slot) => {+ let (id, source) = &mem_docs[slot];+ (id.as_str(), source.as_ref())+ }+ Self::Segment { view, position } => {+ let doc = &segment_views[view].docs[position];+ // A segment candidate is only ever created for a document+ // with a string `_id`.+ let id = doc.get("_id").and_then(Value::as_str).unwrap_or_default();+ (id, crate::vector_column::stored_source_view(doc))+ }+ }+ }++ /// The candidate's id and an OWNED source — paid for returned hits only.+ fn hydrate(+ &self,+ mem_docs: &[(String, Arc<Value>)],+ segment_views: &[KnnSegmentView],+ ) -> (String, Value) {+ match *self {+ Self::Memtable(slot) => {+ let (id, source) = &mem_docs[slot];+ (id.clone(), (**source).clone())+ }+ Self::Segment { view, position } => {+ let doc = &segment_views[view].docs[position];+ let id = doc+ .get("_id")+ .and_then(Value::as_str)+ .unwrap_or_default()+ .to_string();+ // Reassembled segment docs have shape+ // { "_id":..., "_seq_no":..., "_source": {...} }+ // and a hit carries the `_source`; legacy pre-M7 segments+ // without `_source` fall through to the wrapper minus `_id`.+ let source = doc.get("_source").cloned().unwrap_or_else(|| {+ let mut d = doc.clone();+ if let Some(obj) = d.as_object_mut() {+ obj.remove("_id");+ }+ d+ });+ (id, source)+ }+ }+ }+}++/// How the exact kNN scan shows one document to its `filter` (#939).+///+/// `doc_matches_query_typed` reads `_id` out of the document it is handed, so+/// the scan used to clone each candidate's WHOLE source — every passage vector+/// included, ~1,900 JSON numbers a document on a multi-passage corpus — only+/// to insert `_id` into the copy, and did so for every candidate whether or+/// not the filter could ever match it. An `ids` filter naming 30 documents+/// cost as much as no filter at all.+enum KnnFilterPlan {+ /// Every node of the filter reads the document ONLY through+ /// `get_field_value(source, <a field it names>)` or `source.get("_id")`.+ /// `get_field_value` looks at the source root under dotted prefixes of the+ /// field name and nowhere else (its literal-key fast path, the+ /// longest-dotted-key walk, and the multi-field parent fallback all do),+ /// so the matcher cannot tell the source from a copy holding just those+ /// root keys. It gets that copy: a few scalars instead of the vectors.+ Projected(Vec<String>),+ /// Anything else sees the whole source plus `_id`, exactly as before.+ FullSource,+}++impl KnnFilterPlan {+ fn for_filter(filter: &QueryNode) -> Self {+ let mut root_keys = Vec::new();+ if collect_filter_root_keys(filter, &mut root_keys) {+ Self::Projected(root_keys)+ } else {+ Self::FullSource+ }+ }++ fn matches(&self, filter: &QueryNode, id: &str, source: &Value, schema: &Schema) -> bool {+ // A non-object source never had `_id` inserted — there is nowhere to+ // put it — so the matcher always saw it as it is.+ let Value::Object(fields) = source else {+ return doc_matches_query_typed(filter, source, schema);+ };+ let mut view = match self {+ Self::FullSource => fields.clone(),+ Self::Projected(root_keys) => {+ let mut view = serde_json::Map::new();+ for key in root_keys {+ if let Some(value) = fields.get(key) {+ view.insert(key.clone(), value.clone());+ }+ }+ view+ }+ };+ view.insert("_id".to_string(), Value::String(id.to_string()));+ doc_matches_query_typed(filter, &Value::Object(view), schema)+ }+}++/// Collect the source-root keys `filter` can read into `out`; `false` when+/// some node is not on the whitelist below and the filter must see the whole+/// source.+///+/// WHITELIST DISCIPLINE: a node belongs here only if its arm in+/// `doc_matches_query_typed` touches the document through+/// `get_field_value(source, field)` / `source.get("_id")` and nothing else.+/// Arms that iterate the source (`match` on a wildcard field, `multi_match`,+/// `query_string`), hand it to a script (`script`, a scripted+/// `minimum_should_match`), or walk nested objects are deliberately absent:+/// declining costs a clone, admitting one wrongly changes which documents+/// match. `knn_filter_projection_agrees_with_the_full_source` pins the+/// equivalence for everything admitted.+fn collect_filter_root_keys(filter: &QueryNode, out: &mut Vec<String>) -> bool {+ match filter {+ QueryNode::MatchAll | QueryNode::MatchNone | QueryNode::Ids { .. } => true,+ QueryNode::Term { field, .. }+ | QueryNode::Terms { field, .. }+ | QueryNode::Range { field, .. }+ | QueryNode::Prefix { field, .. }+ | QueryNode::Exists { field } => push_field_root_keys(field, out),+ QueryNode::Constant { query, .. }+ | QueryNode::Boosted { query, .. }+ | QueryNode::Named { query, .. } => collect_filter_root_keys(query, out),+ QueryNode::Bool {+ must,+ should,+ must_not,+ filter,+ minimum_should_match,+ ..+ } => {+ match minimum_should_match {+ // Runs Painless over the whole document.+ Some(MinShouldMatch::Script { .. }) => return false,+ // `terms_set`: the required count is read from this field.+ Some(MinShouldMatch::Field(name)) => {+ if !push_field_root_keys(name, out) {+ return false;+ }+ }+ _ => {}+ }+ must.iter()+ .chain(should)+ .chain(must_not)+ .chain(filter)+ .all(|clause| collect_filter_root_keys(clause, out))+ }+ _ => false,+ }+}++/// Every dotted prefix of `field`, itself included — the only root keys+/// `get_field_value(source, field)` can look up. `false` for a pattern.+fn push_field_root_keys(field: &str, out: &mut Vec<String>) -> bool {+ if field.contains('*') {+ return false;+ }+ let mut push = |key: &str| {+ if !out.iter().any(|existing| existing == key) {+ out.push(key.to_string());+ }+ };+ for (at, ch) in field.char_indices() {+ if ch == '.' {+ push(&field[..at]);+ }+ }+ push(field);+ true+}+/// Extract `field` from `source` as an all-numbers f32 vector./// Returns `None` when the field is absent, not an array, empty, or holds/// any non-number element.@@ -52715,3 +53165,829 @@ mod pinned_knn_fts_892_tests {);}}++/// #939 — the exact kNN scan ranks addresses and hydrates only the winners.+///+/// The rewrite is meant to be invisible: same documents, same order, same+/// scores to the bit. These tests pin that against `legacy_scan`, a transcript+/// of the scan as it stood before the change (clone every live source, clone+/// it again for the filter, re-walk the JSON numbers per passage), so the+/// equivalence is checked in CI and not only by a one-off benchmark.+#[cfg(test)]+mod exact_scan_hydration_tests {+ use super::*;+ use crate::Engine;+ use tempfile::TempDir;++ const DIM: usize = 8;++ fn engine(dir: &TempDir) -> Engine {+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ Engine::new(config).expect("engine")+ }++ /// `compute_vector_similarity` as it stood before #939, verbatim. The+ /// oracle must not share code with what it checks.+ fn legacy_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {+ let dot: f64 = a+ .iter()+ .zip(b.iter())+ .map(|(x, y)| (*x as f64) * (*y as f64))+ .sum();+ let result: f64 = match sim {+ "l2_norm" => {+ let sq: f64 = a+ .iter()+ .zip(b.iter())+ .map(|(x, y)| {+ let d = (*x as f64) - (*y as f64);+ d * d+ })+ .sum();+ 1.0 / (1.0 + sq)+ }+ "dot_product" => (1.0 + dot) / 2.0,+ "max_inner_product" => {+ if dot < 0.0 {+ 1.0 / (1.0 - dot)+ } else {+ dot + 1.0+ }+ }+ _ => {+ let na: f64 = a+ .iter()+ .map(|x| (*x as f64) * (*x as f64))+ .sum::<f64>()+ .sqrt();+ let nb: f64 = b+ .iter()+ .map(|x| (*x as f64) * (*x as f64))+ .sum::<f64>()+ .sqrt();+ let denom = na * nb;+ let cos = if denom > 0.0 { dot / denom } else { 0.0 };+ (1.0 + cos) / 2.0+ }+ };+ result as f32+ }++ fn json_vector(value: &Value) -> Option<Vec<f32>> {+ match value {+ Value::Array(a) => Some(+ a.iter()+ .filter_map(|v| v.as_f64().map(|f| f as f32))+ .collect(),+ ),+ _ => None,+ }+ }++ /// The exact scan as it stood before #939. Returns the ranked, `k`-capped+ /// pool as `(id, score bits, passage ordinal, source)`.+ #[allow(clippy::too_many_arguments)]+ async fn legacy_scan(+ idx: &Index,+ field: &str,+ query_vec: &[f32],+ k: usize,+ filter: Option<&QueryNode>,+ similarity: &str,+ boost: Option<f32>,+ min_similarity: Option<f32>,+ ) -> Vec<(String, u32, Option<u32>, Value)> {+ let schema = idx.schema().await;+ let mut candidates: Vec<(String, Value)> = idx.memtable.all_docs_with_sources();+ let snap = idx.store.snapshot();+ let mut seen: HashSet<String> = candidates.iter().map(|(id, _)| id.clone()).collect();+ for meta in snap.segments.iter() {+ let Some(docs) = idx.stored_values_for(&meta.id) else {+ continue;+ };+ for doc in docs.iter() {+ let Some(id) = doc.get("_id").and_then(Value::as_str).map(str::to_string) else {+ continue;+ };+ if let Some(ver) = idx.store.version_map.get(&id) {+ if ver.deleted {+ continue;+ }+ if let Some(doc_seq) = doc.get("_seq_no").and_then(Value::as_u64) {+ if doc_seq < ver.seq_no {+ continue;+ }+ }+ }+ if !seen.insert(id.clone()) {+ continue;+ }+ let src = doc.get("_source").cloned().unwrap_or_else(|| {+ let mut d = doc.clone();+ if let Some(obj) = d.as_object_mut() {+ obj.remove("_id");+ }+ d+ });+ candidates.push((id, src));+ }+ }+ let passes = |id: &str, src: &Value| -> bool {+ let Some(f) = filter else {+ return true;+ };+ let mut src_with_id = src.clone();+ if let Some(obj) = src_with_id.as_object_mut() {+ obj.insert("_id".to_string(), Value::String(id.to_string()));+ }+ doc_matches_query_typed(f, &src_with_id, &schema)+ };+ let use_sq8 = lookup_vector_quantization(&schema, field).as_deref() == Some("scalar8");+ let mut scored: Vec<(String, f32, Value, Option<u32>)> = Vec::new();+ if use_sq8 {+ let normalize = !matches!(similarity, "l2_norm" | "dot_product" | "max_inner_product");+ let dim = query_vec.len();+ let mut cand: Vec<(String, Value, Vec<f32>)> = Vec::new();+ for (id, src) in candidates {+ if !passes(&id, &src) {+ continue;+ }+ let Some(mut doc_vec) = get_field_value(&src, field).as_ref().and_then(json_vector)+ else {+ continue;+ };+ if doc_vec.len() != dim {+ continue;+ }+ if normalize {+ l2_normalize_vec(&mut doc_vec);+ }+ cand.push((id, src, doc_vec));+ }+ let params = Sq8Params::fit_borrowed(cand.iter().map(|(_, _, v)| v.as_slice()), dim);+ let mut codes = vec![0u8; dim];+ let mut decoded = vec![0.0f32; dim];+ for (id, src, v) in cand {+ params.encode_into(&v, &mut codes);+ params.decode_into(&codes, &mut decoded);+ scored.push((+ id,+ legacy_similarity(similarity, query_vec, &decoded),+ src,+ None,+ ));+ }+ } else {+ let chunk_field = format!("{field}_chunks");+ for (id, src) in candidates {+ if !passes(&id, &src) {+ continue;+ }+ let (score, ordinal) = if let Some(Value::Array(chunks)) =+ get_field_value(&src, &chunk_field)+ {+ let mut best: Option<(f32, u32)> = None;+ for (chunk_position, cv) in chunks.iter().enumerate() {+ let Some(dv) = json_vector(cv) else {+ continue;+ };+ if dv.len() != query_vec.len() {+ continue;+ }+ let s = legacy_similarity(similarity, query_vec, &dv);+ if best.is_none_or(|(current, _)| s > current) {+ best = Some((s, chunk_position as u32));+ }+ }+ match best {+ Some((score, ordinal)) => (score, Some(ordinal)),+ None => continue,+ }+ } else {+ let Some(doc_vec) = get_field_value(&src, field).as_ref().and_then(json_vector)+ else {+ continue;+ };+ if doc_vec.len() != query_vec.len() {+ continue;+ }+ (legacy_similarity(similarity, query_vec, &doc_vec), Some(0))+ };+ scored.push((id, score, src, ordinal));+ }+ }+ if let Some(raw) = min_similarity {+ let cut = raw_similarity_to_score(similarity, raw);+ scored.retain(|(_, score, _, _)| *score >= cut);+ }+ if let Some(b) = boost {+ if (b - 1.0).abs() > f32::EPSILON {+ for (_, score, _, _) in scored.iter_mut() {+ *score *= b;+ }+ }+ }+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));+ scored.truncate(k.max(1));+ scored+ .into_iter()+ .map(|(id, score, src, ordinal)| (id, score.to_bits(), ordinal, src))+ .collect()+ }++ /// Deterministic pseudo-random unit-ish vectors; no `rand` dependency.+ struct Lcg(u64);+ impl Lcg {+ fn next_f32(&mut self) -> f32 {+ self.0 = self+ .0+ .wrapping_mul(6364136223846793005)+ .wrapping_add(1442695040888963407);+ (((self.0 >> 33) as f64) / ((1u64 << 31) as f64) - 1.0) as f32+ }+ fn vector(&mut self) -> Vec<f32> {+ (0..DIM).map(|_| self.next_f32()).collect()+ }+ }++ fn as_json(vector: &[f32]) -> Value {+ Value::Array(vector.iter().map(|v| Value::from(*v as f64)).collect())+ }++ fn vector_schema(quantization: Option<&str>) -> Schema {+ let mut schema = Schema::empty();+ let mut body = FieldConfig::new("body", FieldType::Text);+ body.options.dimensions = Some(DIM);+ body.options.similarity = Some("cosine".into());+ body.embedding = Some(xerj_common::types::EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some("body_vector".into()),+ });+ schema.add_field(body).unwrap();+ let mut companion = FieldConfig::new("body_vector", FieldType::Vector);+ companion.options.dimensions = Some(DIM);+ companion.options.similarity = Some("cosine".into());+ companion.options.quantization = quantization.map(str::to_string);+ schema.add_field(companion).unwrap();+ schema+ .add_field(FieldConfig::new("tag", FieldType::Keyword))+ .unwrap();+ schema+ .add_field(FieldConfig::new("n", FieldType::Long))+ .unwrap();+ schema+ .add_field(FieldConfig::new("title", FieldType::Text))+ .unwrap();+ schema+ }++ /// A document. Every third is multi-passage; a run of ids shares one+ /// vector so the rank has exact score ties to break.+ fn document(id: usize, rng: &mut Lcg, shared: &[f32]) -> Value {+ let pooled = if (10..16).contains(&id) {+ shared.to_vec()+ } else {+ rng.vector()+ };+ let mut doc = serde_json::json!({+ "title": format!("doc {id} {}", if id % 4 == 0 { "cell" } else { "other" }),+ "tag": format!("t{}", id % 5),+ "n": (id * 37) % 1000,+ "body": format!("body text {id}"),+ "body_vector": as_json(&pooled),+ });+ if id % 3 == 0 {+ let passages: Vec<Value> = (0..2 + id % 4).map(|_| as_json(&rng.vector())).collect();+ doc["body_vector_chunks"] = Value::Array(passages);+ }+ doc+ }++ /// Two segments (the second superseding and deleting part of the first)+ /// plus a live memtable, with every awkward document shape the scan has a+ /// rule for.+ async fn seed(idx: &Arc<Index>) {+ let mut rng = Lcg(0x939);+ let shared = rng.vector();+ for id in 0..150 {+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))+ .await+ .unwrap();+ }+ // Shapes with a rule of their own.+ idx.index_document_prepared(+ Some("malformed-passages".into()),+ serde_json::json!({+ "title": "cell malformed", "tag": "t1", "n": 5, "body": "x",+ "body_vector": as_json(&rng.vector()),+ // ordinal 1 is not a vector, ordinal 3 has the wrong length:+ // both skipped, neither renumbers ordinal 2.+ "body_vector_chunks": [as_json(&rng.vector()), "junk", as_json(&rng.vector()), [1.0, 2.0]],+ }),+ )+ .await+ .unwrap();+ idx.index_document_prepared(+ Some("empty-passages".into()),+ serde_json::json!({+ "title": "cell empty", "tag": "t1", "n": 6, "body": "x",+ "body_vector": as_json(&shared),+ "body_vector_chunks": [],+ }),+ )+ .await+ .unwrap();+ idx.index_document_prepared(+ Some("no-vector".into()),+ serde_json::json!({"title": "cell none", "tag": "t1", "n": 7, "body": "x"}),+ )+ .await+ .unwrap();+ idx.flush().await.unwrap();++ // Second segment: supersede some of the first, delete some, add more.+ for id in (0..150).step_by(9) {+ idx.index_document_prepared(+ Some(format!("d{id}")),+ document(id + 1000, &mut rng, &shared),+ )+ .await+ .unwrap();+ }+ for id in (5..150).step_by(17) {+ idx.delete_document(&format!("d{id}")).await.unwrap();+ }+ for id in 150..260 {+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))+ .await+ .unwrap();+ }+ idx.flush().await.unwrap();++ // Live memtable over both: updates of flushed documents and new ones.+ for id in (3..260).step_by(31) {+ idx.index_document_prepared(+ Some(format!("d{id}")),+ document(id + 2000, &mut rng, &shared),+ )+ .await+ .unwrap();+ }+ for id in 260..290 {+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))+ .await+ .unwrap();+ }+ assert!(+ idx.store.snapshot().segments.len() >= 2,+ "the fixture needs flushed segments beside the memtable"+ );+ }++ fn filters() -> Vec<(&'static str, Option<QueryNode>)> {+ let term = QueryNode::Term {+ field: "tag".into(),+ value: Value::from("t3"),+ boost: None,+ };+ let range = QueryNode::Range {+ field: "n".into(),+ gte: Some(Value::from(100)),+ gt: None,+ lte: None,+ lt: Some(Value::from(800)),+ boost: None,+ };+ vec![+ ("none", None),+ (+ "ids",+ Some(QueryNode::Ids {+ values: (0..290).step_by(7).map(|id| format!("d{id}")).collect(),+ }),+ ),+ ("term", Some(term.clone())),+ (+ "bool(term AND range, NOT exists)",+ Some(QueryNode::Bool {+ must: Vec::new(),+ should: Vec::new(),+ filter: vec![term.clone(), range.clone()],+ must_not: vec![QueryNode::Exists {+ field: "body_vector_chunks".into(),+ }],+ minimum_should_match: None,+ }),+ ),+ (+ // Reads the VECTOR fields themselves: the projection must+ // carry them rather than assume a filter never looks.+ "exists(vector)",+ Some(QueryNode::Exists {+ field: "body_vector".into(),+ }),+ ),+ (+ // Not on the projection whitelist: takes the full-source path.+ "match (full source)",+ Some(QueryNode::Match {+ field: "title".into(),+ query: "cell".into(),+ operator: xerj_query::ast::BoolOperator::Or,+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ }),+ ),+ ]+ }++ async fn assert_scan_matches_legacy(idx: &Arc<Index>, field: &str, sq8: bool) {+ let mut rng = Lcg(0xBEEF);+ let mut compared = 0usize;+ let mut saw_later_passage = false;+ let similarities: &[&str] = if sq8 {+ &["cosine", "l2_norm"]+ } else {+ &["cosine", "dot_product", "l2_norm", "max_inner_product"]+ };+ for _ in 0..4 {+ let query = rng.vector();+ for similarity in similarities {+ for (filter_name, filter) in filters() {+ for (k, boost, min_similarity) in [+ (1usize, None, None),+ (10, Some(2.5f32), None),+ (25, None, Some(0.1f32)),+ (1000, None, None),+ ] {+ let expected = legacy_scan(+ idx,+ field,+ &query,+ k,+ filter.as_ref(),+ similarity,+ boost,+ min_similarity,+ )+ .await;+ let request = SearchRequest {+ size: k,+ fields: vec![PASSAGE_RESPONSE_FIELD.into()],+ ..SearchRequest::default()+ };+ let got = idx+ .run_knn_brute_force(+ &request,+ field,+ &query,+ k,+ filter.clone().map(Box::new),+ similarity,+ boost,+ min_similarity,+ )+ .await+ .unwrap();+ let context = format!("similarity={similarity} filter={filter_name} k={k}");+ assert_eq!(got.total.value, expected.len() as u64, "total: {context}");+ assert_eq!(+ got.hits+ .iter()+ .map(|h| (h.id.clone(), h.score.to_bits()))+ .collect::<Vec<_>>(),+ expected+ .iter()+ .map(|(id, bits, _, _)| (id.clone(), *bits))+ .collect::<Vec<_>>(),+ "ids, order and score BITS: {context}"+ );+ for (hit, (_, _, ordinal, source)) in got.hits.iter().zip(&expected) {+ // Hydrated from the right document: the user+ // fields are the legacy scan's, value for value.+ for key in ["title", "tag", "n", "body"] {+ assert_eq!(+ hit.source.get(key),+ source.get(key),+ "{key}: {context}"+ );+ }+ if let Some(ordinal) = ordinal {+ saw_later_passage |= *ordinal > 0;+ }+ }+ compared += got.hits.len();+ }+ }+ }+ }+ assert!(+ compared > 1000,+ "the comparison must have teeth: {compared}"+ );+ if !sq8 {+ assert!(+ saw_later_passage,+ "no query was won by a passage past the first"+ );+ }+ }++ #[tokio::test]+ async fn exact_scan_is_bit_identical_to_the_clone_everything_scan() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("scan-939", vector_schema(None))+ .unwrap();+ let idx = engine.get_index("scan-939").unwrap();+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;+ seed(&idx).await;+ // Cold (columns derived in the scan) and warm (columns cached) must+ // agree with the oracle and therefore with each other.+ assert_scan_matches_legacy(&idx, "body_vector", false).await;+ assert!(+ !idx.vector_column_cache.is_empty(),+ "columns were never published"+ );+ assert_scan_matches_legacy(&idx, "body_vector", false).await;+ }++ #[tokio::test]+ async fn scalar8_scan_is_bit_identical_to_the_clone_everything_scan() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("scan-939-sq8", vector_schema(Some("scalar8")))+ .unwrap();+ let idx = engine.get_index("scan-939-sq8").unwrap();+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;+ seed(&idx).await;+ assert_scan_matches_legacy(&idx, "body_vector", true).await;+ assert_scan_matches_legacy(&idx, "body_vector", true).await;+ }++ #[tokio::test]+ async fn vector_columns_are_per_segment_budgeted_and_retired_by_merge() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("scan-939-cache", vector_schema(None))+ .unwrap();+ let idx = engine.get_index("scan-939-cache").unwrap();+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;+ seed(&idx).await;+ let query = Lcg(7).vector();+ let request = SearchRequest {+ size: 10,+ ..SearchRequest::default()+ };+ let scan = || {+ idx.run_knn_brute_force(+ &request,+ "body_vector",+ &query,+ 10,+ None,+ "cosine",+ None,+ None,+ )+ };++ let before = scan().await.unwrap();+ let segments: Vec<String> = idx+ .store+ .snapshot()+ .segments+ .iter()+ .map(|m| m.id.clone())+ .collect();+ // One column per segment that HAS stored documents. A segment holding+ // only tombstones has no stored section; the scan skips it, so there+ // is nothing to derive a column from.+ let mut with_stored = 0usize;+ for id in &segments {+ let has_stored = idx.stored_values_for(id).is_some();+ with_stored += usize::from(has_stored);+ assert_eq!(+ idx.vector_column_cache+ .contains_key(&format!("{id}\u{1}body_vector\u{1}c")),+ has_stored,+ "segment {id}: a column exists exactly when stored documents do"+ );+ }+ assert!(with_stored >= 2, "the fixture needs several real segments");+ assert_eq!(idx.vector_column_cache.len(), with_stored);+ let charged = idx.segment_hydration_budget.snapshot().category_current+ [SegmentCacheCategory::VectorColumn as usize];+ assert!(+ charged > 0,+ "a published column must be charged to the hydration budget"+ );++ // A merge retires the segments; their columns must go with them, and+ // the answer must not change.+ idx.flush().await.unwrap();+ idx.force_merge(1).await.unwrap();+ let live: HashSet<String> = idx+ .store+ .snapshot()+ .segments+ .iter()+ .map(|m| m.id.clone())+ .collect();+ for entry in idx.vector_column_cache.iter() {+ let segment = entry.key().split('\u{1}').next().unwrap();+ assert!(+ live.contains(segment),+ "column outlived its segment: {}",+ entry.key()+ );+ }+ let after = scan().await.unwrap();+ let rows = |r: &SearchResult| {+ r.hits+ .iter()+ .map(|h| (h.id.clone(), h.score.to_bits()))+ .collect::<Vec<_>>()+ };+ assert_eq!(+ rows(&before),+ rows(&after),+ "a merge changed an exact answer"+ );++ idx.release_memory();+ assert!(idx.vector_column_cache.is_empty());+ assert_eq!(rows(&before), rows(&scan().await.unwrap()));+ }++ #[test]+ fn query_scorer_is_bit_identical_to_compute_vector_similarity() {+ let mut rng = Lcg(42);+ let zero = vec![0.0f32; DIM];+ for similarity in [+ "cosine",+ "dot_product",+ "l2_norm",+ "max_inner_product",+ "anything-else",+ ] {+ for round in 0..500 {+ let query = if round == 0 {+ zero.clone()+ } else {+ rng.vector()+ };+ let stored = if round == 1 {+ zero.clone()+ } else {+ rng.vector()+ };+ let scorer = QueryScorer::new(similarity, &query);+ let got = scorer.score(crate::vector_column::StoredVector {+ values: &stored,+ ordinal: 0,+ norm: crate::vector_column::norm_f64(&stored),+ });+ let legacy = legacy_similarity(similarity, &query, &stored);+ assert_eq!(+ got.to_bits(),+ legacy.to_bits(),+ "{similarity} round {round}"+ );+ assert_eq!(+ compute_vector_similarity(similarity, &query, &stored).to_bits(),+ legacy.to_bits(),+ "refactored compute_vector_similarity drifted: {similarity} round {round}"+ );+ }+ }+ }++ #[test]+ fn knn_filter_projection_agrees_with_the_full_source() {+ let schema = vector_schema(None);+ let docs = [+ serde_json::json!({"tag": "t3", "n": 150, "a": {"b": {"c": 1}}, "a.b": {"c": 2}, "body_vector": [1.0], "_id": "shadowed"}),+ serde_json::json!({"tag": ["t1", "t3"], "n": 900, "a.b.c": 3, "_routing": "r"}),+ serde_json::json!({"tag": null, "n": "150", "a": [{"b": {"c": 4}}, {"b": {"c": 5}}]}),+ serde_json::json!({"title": "cell", "count": 2}),+ serde_json::json!({}),+ Value::Null,+ ];+ let term = |field: &str, value: Value| QueryNode::Term {+ field: field.into(),+ value,+ boost: None,+ };+ let exists = |field: &str| QueryNode::Exists {+ field: field.into(),+ };+ let projected = vec![+ QueryNode::MatchAll,+ QueryNode::MatchNone,+ QueryNode::Ids {+ values: vec!["id-0".into(), "id-2".into()],+ },+ term("tag", Value::from("t3")),+ term("_id", Value::from("id-1")),+ term("a.b.c", Value::from(1)),+ term("a.b.c", Value::from(3)),+ term("a.b.c", Value::from(4)),+ term("tag.keyword", Value::from("t3")),+ exists("a.b"),+ exists("a.b.c"),+ exists("_routing"),+ exists("_id"),+ exists("body_vector"),+ exists("missing"),+ QueryNode::Terms {+ field: "tag".into(),+ values: vec![Value::from("t1"), Value::from("zz")],+ boost: None,+ },+ QueryNode::Range {+ field: "n".into(),+ gte: Some(Value::from(100)),+ gt: None,+ lte: None,+ lt: Some(Value::from(800)),+ boost: None,+ },+ QueryNode::Prefix {+ field: "tag".into(),+ value: "t".into(),+ boost: None,+ constant_score: false,+ },+ QueryNode::Bool {+ must: vec![exists("tag")],+ should: vec![term("tag", Value::from("t3")), term("n", Value::from(900))],+ filter: Vec::new(),+ must_not: vec![term("_id", Value::from("id-0"))],+ minimum_should_match: Some(MinShouldMatch::Field("count".into())),+ },+ QueryNode::Constant {+ score: 1.0,+ query: Box::new(term("tag", Value::from("t3"))),+ },+ ];+ for filter in &projected {+ let plan = KnnFilterPlan::for_filter(filter);+ assert!(+ matches!(plan, KnnFilterPlan::Projected(_)),+ "should project: {filter:?}"+ );+ for (n, doc) in docs.iter().enumerate() {+ let id = format!("id-{n}");+ assert_eq!(+ plan.matches(filter, &id, doc, &schema),+ KnnFilterPlan::FullSource.matches(filter, &id, doc, &schema),+ "projection changed the answer: {filter:?} on {doc}"+ );+ }+ }+ // Declined: these read the document some other way.+ let declined = vec![+ term("ta*", Value::from("t3")),+ QueryNode::Script {+ source: "true".into(),+ params: None,+ },+ QueryNode::Bool {+ must: Vec::new(),+ should: vec![term("tag", Value::from("t3"))],+ filter: Vec::new(),+ must_not: Vec::new(),+ minimum_should_match: Some(MinShouldMatch::Script {+ source: "1".into(),+ params: None,+ }),+ },+ ];+ for filter in &declined {+ assert!(+ matches!(KnnFilterPlan::for_filter(filter), KnnFilterPlan::FullSource),+ "must not project: {filter:?}"+ );+ }+ }++ /// The full-source plan IS the old behaviour: the whole source with `_id`+ /// inserted over whatever the document carried under that key.+ #[test]+ fn full_source_plan_inserts_the_id_like_the_old_scan_did() {+ let schema = vector_schema(None);+ let filter = QueryNode::Ids {+ values: vec!["real".into()],+ };+ let doc = serde_json::json!({"_id": "shadowed", "tag": "t"});+ assert!(KnnFilterPlan::FullSource.matches(&filter, "real", &doc, &schema));+ assert!(!KnnFilterPlan::FullSource.matches(&filter, "other", &doc, &schema));+ }+}diff --git a/engine/crates/xerj-engine/src/ingest_memory.rs b/engine/crates/xerj-engine/src/ingest_memory.rsindex ea58c6374..64ba4e522 100644--- a/engine/crates/xerj-engine/src/ingest_memory.rs+++ b/engine/crates/xerj-engine/src/ingest_memory.rs@@ -121,10 +121,11 @@ pub enum Category {CacheRowSequences,CacheDecodedStored,CacheFtsReader,+ CacheVectorColumn,}impl Category {- pub const ALL: [Self; 22] = [+ pub const ALL: [Self; 23] = [Self::HttpBody,Self::HttpRewriteBuffer,Self::RawSource,@@ -147,6 +148,7 @@ impl Category {Self::CacheRowSequences,Self::CacheDecodedStored,Self::CacheFtsReader,+ Self::CacheVectorColumn,];const COUNT: usize = Self::ALL.len();diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rsindex 20e7e3ca6..3a9c42afd 100644--- a/engine/crates/xerj-engine/src/lib.rs+++ b/engine/crates/xerj-engine/src/lib.rs@@ -38,6 +38,7 @@ pub mod segment_cache_estimates;pub mod slow_query;pub mod sql;pub mod turbo_ingest;+mod vector_column;pub mod wal_tap;mod write_publication;diff --git a/engine/crates/xerj-engine/src/segment_cache_budget.rs b/engine/crates/xerj-engine/src/segment_cache_budget.rsindex 4d938872c..b2e06e719 100644--- a/engine/crates/xerj-engine/src/segment_cache_budget.rs+++ b/engine/crates/xerj-engine/src/segment_cache_budget.rs@@ -10,7 +10,7 @@ use std::sync::Arc;use crate::ingest_memory::{Category, Retained};-pub const CATEGORY_COUNT: usize = 8;+pub const CATEGORY_COUNT: usize = 9;#[derive(Clone, Copy, Debug, Eq, PartialEq)]#[repr(usize)]@@ -24,6 +24,9 @@ pub enum SegmentCacheCategory {DecodedStored,/// Decompressed `.post` + `.meta` blobs behind a cached `FtsIndexReader`.FtsReader,+ /// Flat `f32` vectors of one field over one segment, derived from its+ /// stored documents for the exact kNN scan (#939).+ VectorColumn,}impl SegmentCacheCategory {@@ -36,6 +39,7 @@ impl SegmentCacheCategory {Self::RowSequences,Self::DecodedStored,Self::FtsReader,+ Self::VectorColumn,];fn trace_category(self) -> Category {@@ -48,6 +52,7 @@ impl SegmentCacheCategory {Self::RowSequences => Category::CacheRowSequences,Self::DecodedStored => Category::CacheDecodedStored,Self::FtsReader => Category::CacheFtsReader,+ Self::VectorColumn => Category::CacheVectorColumn,}}}diff --git a/engine/crates/xerj-engine/src/vector_column.rs b/engine/crates/xerj-engine/src/vector_column.rsnew file mode 100644index 000000000..b99d3cdd9--- /dev/null+++ b/engine/crates/xerj-engine/src/vector_column.rs@@ -0,0 +1,446 @@+//! Flat `f32` view of one vector field over one immutable segment (#939).+//!+//! The exact kNN scan used to deep-clone every stored `_source` — vectors+//! included, as `serde_json::Value::Number`s — into its candidate list, clone+//! it a second time to evaluate the filter, clone the passage-vector array a+//! third time through `get_field_value`, and then re-walk those JSON numbers+//! into a fresh `Vec<f32>` per passage. All of that ran per document, per+//! query, and none of it was arithmetic: on 5,183 multi-passage documents it+//! was ~400 ms around ~14 ms of model time.+//!+//! This module holds the vectors once, as plain `f32`s, per segment. Segments+//! are immutable, so the column is derived exactly once and then shared; which+//! documents are *live* is still decided per query, by the caller.+//!+//! # Reference designs (approach adapted, no code copied)+//!+//! * **Flattened multi-vector + `dim`.** qdrant stores a multi-vector as one+//! contiguous buffer and slices it per sub-vector+//! (`lib/segment/src/data_types/vectors.rs:271`, max-sim at+//! `lib/segment/src/vector_storage/query_scorer/mod.rs:77`; Apache-2.0,+//! 74f3e85). Ours differs where XERJ's contract differs: one query vector,+//! best passage wins, lowest ordinal on an exact tie, and passages may be+//! skipped without renumbering the ones after them.+//! * **Score an address, hydrate the winners.** qdrant ranks+//! `ScoredPointOffset { idx, score }` and only then retrieves payloads+//! (`lib/segment/src/segment/read_view/search.rs:222`); tantivy ranks+//! `DocAddress { segment_ord, doc_id }` and fetches stored documents by+//! address (`src/lib.rs:338`, `src/core/searcher.rs:88`; MIT, 3a55bc9).+//!+//! # Exactness contract+//!+//! Scores produced from this column are **bit-identical** to the scan it+//! replaces. That is a property of construction, not of tolerance:+//!+//! * elements are converted with the same expression the scan used+//! (`as_f64()` then `as f32`, non-numbers dropped);+//! * a vector whose converted length differs from the query's is skipped at+//! query time, exactly as before, and keeps its original passage ordinal;+//! * a document whose `<field>_chunks` is an array with no usable passage is+//! skipped — it does **not** fall back to the pooled vector, because the old+//! scan did not;+//! * the dot product and the norms are the same strictly sequential `f64`+//! reductions ([`dot_f64`], [`norm_f64`]), shared with+//! `compute_vector_similarity` so the two cannot drift. Precomputing a+//! document vector's norm stores the value the scan would have recomputed.++use serde_json::Value;+use std::borrow::Cow;++use crate::index::get_field_value;++/// Which stored vectors a column holds for each document.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+pub(crate) enum ColumnMode {+ /// `<field>_chunks` passage vectors when that companion is an array, the+ /// pooled `<field>` vector otherwise. The default exact scan.+ BestPassage,+ /// Always the pooled `<field>` vector. The `scalar8` scan reads only that.+ PooledOnly,+}++impl ColumnMode {+ /// Distinguishes the two shapes in a cache key.+ pub(crate) fn key_tag(self) -> char {+ match self {+ Self::BestPassage => 'c',+ Self::PooledOnly => 'p',+ }+ }+}++/// One stored vector: where it lives in the flat buffer, which passage it was,+/// and its Euclidean norm.+#[derive(Clone, Copy, Debug)]+struct VectorEntry {+ start: usize,+ len: usize,+ ordinal: u32,+ norm: f64,+}++/// The vectors one document contributes.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum DocVectors {+ /// Nothing usable: the scan skips the document.+ Absent,+ /// The pooled vector, reported as passage ordinal 0.+ Pooled(usize),+ /// `count` passage vectors starting at entry `first`. `count == 0` is a+ /// passage array with no usable element, which also skips the document.+ Chunks { first: usize, count: usize },+}++/// A borrowed stored vector handed to the scorer.+#[derive(Clone, Copy, Debug)]+pub(crate) struct StoredVector<'a> {+ pub(crate) values: &'a [f32],+ pub(crate) ordinal: u32,+ pub(crate) norm: f64,+}++/// What a document offers the scorer.+pub(crate) enum DocVectorsView<'a> {+ Absent,+ Pooled(StoredVector<'a>),+ Chunks(ChunkIter<'a>),+}++pub(crate) struct ChunkIter<'a> {+ column: &'a SegmentVectorColumn,+ next: usize,+ end: usize,+}++impl<'a> Iterator for ChunkIter<'a> {+ type Item = StoredVector<'a>;++ fn next(&mut self) -> Option<Self::Item> {+ if self.next >= self.end {+ return None;+ }+ let vector = self.column.stored(self.next);+ self.next += 1;+ Some(vector)+ }+}++/// Flat vectors for one field over a run of documents, addressed by position.+#[derive(Debug, Default)]+pub(crate) struct SegmentVectorColumn {+ data: Vec<f32>,+ entries: Vec<VectorEntry>,+ docs: Vec<DocVectors>,+}++impl SegmentVectorColumn {+ /// Number of document positions covered. A column abandoned part-way (the+ /// request's deadline passed) covers a prefix of its segment.+ #[cfg(test)]+ pub(crate) fn len(&self) -> usize {+ self.docs.len()+ }++ fn stored(&self, entry: usize) -> StoredVector<'_> {+ let e = &self.entries[entry];+ StoredVector {+ values: &self.data[e.start..e.start + e.len],+ ordinal: e.ordinal,+ norm: e.norm,+ }+ }++ /// The vectors of the document at `position`; `Absent` past the end.+ pub(crate) fn doc(&self, position: usize) -> DocVectorsView<'_> {+ match self.docs.get(position) {+ None | Some(DocVectors::Absent) => DocVectorsView::Absent,+ Some(DocVectors::Pooled(entry)) => DocVectorsView::Pooled(self.stored(*entry)),+ Some(DocVectors::Chunks { first, count }) => DocVectorsView::Chunks(ChunkIter {+ column: self,+ next: *first,+ end: first + count,+ }),+ }+ }++ /// Bytes this column keeps alive, for the segment hydration budget.+ pub(crate) fn retained_bytes(&self) -> u64 {+ let bytes = self.data.capacity() * std::mem::size_of::<f32>()+ + self.entries.capacity() * std::mem::size_of::<VectorEntry>()+ + self.docs.capacity() * std::mem::size_of::<DocVectors>()+ + std::mem::size_of::<Self>();+ u64::try_from(bytes).unwrap_or(u64::MAX)+ }+}++/// Builds a [`SegmentVectorColumn`] one document at a time, in position order.+pub(crate) struct ColumnBuilder {+ field: String,+ chunk_field: String,+ mode: ColumnMode,+ column: SegmentVectorColumn,+}++impl ColumnBuilder {+ pub(crate) fn new(field: &str, mode: ColumnMode) -> Self {+ Self {+ field: field.to_string(),+ chunk_field: format!("{field}_chunks"),+ mode,+ column: SegmentVectorColumn::default(),+ }+ }++ /// Forget every document but keep the allocations — the memtable path+ /// reuses one builder as per-document scratch.+ pub(crate) fn clear(&mut self) {+ self.column.data.clear();+ self.column.entries.clear();+ self.column.docs.clear();+ }++ /// The column built so far.+ pub(crate) fn column(&self) -> &SegmentVectorColumn {+ &self.column+ }++ pub(crate) fn finish(mut self) -> SegmentVectorColumn {+ self.column.data.shrink_to_fit();+ self.column.entries.shrink_to_fit();+ self.column.docs.shrink_to_fit();+ self.column+ }++ /// Append a segment's stored document: `{"_id", "_seq_no", "_source"}`,+ /// or a legacy pre-`_source` document whose fields sit at the top level.+ pub(crate) fn push_stored_doc(&mut self, stored: &Value) {+ self.push_source(stored_source_view(stored));+ }++ /// Append a document given its source object.+ pub(crate) fn push_source(&mut self, source: &Value) {+ let doc = match self.mode {+ ColumnMode::PooledOnly => self.push_pooled(source),+ ColumnMode::BestPassage => match field_value_cow(source, &self.chunk_field) {+ Some(chunks) => match chunks.as_ref() {+ Value::Array(chunks) => self.push_chunks(chunks),+ _ => self.push_pooled(source),+ },+ None => self.push_pooled(source),+ },+ };+ self.column.docs.push(doc);+ }++ fn push_chunks(&mut self, chunks: &[Value]) -> DocVectors {+ let first = self.column.entries.len();+ for (position, chunk) in chunks.iter().enumerate() {+ let Value::Array(elements) = chunk else {+ continue;+ };+ // The ordinal is the passage's position in the ORIGINAL array, so+ // a skipped passage never renumbers the ones after it.+ let Ok(ordinal) = u32::try_from(position) else {+ continue;+ };+ self.push_vector(elements, ordinal);+ }+ DocVectors::Chunks {+ first,+ count: self.column.entries.len() - first,+ }+ }++ fn push_pooled(&mut self, source: &Value) -> DocVectors {+ match field_value_cow(source, &self.field).as_deref() {+ Some(Value::Array(elements)) => DocVectors::Pooled(self.push_vector(elements, 0)),+ _ => DocVectors::Absent,+ }+ }++ fn push_vector(&mut self, elements: &[Value], ordinal: u32) -> usize {+ let start = self.column.data.len();+ self.column+ .data+ .extend(elements.iter().filter_map(|v| v.as_f64().map(|f| f as f32)));+ let len = self.column.data.len() - start;+ let norm = norm_f64(&self.column.data[start..]);+ self.column.entries.push(VectorEntry {+ start,+ len,+ ordinal,+ norm,+ });+ self.column.entries.len() - 1+ }+}++/// The source object of a segment's stored document, without cloning it.+///+/// Reassembled segment documents are `{"_id", "_seq_no", "_source": {...}}`;+/// legacy pre-M7 segments keep the fields at the top level, beside `_id`.+pub(crate) fn stored_source_view(stored: &Value) -> &Value {+ stored.get("_source").unwrap_or(stored)+}++/// [`get_field_value`] without the clone when the field is a literal key at+/// the source root — which is where a vector field and its generated+/// companions live. Anything else (a dotted path, an array of objects, the+/// multi-field fallback) is answered by `get_field_value` itself, so the two+/// resolve identically by construction.+fn field_value_cow<'a>(source: &'a Value, field: &str) -> Option<Cow<'a, Value>> {+ if let Value::Object(map) = source {+ if let Some(value) = map.get(field) {+ return Some(Cow::Borrowed(value));+ }+ }+ get_field_value(source, field).map(Cow::Owned)+}++/// Strictly sequential `f64` dot product. Shared with+/// `compute_vector_similarity`; do not reorder or widen this reduction — a+/// different summation order changes the low bits of every score.+#[inline]+pub(crate) fn dot_f64(a: &[f32], b: &[f32]) -> f64 {+ a.iter()+ .zip(b.iter())+ .map(|(x, y)| (*x as f64) * (*y as f64))+ .sum()+}++/// Strictly sequential `f64` Euclidean norm. Same warning as [`dot_f64`].+#[inline]+pub(crate) fn norm_f64(v: &[f32]) -> f64 {+ v.iter()+ .map(|x| (*x as f64) * (*x as f64))+ .sum::<f64>()+ .sqrt()+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ fn views(column: &SegmentVectorColumn, position: usize) -> Vec<(u32, Vec<f32>)> {+ match column.doc(position) {+ DocVectorsView::Absent => Vec::new(),+ DocVectorsView::Pooled(v) => vec![(v.ordinal, v.values.to_vec())],+ DocVectorsView::Chunks(iter) => iter.map(|v| (v.ordinal, v.values.to_vec())).collect(),+ }+ }++ #[test]+ fn best_passage_prefers_the_chunk_companion_over_the_pooled_vector() {+ let mut b = ColumnBuilder::new("body_vector", ColumnMode::BestPassage);+ b.push_stored_doc(&json!({"_id": "a", "_source": {+ "body_vector": [9.0, 9.0],+ "body_vector_chunks": [[1.0, 0.0], [0.0, 1.0]],+ }}));+ let column = b.finish();+ assert_eq!(+ views(&column, 0),+ vec![(0, vec![1.0, 0.0]), (1, vec![0.0, 1.0])]+ );+ }++ #[test]+ fn skipped_passages_keep_the_ordinals_of_the_ones_after_them() {+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);+ b.push_source(&json!({"v_chunks": [[1.0, 0.0], "not a vector", [0.0, 1.0]]}));+ let column = b.finish();+ assert_eq!(+ views(&column, 0),+ vec![(0, vec![1.0, 0.0]), (2, vec![0.0, 1.0])]+ );+ }++ #[test]+ fn an_empty_passage_array_skips_the_document_instead_of_falling_back() {+ // The scan this replaces `continue`d here; falling back to the pooled+ // vector would surface a document it never returned.+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);+ b.push_source(&json!({"v": [1.0, 0.0], "v_chunks": ["x", 3]}));+ b.push_source(&json!({"v": [1.0, 0.0], "v_chunks": []}));+ let column = b.finish();+ assert!(views(&column, 0).is_empty());+ assert!(views(&column, 1).is_empty());+ }++ #[test]+ fn a_non_array_chunk_companion_falls_back_to_the_pooled_vector() {+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);+ b.push_source(&json!({"v": [0.5, 0.25], "v_chunks": "nope"}));+ let column = b.finish();+ assert_eq!(views(&column, 0), vec![(0, vec![0.5, 0.25])]);+ }++ #[test]+ fn pooled_only_ignores_the_chunk_companion() {+ let mut b = ColumnBuilder::new("v", ColumnMode::PooledOnly);+ b.push_source(&json!({"v": [0.5, 0.25], "v_chunks": [[1.0, 0.0]]}));+ let column = b.finish();+ assert_eq!(views(&column, 0), vec![(0, vec![0.5, 0.25])]);+ }++ #[test]+ fn non_numeric_elements_are_dropped_like_the_scan_dropped_them() {+ // `filter_map(as_f64)` shortens the vector; the length check against+ // the query then rejects it at query time, as it always did.+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);+ b.push_source(&json!({"v": [1.0, "x", 2.0]}));+ let column = b.finish();+ assert_eq!(views(&column, 0), vec![(0, vec![1.0, 2.0])]);+ }++ #[test]+ fn missing_and_non_array_vectors_are_absent_and_positions_stay_aligned() {+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);+ b.push_source(&json!({"other": 1}));+ b.push_source(&json!({"v": "text"}));+ b.push_source(&json!({"v": [3.0]}));+ let column = b.finish();+ assert_eq!(column.len(), 3);+ assert!(views(&column, 0).is_empty());+ assert!(views(&column, 1).is_empty());+ assert_eq!(views(&column, 2), vec![(0, vec![3.0])]);+ assert!(views(&column, 99).is_empty(), "past the end is absent");+ }++ #[test]+ fn legacy_documents_without_source_are_read_at_the_top_level() {+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);+ b.push_stored_doc(&json!({"_id": "legacy", "v": [1.0, 2.0]}));+ let column = b.finish();+ assert_eq!(views(&column, 0), vec![(0, vec![1.0, 2.0])]);+ }++ #[test]+ fn dotted_vector_fields_resolve_through_get_field_value() {+ let mut b = ColumnBuilder::new("emb.v", ColumnMode::BestPassage);+ b.push_source(&json!({"emb": {"v": [1.0, 2.0]}}));+ let column = b.finish();+ assert_eq!(views(&column, 0), vec![(0, vec![1.0, 2.0])]);+ }++ #[test]+ fn stored_norm_is_the_value_the_scan_would_recompute() {+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);+ b.push_source(&json!({"v": [0.1, -0.7, 0.33, 1e-3]}));+ let column = b.finish();+ let DocVectorsView::Pooled(v) = column.doc(0) else {+ panic!("expected a pooled vector");+ };+ assert_eq!(v.norm.to_bits(), norm_f64(v.values).to_bits());+ }++ #[test]+ fn clear_reuses_the_builder_as_scratch() {+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);+ b.push_source(&json!({"v": [1.0]}));+ b.clear();+ b.push_source(&json!({"v": [2.0]}));+ assert_eq!(b.column().len(), 1);+ assert_eq!(views(b.column(), 0), vec![(0, vec![2.0])]);+ }+}