Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(knn): exact scan ranks addresses and hydrates only the winners (#939)

xerj-org61966df3cc29
diff --git a/benchmarks/hybrid-scan-939/common.py b/benchmarks/hybrid-scan-939/common.py
new file mode 100755
index 000000000..02820bc74
--- /dev/null
+++ b/benchmarks/hybrid-scan-939/common.py
@@ -0,0 +1,89 @@
+"""Shared helpers for the #939 exact-scan benchmark. Standard library only."""
+import hashlib
+import json
+import os
+import urllib.error
+import urllib.request
+
+API_KEY = os.environ.get("XERJ_API_KEY", "")
+
+
+def http(url, method, path, body=None, content_type="application/json", timeout=600, raw=False):
+ data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode()
+ headers = {"content-type": content_type}
+ if API_KEY:
+ headers["authorization"] = "ApiKey " + API_KEY
+ req = urllib.request.Request(url + path, data=data, method=method, headers=headers)
+ try:
+ payload = urllib.request.urlopen(req, timeout=timeout).read()
+ except urllib.error.HTTPError as e:
+ return {"_err": e.code, "body": e.read().decode()[:400]}
+ return payload.decode() if raw else json.loads(payload)
+
+
+def loadavg():
+ with open("/proc/loadavg") as f:
+ return f.read().split()[0]
+
+
+def cache_counters(url):
+ """(hits, misses) of the result cache, scraped from the node's own gauges.
+
+ A run that claims to have measured the engine must show hits did not move.
+ """
+ text = http(url, "GET", "/v1/metrics", raw=True)
+ if isinstance(text, dict):
+ raise SystemExit(f"cannot read /v1/metrics: {text}")
+ found = {}
+ for line in text.splitlines():
+ for name in ("xerj_query_cache_hits", "xerj_query_cache_misses"):
+ if line.startswith(name + " ") or line.startswith(name + "{"):
+ found[name] = found.get(name, 0) + int(float(line.rsplit(" ", 1)[1]))
+ if len(found) != 2:
+ raise SystemExit(f"result-cache gauges missing from /v1/metrics: {found}")
+ return found["xerj_query_cache_hits"], found["xerj_query_cache_misses"]
+
+
+def distinct_queries(data_dir, n):
+ """First n DISTINCT query texts, in file order."""
+ seen, out = set(), []
+ with open(data_dir + "/queries.jsonl") as f:
+ for line in f:
+ text = json.loads(line)["text"]
+ if text not in seen:
+ seen.add(text)
+ out.append(text)
+ if len(out) == n:
+ break
+ if len(out) < n:
+ raise SystemExit(f"only {len(out)} distinct queries in {data_dir}, need {n}")
+ return out
+
+
+def bucket_of(doc_id):
+ """Deterministic 10-way keyword partition of the corpus, for the filter arms."""
+ return "b%d" % (int(hashlib.md5(doc_id.encode()).hexdigest(), 16) % 10)
+
+
+def n_of(doc_id):
+ """Deterministic integer in [0, 1000), for the range-filter arm."""
+ return int(hashlib.md5(("n" + doc_id).encode()).hexdigest(), 16) % 1000
+
+
+def bm25_query(q):
+ return {"multi_match": {"query": q, "fields": ["title", "text"]}}
+
+
+def semantic_query(q, k=10, flt=None):
+ body = {"field": "body", "query": q, "k": k}
+ if flt is not None:
+ body["filter"] = flt
+ return {"semantic": body}
+
+
+def hybrid_query(q, k=10):
+ return {"hybrid": {"queries": [{"query": bm25_query(q)}, {"query": semantic_query(q, k)}], "fusion": "rrf"}}
+
+
+def pct(sorted_values, p):
+ return sorted_values[min(len(sorted_values) - 1, int(len(sorted_values) * p))]
diff --git a/benchmarks/hybrid-scan-939/compare.py b/benchmarks/hybrid-scan-939/compare.py
new file mode 100755
index 000000000..e730de04b
--- /dev/null
+++ b/benchmarks/hybrid-scan-939/compare.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Compare two identity.py dumps. Exit 1 on ANY difference.
+
+ python3 compare.py before.json after.json
+
+Tolerance is ZERO: same ids, same order, same `_score` (the exact JSON number),
+same `_source`, same totals and relations, same passages. The scan rewrite in
+#939 is meant to be bit-identical, so a tolerance would only hide a defect.
+"""
+import json
+import sys
+
+a, b = (json.load(open(p)) for p in sys.argv[1:3])
+bad = 0
+if a.keys() != b.keys():
+ print("ARMS DIFFER:", sorted(a.keys() ^ b.keys()))
+ bad += 1
+for arm in sorted(a.keys() & b.keys()):
+ rows_a, rows_b = a[arm], b[arm]
+ diffs, hits = 0, 0
+ if len(rows_a) != len(rows_b):
+ print(f"{arm}: query count differs {len(rows_a)} vs {len(rows_b)}")
+ bad += 1
+ continue
+ for ra, rb in zip(rows_a, rows_b):
+ hits += len(ra["response"].get("hits", {}).get("hits", []))
+ if ra != rb:
+ diffs += 1
+ if diffs <= 3:
+ ha = [(h["_id"], h.get("_score")) for h in ra["response"].get("hits", {}).get("hits", [])]
+ hb = [(h["_id"], h.get("_score")) for h in rb["response"].get("hits", {}).get("hits", [])]
+ first = next((i for i, (x, y) in enumerate(zip(ha, hb)) if x != y), None)
+ print(f" {arm}: DIFF q={ra['q'][:60]!r} first_differing_rank={first}"
+ f" before={ha[first] if first is not None else None} after={hb[first] if first is not None else None}"
+ f" total {ra['response'].get('hits', {}).get('total')} vs {rb['response'].get('hits', {}).get('total')}")
+ print(f"{arm:32s} queries={len(rows_a)} hits_compared={hits} differing_responses={diffs}")
+ bad += diffs
+print("IDENTICAL" if bad == 0 else f"NOT IDENTICAL: {bad} differing responses")
+sys.exit(0 if bad == 0 else 1)
diff --git a/benchmarks/hybrid-scan-939/identity.py b/benchmarks/hybrid-scan-939/identity.py
new file mode 100755
index 000000000..88b526319
--- /dev/null
+++ b/benchmarks/hybrid-scan-939/identity.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Dump full responses for a fixed query set, to compare two binaries.
+
+ XERJ_API_KEY=... python3 identity.py http://localhost:10840 scifact /path/to/beir/scifact 100 out.json
+
+Read-only. Deeper and heavier than latency.py on purpose: `size`/`k` 100 so a
+reordering far from the top still shows, and `_source` ON for the vector arms
+so the hydrated documents are compared too, not only ids and scores. `took` is
+dropped; everything else in the response is kept verbatim.
+
+Run it against the OLD binary and the NEW binary over the SAME data directory
+(same segments, same document order), then feed both dumps to compare.py.
+"""
+import json
+import sys
+
+from common import bm25_query, distinct_queries, http, hybrid_query, semantic_query
+
+url, index, data_dir, n, out = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5]
+qs = distinct_queries(data_dir, n)
+K = 100
+
+
+def search(query, size, source):
+ r = http(url, "POST", f"/{index}/_search", {"size": size, "_source": source, "query": query})
+ if "_err" in r:
+ raise SystemExit(f"search failed: {r}")
+ r.pop("took", None)
+ return r
+
+
+top30 = {q: [h["_id"] for h in search(bm25_query(q), 30, False)["hits"]["hits"]] for q in qs}
+TERM = {"term": {"bucket": "b3"}}
+TERM_AND_RANGE = {"bool": {"filter": [{"term": {"bucket": "b3"}}, {"range": {"n": {"gte": 100, "lt": 900}}}]}}
+# Filters the projection fast path must DECLINE (not on its whitelist), so the
+# fallback path is compared too.
+MATCH_FILTER = {"match": {"title": "cell"}}
+
+arms = {
+ "semantic": lambda q: search(semantic_query(q, K), K, True),
+ "semantic_ids": lambda q: search(semantic_query(q, K, {"ids": {"values": top30[q]}}), K, True),
+ "semantic_term": lambda q: search(semantic_query(q, K, TERM), K, True),
+ "semantic_term_range": lambda q: search(semantic_query(q, K, TERM_AND_RANGE), K, True),
+ "semantic_match_filter": lambda q: search(semantic_query(q, K, MATCH_FILTER), K, True),
+ "semantic_passage": lambda q: http(url, "POST", f"/{index}/_search", {
+ "size": 10, "_source": False, "fields": ["_passage"], "query": semantic_query(q, 10)}),
+ "hybrid": lambda q: search(hybrid_query(q, K), K, True),
+ "bool_must_semantic_filter_ids": lambda q: search(
+ {"bool": {"must": [semantic_query(q, K)], "filter": [{"ids": {"values": top30[q]}}]}}, K, False),
+}
+dump = {}
+for name, fn in arms.items():
+ rows = []
+ for q in qs:
+ r = fn(q)
+ r.pop("took", None)
+ rows.append({"q": q, "response": r})
+ dump[name] = rows
+ hits = sum(len(row["response"].get("hits", {}).get("hits", [])) for row in rows)
+ print(f"{name:32s} queries={len(rows)} hits_total={hits}", flush=True)
+with open(out, "w") as f:
+ json.dump(dump, f, sort_keys=True)
+print("wrote", out)
diff --git a/benchmarks/hybrid-scan-939/latency.py b/benchmarks/hybrid-scan-939/latency.py
new file mode 100755
index 000000000..6a027c062
--- /dev/null
+++ b/benchmarks/hybrid-scan-939/latency.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Per-arm search latency, closed loop, one client.
+
+ XERJ_API_KEY=... python3 latency.py http://localhost:10840 scifact /path/to/beir/scifact 300
+
+Read-only (`_search`, `/v1/metrics`). Every arm sends N DISTINCT query texts,
+so no request inside an arm repeats. Arms differ in request body, and the
+result cache is keyed on the whole body, so arms cannot serve each other. That
+is not taken on trust: the node's own result-cache hit gauge is scraped around
+every arm and the delta printed. A row whose `cache_hits` is not 0 measured
+the cache, not the engine.
+
+The last arm repeats ONE request on purpose and is expected to show hits: it
+is there so the trap stays visible to whoever runs this next.
+
+Machine load (1-minute loadavg) is printed per arm. A latency taken while
+other jobs saturate the box is an upper bound, not a measurement of the engine.
+"""
+import sys
+import time
+
+from common import (bm25_query, cache_counters, distinct_queries, http, hybrid_query, loadavg, pct,
+ semantic_query)
+
+url, index, data_dir, n = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
+qs = distinct_queries(data_dir, n)
+
+
+def search(query, size=10):
+ r = http(url, "POST", f"/{index}/_search", {"size": size, "_source": False, "query": query})
+ if "_err" in r:
+ raise SystemExit(f"search failed: {r}")
+ return r
+
+
+def timed(query):
+ t = time.perf_counter()
+ r = search(query)
+ return time.perf_counter() - t, r
+
+
+# The ids filter names the BM25 top-30 of the same text. Those lookups are
+# made up front so they are not inside any timed region.
+print("preparing ids filters (untimed) ...", flush=True)
+top30 = {q: [h["_id"] for h in search(bm25_query(q), 30)["hits"]["hits"]] for q in qs}
+
+TERM = {"term": {"bucket": "b3"}}
+TERM_AND_RANGE = {"bool": {"filter": [{"term": {"bucket": "b3"}}, {"range": {"n": {"gte": 100, "lt": 900}}}]}}
+
+arms = [
+ ("bm25", lambda q: timed(bm25_query(q))),
+ ("semantic (unfiltered)", lambda q: timed(semantic_query(q))),
+ ("semantic + ids filter (30 docs)", lambda q: timed(semantic_query(q, flt={"ids": {"values": top30[q]}}))),
+ ("semantic + term filter (~10%)", lambda q: timed(semantic_query(q, flt=TERM))),
+ ("semantic + term AND range filter", lambda q: timed(semantic_query(q, flt=TERM_AND_RANGE))),
+ ("hybrid rrf", lambda q: timed(hybrid_query(q))),
+ ("semantic, SAME request repeated", lambda q: timed(semantic_query(qs[0], k=11))),
+]
+print(f"index={index} docs={http(url, 'GET', f'/{index}/_count').get('count')} distinct_queries={len(qs)}")
+for name, fn in arms:
+ hits0, _ = cache_counters(url)
+ load0 = loadavg()
+ print("ARM_START", name, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()), flush=True)
+ lat, took = [], []
+ for q in qs:
+ dt, r = fn(q)
+ lat.append(dt * 1000)
+ took.append(r.get("took", 0))
+ print("ARM_END", name, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()), flush=True)
+ hits1, _ = cache_counters(url)
+ lat.sort()
+ took.sort()
+ print(f"{name:34s} n={len(lat)} p50={pct(lat, .5):7.1f}ms p95={pct(lat, .95):7.1f}ms"
+ f" server_took p50={pct(took, .5)}ms p95={pct(took, .95)}ms"
+ f" cache_hits={hits1 - hits0} loadavg={load0}->{loadavg()}", flush=True)
diff --git a/benchmarks/hybrid-scan-939/load.py b/benchmarks/hybrid-scan-939/load.py
new file mode 100755
index 000000000..27660838a
--- /dev/null
+++ b/benchmarks/hybrid-scan-939/load.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Load a BEIR corpus into a throwaway XERJ node.
+
+ XERJ_API_KEY=... python3 load.py http://localhost:10840 scifact /path/to/beir/scifact [clients]
+
+WRITES: it deletes and recreates the index named on the command line. Point it
+only at a node you started yourself, on a port you own.
+
+`body` is `semantic_text` (title + ". " + text, as the BEIR harness this was
+copied from built it), so every document longer than 512 characters is stored
+as several passage vectors — the shape issue #939 is about. `bucket` and `n`
+are deterministic functions of `_id` that give the filter arms something to
+select on. `clients` > 1 sends `_bulk` requests concurrently (useful under
+`--embed-mode neural`, see #938); document order inside the index then depends
+on arrival order, so build the index ONCE and run both binaries over that same
+data directory when comparing results.
+"""
+import json
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+from common import bucket_of, http, loadavg, n_of
+
+url, index, data_dir = sys.argv[1], sys.argv[2], sys.argv[3]
+clients = int(sys.argv[4]) if len(sys.argv) > 4 else 1
+BULK = 100
+
+print("delete:", http(url, "DELETE", "/" + index))
+print("create:", http(url, "PUT", "/" + index, {"mappings": {"properties": {
+ "title": {"type": "text"}, "text": {"type": "text"},
+ "bucket": {"type": "keyword"}, "n": {"type": "integer"},
+ "body": {"type": "semantic_text"}}}}))
+
+docs = [json.loads(line) for line in open(data_dir + "/corpus.jsonl")]
+
+
+def send(start):
+ lines = []
+ for d in docs[start:start + BULK]:
+ lines.append(json.dumps({"index": {"_index": index, "_id": d["_id"]}}))
+ lines.append(json.dumps({
+ "title": d["title"], "text": d["text"],
+ "bucket": bucket_of(d["_id"]), "n": n_of(d["_id"]),
+ "body": d["title"] + ". " + d["text"]}))
+ r = http(url, "POST", "/_bulk", ("\n".join(lines) + "\n").encode(), "application/x-ndjson", timeout=3600)
+ if r.get("errors") or "_err" in r:
+ raise SystemExit("bulk problem: " + str(r)[:400])
+ return len(lines) // 2
+
+
+t0 = time.time()
+done = 0
+with ThreadPoolExecutor(max_workers=clients) as pool:
+ for n in pool.map(send, range(0, len(docs), BULK)):
+ done += n
+ if done % 1000 < BULK:
+ print(f"indexed {done}/{len(docs)} {done / (time.time() - t0):.1f} docs/s loadavg={loadavg()}", flush=True)
+print("refresh:", http(url, "POST", f"/{index}/_refresh"))
+print("flush:", http(url, "POST", f"/{index}/_flush"))
+print("count:", http(url, "GET", f"/{index}/_count"))
+print(f"wall={time.time() - t0:.1f}s clients={clients} loadavg={loadavg()}")
diff --git a/benchmarks/hybrid-scan-939/phases.py b/benchmarks/hybrid-scan-939/phases.py
new file mode 100755
index 000000000..086682306
--- /dev/null
+++ b/benchmarks/hybrid-scan-939/phases.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Split traced semantic requests into phases, per latency.py arm.
+
+ python3 phases.py server.log latency.log
+
+Start the node with XERJ_TRACE_SEMANTIC_PHASES=1 NO_COLOR=1. latency.py prints
+ARM_START/ARM_END stamps (UTC, 1 s resolution); this buckets the node's
+`semantic_phase=` lines by those windows. One client, closed loop, so lines of
+one request are contiguous: embed_start → embed_complete → start_brute →
+load_segment* → scored → complete.
+"""
+import re
+import sys
+from datetime import datetime, timezone
+
+server_log, latency_log = sys.argv[1], sys.argv[2]
+arms = []
+start = {}
+for line in open(latency_log):
+ m = re.match(r"ARM_(START|END) (.+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)$", line.strip())
+ if not m:
+ continue
+ t = datetime.strptime(m.group(3), "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc)
+ if m.group(1) == "START":
+ start[m.group(2)] = t
+ else:
+ arms.append((m.group(2), start[m.group(2)], t))
+
+ts_re = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)\.\d+Z")
+kv = lambda line, key: (lambda m: int(m.group(1)) if m else None)(re.search(key + r"=(\d+)", line))
+requests, cur = [], None
+for line in open(server_log, errors="replace"):
+ if "semantic_phase=" not in line:
+ continue
+ m = ts_re.match(line)
+ if not m:
+ continue
+ t = datetime.strptime(m.group(1), "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc)
+ if "semantic_phase=embed_start" in line:
+ cur = {"t": t}
+ requests.append(cur)
+ elif cur is None:
+ continue
+ elif "semantic_phase=embed_complete" in line:
+ cur["embed"] = kv(line, "elapsed_ms")
+ elif "semantic_phase=scored" in line:
+ cur["collect"] = kv(line, "collect_ms")
+ cur["scored_at"] = kv(line, "elapsed_ms")
+ cur["scored_docs"] = kv(line, "scored")
+ elif "semantic_phase=complete" in line:
+ cur["total"] = kv(line, "elapsed_ms")
+
+
+def p50(xs):
+ xs = sorted(x for x in xs if x is not None)
+ return xs[len(xs) // 2] if xs else float("nan")
+
+
+for name, t0, t1 in arms:
+ rs = [r for r in requests if t0 <= r["t"] <= t1 and "total" in r]
+ if not rs:
+ continue
+ score = [r["scored_at"] - r["collect"] for r in rs if r.get("scored_at") is not None and r.get("collect") is not None]
+ tail = [r["total"] - r["scored_at"] for r in rs if r.get("scored_at") is not None]
+ print(f"{name:34s} n={len(rs):4d} embed p50={p50([r.get('embed') for r in rs]):6.1f}"
+ f" collect p50={p50([r.get('collect') for r in rs]):6.1f} score p50={p50(score):6.1f}"
+ f" topk+hits p50={p50(tail):6.1f} scan_total p50={p50([r['total'] for r in rs]):6.1f}"
+ f" scored_docs p50={p50([r.get('scored_docs') for r in rs])}")
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 1b4c97c4d..0044a617b 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -6867,6 +6867,7 @@ dependencies = [
"chrono-tz",
"criterion",
"dashmap",
+ "futures-util",
"libc",
"memchr",
"parking_lot",
diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.toml
index df749b25b..93261f4e8 100644
--- a/engine/crates/xerj-engine/Cargo.toml
+++ b/engine/crates/xerj-engine/Cargo.toml
@@ -19,6 +19,9 @@ xerj-wasm = { path = "../xerj-wasm" }
xerj-cluster = { path = "../xerj-cluster" }
tokio = { workspace = true }
+# `join_all` for the hybrid legs (#939). Already in the lockfile via the
+# workspace's `futures`; no new crate is pulled in.
+futures-util = "0.3"
serde = { workspace = true }
serde_json = { workspace = true }
simd-json = { workspace = true }
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 55a4ead28..446d4e819 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -6245,6 +6245,25 @@ mod semantic_deadline_regression_tests {
Arc::new(reader),
);
}
+ SegmentCacheCategory::VectorColumn => {
+ let key = format!("{old_id}\u{1}embedding\u{1}c");
+ let mut builder = crate::vector_column::ColumnBuilder::new(
+ "embedding",
+ crate::vector_column::ColumnMode::BestPassage,
+ );
+ builder.push_stored_doc(&serde_json::json!({
+ "_id": "v1",
+ "_source": {"embedding": [1.0, 0.0]}
+ }));
+ let _ = idx.publish_current(
+ &idx.vector_column_cache,
+ &old_id,
+ key,
+ category,
+ 1,
+ builder.finish(),
+ );
+ }
}));
}
@@ -6288,6 +6307,10 @@ mod semantic_deadline_regression_tests {
.sort_shadow_cache
.iter()
.any(|entry| entry.key().starts_with(&prefix)));
+ assert!(!idx
+ .vector_column_cache
+ .iter()
+ .any(|entry| entry.key().starts_with(&prefix)));
}
#[tokio::test]
@@ -8134,6 +8157,25 @@ pub struct Index {
/// it is asked for; a reader opened for a narrower set must never satisfy
/// a query needing a wider one.
fts_reader_cache: Arc<dashmap::DashMap<String, Resident<Arc<FtsIndexReader>>>>,
+ /// Flat `f32` vectors of one vector field over one segment, keyed
+ /// `"<segment_id>\u{1}<field>\u{1}<mode>"` (#939).
+ ///
+ /// The exact kNN scan used to read every vector out of the parsed
+ /// `_source` tree on every query — after deep-cloning that tree, vectors
+ /// included. `stored_value_cache` saves the disk read and the parse; it
+ /// cannot save a walk over ~10 M `Value::Number`s per query on a 5 k
+ /// document multi-passage corpus. This holds the same numbers once, as
+ /// `f32`, at ~1/8 of the bytes the parsed tree spends on them.
+ ///
+ /// Same lifecycle as every other cache here: segments are immutable so an
+ /// entry never goes stale, it is evicted by segment-id prefix when a merge
+ /// retires the segment, and it is charged to the hydration budget — over
+ /// budget the scan builds the column per query and drops it, which is
+ /// still cheaper than the clones it replaced. Liveness is NOT in the
+ /// column: deletes and updates are resolved per query against the version
+ /// map, exactly as before.
+ vector_column_cache:
+ Arc<dashmap::DashMap<String, Resident<crate::vector_column::SegmentVectorColumn>>>,
/// Per-(segment, query-shape) match-count cache for the
/// `try_shortcut_count` Bool intersection arm. The fused columnar
/// walk is O(anchor-predicate matches) per segment PER QUERY — for a
@@ -8299,7 +8341,8 @@ impl Index {
!collect_dense_vector_fields(&schema.schema).is_empty()
}
- /// Aggregate DashMap table capacities for the seven hydration families.
+ /// Aggregate DashMap table capacities for the hydration families, in
+ /// `SegmentCacheCategory` order.
/// These are diagnostic table slots, separate from the refundable
/// retained-payload/key budget.
pub fn segment_hydration_cache_capacities(&self) -> [usize; CATEGORY_COUNT] {
@@ -8312,6 +8355,7 @@ impl Index {
self.row_seq_cache.capacity(),
self.decoded_stored_cache.capacity(),
self.fts_reader_cache.capacity(),
+ self.vector_column_cache.capacity(),
]
}
@@ -8623,6 +8667,7 @@ impl Index {
stored_slices_cache: Arc::new(per_index_map()),
decoded_stored_cache: Arc::new(per_index_map()),
fts_reader_cache: Arc::new(per_index_map()),
+ vector_column_cache: Arc::new(per_index_map()),
shortcut_count_cache: Arc::new(per_index_map()),
ghost_positions_cache: Arc::new(per_index_map()),
regexp_expand_cache: Arc::new(per_index_map()),
@@ -9019,6 +9064,7 @@ impl Index {
stored_slices_cache: Arc::new(per_index_map()),
decoded_stored_cache: Arc::new(per_index_map()),
fts_reader_cache: Arc::new(per_index_map()),
+ vector_column_cache: Arc::new(per_index_map()),
shortcut_count_cache: Arc::new(per_index_map()),
ghost_positions_cache: Arc::new(per_index_map()),
regexp_expand_cache: Arc::new(per_index_map()),
@@ -13770,6 +13816,8 @@ impl Index {
boost: Option<f32>,
min_similarity: Option<f32>,
) -> Result<SearchResult> {
+ use crate::vector_column::{ColumnBuilder, ColumnMode, DocVectorsView};
+
let started = std::time::Instant::now();
let mut timed_out = false;
let trace_phases = std::env::var_os("XERJ_TRACE_SEMANTIC_PHASES").is_some();
@@ -13787,18 +13835,49 @@ impl Index {
// this is the whole filtered-kNN path. Owned clone; the lock is released.
let dmq_schema = self.schema().await;
- // ── Collect all candidate (doc_id, source) pairs ──────────────
- let mut candidates: Vec<(String, Value)> = Vec::new();
- // Memtable first (newest writes).
- {
- let mem = &*self.memtable;
- candidates.extend(mem.all_docs_with_sources());
- }
+ // ── Determine whether this field opts into SQ8 (scalar8) ──────
+ // Default fields keep the exact f32 brute-force scan below,
+ // byte-identical to before. A `scalar8` field takes the branch above
+ // instead: it reads the same live f32 vector out of `_source`, but
+ // rounds it through 1-byte-per-dimension SQ8 codes before scoring, so
+ // the score carries `scalar8`'s precision loss. Nothing is cached —
+ // neither the codes nor the codebook outlive the query (#371).
+ let use_sq8 = {
+ let schema = self.schema.read().await;
+ lookup_vector_quantization(&schema.schema, field).as_deref() == Some("scalar8")
+ };
+ // `scalar8` reads only the pooled vector; the default scan prefers the
+ // per-passage companion. The two shapes are cached under different keys.
+ let column_mode = if use_sq8 {
+ ColumnMode::PooledOnly
+ } else {
+ ColumnMode::BestPassage
+ };
+
+ // ── Collect candidate ADDRESSES, not documents (#939) ─────────
+ // A candidate is where a live document sits — a memtable slot or a
+ // (segment, position) pair — never a copy of it. The scan this
+ // replaces deep-cloned every live `_source`, vectors included, into
+ // this list before it had scored or filtered anything: ~10 M
+ // `Value::Number`s per query on a 5 k-document multi-passage corpus,
+ // and half of each request. Sources are now cloned for the documents
+ // that are RETURNED, after ranking. Same shape as qdrant's
+ // `ScoredPointOffset` → `process_search_result` and tantivy's
+ // `DocAddress` → `Searcher::doc` (pointers in `vector_column.rs`).
+ //
+ // Candidate ORDER is unchanged — memtable first, then segments in
+ // snapshot order, then position — because the rank below is a stable
+ // sort and that order is what breaks score ties.
+ // Memtable first (newest writes). `Arc` clones, not tree clones.
+ let mem_docs: Vec<(String, Arc<Value>)> = self.memtable.all_docs_with_sources_arc();
+ let mut candidates: Vec<KnnCandidateAt> =
+ (0..mem_docs.len()).map(KnnCandidateAt::Memtable).collect();
// Then every flushed segment's stored section.
let snap = self.store.snapshot();
// Track seen IDs so later-segment copies don't duplicate memtable entries.
- let mut seen: HashSet<String> = candidates.iter().map(|(id, _)| id.clone()).collect();
- 'segments: for meta in snap.segments.iter() {
+ let mut seen: HashSet<String> = mem_docs.iter().map(|(id, _)| id.clone()).collect();
+ let mut segment_views: Vec<KnnSegmentView> = Vec::with_capacity(snap.segments.len());
+ for meta in snap.segments.iter() {
let segment_started = std::time::Instant::now();
// Cache-backed: first KNN against this segment pays the
// I/O + decompress + serde_json parse, every subsequent
@@ -13829,6 +13908,22 @@ impl Index {
timed_out = true;
break;
}
+ // The segment's vectors as flat `f32`s. Cached per immutable
+ // segment; on a miss it is derived in the SAME pass that collects
+ // candidates, so a cold query walks the stored documents once and
+ // keeps the cooperative checkpoints it always had. Every position
+ // is pushed — live or not — because liveness is this query's
+ // business and the column outlives it.
+ let column_key = format!("{}\u{1}{field}\u{1}{}", meta.id, column_mode.key_tag());
+ let cached_column = self
+ .vector_column_cache
+ .get(&column_key)
+ .map(|entry| Arc::clone(entry.value()));
+ let mut builder = cached_column
+ .is_none()
+ .then(|| ColumnBuilder::new(field, column_mode));
+ let view = segment_views.len();
+ let mut segment_complete = true;
// Cache-backed iteration: stored_values_for() handles the
// I/O + decode + parse; subsequent KNN over the same segment
// is an Arc clone. The underlying parse uses serde_json (not
@@ -13838,13 +13933,16 @@ impl Index {
for (position, doc) in docs_arc.iter().enumerate() {
if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {
timed_out = true;
- break 'segments;
+ segment_complete = false;
+ break;
}
- let id = match doc.get("_id").and_then(Value::as_str) {
- Some(s) => s.to_string(),
- None => continue,
+ if let Some(builder) = builder.as_mut() {
+ builder.push_stored_doc(doc);
+ }
+ let Some(id) = doc.get("_id").and_then(Value::as_str) else {
+ continue;
};
- if let Some(ver) = self.store.version_map.get(&id) {
+ if let Some(ver) = self.store.version_map.get(id) {
// Skip tombstoned (deleted) docs.
if ver.deleted {
continue;
@@ -13864,46 +13962,65 @@ impl Index {
}
}
}
- if !seen.insert(id.clone()) {
+ if seen.contains(id) {
continue;
}
- // Reassembled segment docs have shape
- // { "_id":..., "_seq_no":..., "_source": {...} }
- // but `get_field_value(src, "embedding")` further down
- // looks for the field at the top level (memtable path
- // shape). Unwrap to `_source` so either layout matches;
- // legacy pre-M7 segments without `_source` fall through
- // to the wrapper.
- let src = doc.get("_source").cloned().unwrap_or_else(|| {
- let mut d = doc.clone();
- if let Some(obj) = d.as_object_mut() {
- obj.remove("_id");
- }
- d
- });
- candidates.push((id, src));
- }
+ seen.insert(id.to_string());
+ candidates.push(KnnCandidateAt::Segment { view, position });
+ }
+ let column = match (cached_column, builder) {
+ (Some(column), _) => column,
+ // A column abandoned at the deadline covers a prefix of the
+ // segment. It serves this request's partial answer and is
+ // dropped — only a complete column is ever published.
+ (None, Some(builder)) if !segment_complete => {
+ CacheResident::uncached(builder.finish())
+ }
+ (None, Some(builder)) => {
+ let column = builder.finish();
+ let bytes = column
+ .retained_bytes()
+ .saturating_add(column_key.len() as u64);
+ // Over budget this comes back uncached: the next query
+ // derives it again, which is what every query did before.
+ self.publish_current(
+ &self.vector_column_cache,
+ &meta.id,
+ column_key,
+ SegmentCacheCategory::VectorColumn,
+ bytes,
+ column,
+ )
+ }
+ (None, None) => unreachable!("a builder exists whenever the column is not cached"),
+ };
+ segment_views.push(KnnSegmentView {
+ docs: docs_arc,
+ column,
+ });
if trace_phases {
tracing::info!(index=%self.name, segment=%meta.id, elapsed_ms=segment_started.elapsed().as_millis() as u64, candidates=candidates.len(), "semantic_phase=load_segment");
}
+ if !segment_complete {
+ break;
+ }
}
let collect_elapsed = started.elapsed();
- // ── Determine whether this field opts into SQ8 (scalar8) ──────
- // Default fields keep the exact f32 brute-force scan below,
- // byte-identical to before. A `scalar8` field takes the branch above
- // instead: it reads the same live f32 vector out of `_source`, but
- // rounds it through 1-byte-per-dimension SQ8 codes before scoring, so
- // the score carries `scalar8`'s precision loss. Nothing is cached —
- // neither the codes nor the codebook outlive the query (#371).
- let use_sq8 = {
- let schema = self.schema.read().await;
- lookup_vector_quantization(&schema.schema, field).as_deref() == Some("scalar8")
- };
+ // The filter runs BEFORE scoring and without the vectors (#939): see
+ // `KnnFilterPlan`. It used to clone the whole source a second time,
+ // per candidate, to insert `_id` beside ~1,900 numbers it never read.
+ let filter_plan = filter.as_deref().map(KnnFilterPlan::for_filter);
+ // Memtable documents have no column (the memtable is mutable), so
+ // their vectors are read out of the `Arc`'d source one document at a
+ // time into this reused scratch — after the filter, never before.
+ let mut memtable_scratch = ColumnBuilder::new(field, column_mode);
// ── Score each candidate against the query vector ─────────────
- let mut scored: Vec<(String, f32, Value, Option<u32>)> =
- Vec::with_capacity(candidates.len());
+ // `(candidate index, score, passage ordinal)`: 16 bytes a candidate,
+ // where this used to carry every candidate's cloned source through the
+ // sort and then drop all but `k` of them.
+ let mut scored: Vec<(usize, f32, Option<u32>)> = Vec::with_capacity(candidates.len());
if use_sq8 {
// Cosine fields are L2-normalised before quantising so SQ8 fits
// over bounded [-1,1] per-dim ranges (much better recall); cosine
@@ -13912,40 +14029,41 @@ impl Index {
let normalize = !matches!(similarity, "l2_norm" | "dot_product" | "max_inner_product");
let dim = query_vec.len();
- // Post-filter candidate vectors for this field: (id, src, doc_vec).
- let mut cand: Vec<(String, Value, Vec<f32>)> = Vec::with_capacity(candidates.len());
- for (position, (id, src)) in candidates.into_iter().enumerate() {
+ // Post-filter candidate vectors for this field: (candidate, doc_vec).
+ let mut cand: Vec<(usize, Vec<f32>)> = Vec::with_capacity(candidates.len());
+ for (position, at) in candidates.iter().enumerate() {
if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {
timed_out = true;
break;
}
- if let Some(ref f) = filter {
- let mut src_with_id = src.clone();
- if let Some(obj) = src_with_id.as_object_mut() {
- obj.insert("_id".to_string(), Value::String(id.clone()));
- }
- if !doc_matches_query_typed(f, &src_with_id, &dmq_schema) {
+ if let (Some(plan), Some(f)) = (filter_plan.as_ref(), filter.as_deref()) {
+ let (id, source) = at.id_and_source(&mem_docs, &segment_views);
+ if !plan.matches(f, id, source, &dmq_schema) {
continue;
}
}
- let vec_val = match get_field_value(&src, field) {
- Some(v) => v,
- None => continue,
+ let vectors = match *at {
+ KnnCandidateAt::Memtable(slot) => {
+ memtable_scratch.clear();
+ memtable_scratch.push_source(&mem_docs[slot].1);
+ memtable_scratch.column().doc(0)
+ }
+ KnnCandidateAt::Segment {
+ view,
+ position: doc_position,
+ } => segment_views[view].column.doc(doc_position),
};
- let mut doc_vec: Vec<f32> = match &vec_val {
- Value::Array(arr) => arr
- .iter()
- .filter_map(|v| v.as_f64().map(|f| f as f32))
- .collect(),
- _ => continue,
+ let DocVectorsView::Pooled(vector) = vectors else {
+ continue;
};
- if doc_vec.len() != dim {
+ if vector.values.len() != dim {
continue;
}
+ let mut doc_vec = vector.values.to_vec();
if normalize {
l2_normalize_vec(&mut doc_vec);
}
- cand.push((id, src, doc_vec));
+ cand.push((position, doc_vec));
}
// Fit this field's SQ8 codebook over the candidate vectors this
@@ -13994,7 +14112,7 @@ impl Index {
// INDEX time, so a Lucene score is a function of the index state
// alone; here it is a function of the query's candidate set too.
// #392 is what would close that gap.
- let params = Sq8Params::fit_borrowed(cand.iter().map(|(_, _, v)| v.as_slice()), dim);
+ let params = Sq8Params::fit_borrowed(cand.iter().map(|(_, v)| v.as_slice()), dim);
debug!(
field,
dim,
@@ -14011,7 +14129,7 @@ impl Index {
// document. `v.len() == dim` was established when `cand` was built.
let mut codes = vec![0u8; dim];
let mut decoded = vec![0.0f32; dim];
- for (position, (id, src, v)) in cand.into_iter().enumerate() {
+ for (position, (candidate, v)) in cand.into_iter().enumerate() {
if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {
timed_out = true;
break;
@@ -14019,7 +14137,7 @@ impl Index {
params.encode_into(&v, &mut codes);
params.decode_into(&codes, &mut decoded);
let score = compute_vector_similarity(similarity, query_vec, &decoded);
- scored.push((id, score, src, None));
+ scored.push((candidate, score, None));
}
} else {
// Per-chunk (passage) companion: multi-chunk `semantic_text` docs
@@ -14030,74 +14148,72 @@ impl Index {
// that section, not on the whole-doc average. Plain `dense_vector`
// kNN and short single-chunk docs have no such companion, so they
// fall through to the exact single-vector scan below (unchanged).
- let chunk_field = format!("{field}_chunks");
- for (position, (id, src)) in candidates.into_iter().enumerate() {
+ let scorer = QueryScorer::new(similarity, query_vec);
+ for (position, at) in candidates.iter().enumerate() {
if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {
timed_out = true;
break;
}
// Apply filter: if the filter doesn't match this doc, skip.
- if let Some(ref f) = filter {
- let mut src_with_id = src.clone();
- if let Some(obj) = src_with_id.as_object_mut() {
- obj.insert("_id".to_string(), Value::String(id.clone()));
- }
- if !doc_matches_query_typed(f, &src_with_id, &dmq_schema) {
+ if let (Some(plan), Some(f)) = (filter_plan.as_ref(), filter.as_deref()) {
+ let (id, source) = at.id_and_source(&mem_docs, &segment_views);
+ if !plan.matches(f, id, source, &dmq_schema) {
continue;
}
}
- let (score, passage_ordinal) =
- if let Some(Value::Array(chunks)) = get_field_value(&src, &chunk_field) {
+ let vectors = match *at {
+ KnnCandidateAt::Memtable(slot) => {
+ memtable_scratch.clear();
+ memtable_scratch.push_source(&mem_docs[slot].1);
+ memtable_scratch.column().doc(0)
+ }
+ KnnCandidateAt::Segment {
+ view,
+ position: doc_position,
+ } => segment_views[view].column.doc(doc_position),
+ };
+ let (score, passage_ordinal) = match vectors {
+ DocVectorsView::Absent => continue,
+ DocVectorsView::Chunks(passages) => {
// Best-matching passage over the stored chunk vectors.
let mut best: Option<(f32, u32)> = None;
- for (chunk_position, cv) in chunks.iter().enumerate() {
- if chunk_position & 31 == 0
- && self.exact_scan_checkpoint(chunk_position, deadline).await
+ for passage in passages {
+ // Bounds ONE pathological document. The per-128-
+ // documents checkpoint above is the scan's cadence;
+ // this used to fire at passage 0 of every document
+ // as well, which was one `yield_now` per document
+ // per query for no bound the outer check lacks.
+ if passage.ordinal > 0
+ && passage.ordinal & 31 == 0
+ && self
+ .exact_scan_checkpoint(passage.ordinal as usize, deadline)
+ .await
{
timed_out = true;
break;
}
- let dv: Vec<f32> = match cv {
- Value::Array(a) => a
- .iter()
- .filter_map(|v| v.as_f64().map(|f| f as f32))
- .collect(),
- _ => continue,
- };
- if dv.len() != query_vec.len() {
+ if passage.values.len() != query_vec.len() {
continue;
}
- let s = compute_vector_similarity(similarity, query_vec, &dv);
- let Some(ordinal) = u32::try_from(chunk_position).ok() else {
- continue;
- };
- choose_passage_winner(&mut best, s, ordinal);
+ choose_passage_winner(
+ &mut best,
+ scorer.score(passage),
+ passage.ordinal,
+ );
}
match best {
Some((score, ordinal)) => (score, Some(ordinal)),
None => continue,
}
- } else {
- let vec_val = match get_field_value(&src, field) {
- Some(v) => v,
- None => continue,
- };
- let doc_vec: Vec<f32> = match &vec_val {
- Value::Array(arr) => arr
- .iter()
- .filter_map(|v| v.as_f64().map(|f| f as f32))
- .collect(),
- _ => continue,
- };
- if doc_vec.len() != query_vec.len() {
+ }
+ DocVectorsView::Pooled(vector) => {
+ if vector.values.len() != query_vec.len() {
continue;
}
- (
- compute_vector_similarity(similarity, query_vec, &doc_vec),
- Some(0),
- )
- };
- scored.push((id, score, src, passage_ordinal));
+ (scorer.score(vector), Some(0))
+ }
+ };
+ scored.push((position, score, passage_ordinal));
}
}
if trace_phases {
@@ -14113,32 +14229,45 @@ impl Index {
// total (live-verified 2026-07-12).
if let Some(raw) = min_similarity {
let cut = raw_similarity_to_score(similarity, raw);
- scored.retain(|(_, score, _, _)| *score >= cut);
+ scored.retain(|(_, score, _)| *score >= cut);
}
// ES applies `boost` to the final `_score` AFTER the similarity
// cutoff (a boosted sub-threshold doc stays excluded; a boosted
// passing doc scores `transform(sim) * boost`).
if let Some(b) = boost {
if (b - 1.0).abs() > f32::EPSILON {
- for (_, score, _, _) in scored.iter_mut() {
+ for (_, score, _) in scored.iter_mut() {
*score *= b;
}
}
}
- // ── Rank, cap the candidate pool at k, then paginate ──────────
- // (shared with the HNSW path so hits format / total semantics
- // cannot drift between the exact and approximate executors)
+ // ── Rank, cap the pool at k, THEN hydrate (#939) ──────────────
+ // Same comparator, same stability and same `k.max(1)` cap as
+ // `knn_result_from_scored`, applied to 16-byte tuples instead of to
+ // every candidate's source. Only the survivors are cloned out of the
+ // shared stored values, so a query pays for `k` sources, not for the
+ // corpus.
+ rank_knn_pool(&mut scored, k, |entry| entry.1);
+ let ranked: Vec<(String, f32, Value, Option<u32>)> = scored
+ .into_iter()
+ .map(|(candidate, score, ordinal)| {
+ let (id, source) = candidates[candidate].hydrate(&mem_docs, &segment_views);
+ (id, score, source, ordinal)
+ })
+ .collect();
+
+ // (result shaping is shared with the HNSW path so hits format / total
+ // semantics cannot drift between the exact and approximate executors)
let generated_companion_fields = {
let schema = self.schema.read().await;
generated_embedding_companion_fields(&schema.schema)
};
- let mut result = knn_result_from_scored(
+ let mut result = knn_result_from_ranked(
self,
request,
field,
- scored,
- k,
+ ranked,
started,
&generated_companion_fields,
);
@@ -15027,11 +15156,22 @@ impl Index {
// for a deeper page.
let per_query_topk = request.size.saturating_add(request.from).max(50);
- // Run each sub-query sequentially. (Parallel `join_all` is the
- // obvious next step but means cloning Self into each future;
- // sequential is correct, easy to reason about, and good enough
- // for v0.7-P1 — the per-query latency is dominated by the kNN
- // / FTS scan, not the await ordering.)
+ // Run the legs CONCURRENTLY, not in parallel (#939). A search runs
+ // inside `block_in_place(block_on(..))`, so these futures share one
+ // thread and only overlap where a leg awaits. The await worth
+ // overlapping is the query embedding of a `semantic` leg under an
+ // active backend: a BERT forward pass on the blocking pool (~14 ms) or
+ // a proxy round trip, during which this thread used to sit idle and
+ // the lexical leg had not started. So legs that embed are POLLED
+ // FIRST — they park on the embedder, the lexical legs run underneath
+ // them — while results are still consumed in the caller's order, which
+ // is what fusion weights and the timeout rule below are defined over.
+ // Under the default lexical embedder the embedding is synchronous and
+ // there is nothing to overlap; this costs nothing there.
+ //
+ // Spawning each leg as its own task would buy real parallelism, but
+ // needs an `Arc<Self>` this `&self` chain does not have, next to the
+ // `block_in_place` hand-off that produced #751. Not for ~10 ms.
let mut sub_results: Vec<(Vec<Hit>, f32)> = Vec::with_capacity(sub_queries.len());
let mut sub_savings: Vec<PayloadSavings> = Vec::new();
let mut any_timed_out = false;
@@ -15041,12 +15181,9 @@ impl Index {
// the true match set, so the reported relation must be Gte, not a false
// Eq. Stays false only when every sub-list was fully retrieved.
let mut fused_count_capped = false;
- for wq in sub_queries {
- if std::time::Instant::now() >= deadline {
- any_timed_out = true;
- break;
- }
- let sub_request = SearchRequest {
+ let sub_requests: Vec<SearchRequest> = sub_queries
+ .iter()
+ .map(|wq| SearchRequest {
query: wq.query.clone(),
from: 0,
size: per_query_topk,
@@ -15066,10 +15203,37 @@ impl Index {
profile: false,
leaf_ts_field: None,
savings: request.savings,
- };
+ })
+ .collect();
+ let mut leg_outcomes: Vec<Option<Result<SearchResult>>> =
+ sub_requests.iter().map(|_| None).collect();
+ if std::time::Instant::now() >= deadline {
+ any_timed_out = true;
+ } else {
+ let mut poll_order: Vec<usize> = (0..sub_requests.len()).collect();
+ // Stable: legs that embed keep their relative order, then the rest.
+ poll_order.sort_by_key(|&leg| peel_semantic_query(&sub_requests[leg].query).is_none());
// Box::pin to break the type-recursion (search_inner ↔
// run_hybrid both async fn).
- let sub_result = Box::pin(self.search_inner(&sub_request, deadline)).await?;
+ let legs = poll_order
+ .iter()
+ .map(|&leg| Box::pin(self.search_inner(&sub_requests[leg], deadline)));
+ for (&leg, outcome) in poll_order
+ .iter()
+ .zip(futures_util::future::join_all(legs).await)
+ {
+ leg_outcomes[leg] = Some(outcome);
+ }
+ }
+ // Consume in the CALLER'S order with the sequential loop's exact
+ // rules: the first failing leg in that order is the error returned,
+ // and a leg that timed out ends the list — the legs after it are
+ // dropped, as they were when they had simply never been started.
+ for (wq, outcome) in sub_queries.into_iter().zip(leg_outcomes) {
+ let Some(outcome) = outcome else {
+ break;
+ };
+ let sub_result = outcome?;
any_timed_out |= sub_result.timed_out;
// #569/#594: this sub-list under-counts the fused total only when the
// leg genuinely has MORE than `per_query_topk` matches — signalled by
@@ -22160,6 +22324,7 @@ impl Index {
self.stored_slices_cache.clear();
self.decoded_stored_cache.clear();
self.fts_reader_cache.clear();
+ self.vector_column_cache.clear();
self.shortcut_count_cache.clear();
self.ghost_positions_cache.clear();
self.regexp_expand_cache.clear();
@@ -22181,6 +22346,7 @@ impl Index {
+ self.stored_slices_cache.len()
+ self.decoded_stored_cache.len()
+ self.fts_reader_cache.len()
+ + self.vector_column_cache.len()
+ self.shortcut_count_cache.len()
+ self.ghost_positions_cache.len()
+ self.regexp_expand_cache.len()
@@ -24815,6 +24981,9 @@ impl Index {
let fts_prefix = format!("{segment_id}\u{1}");
self.fts_reader_cache
.retain(|key, _| !key.starts_with(&fts_prefix));
+ // Keyed by "<segment_id>\u{1}<field>\u{1}<mode>" — same prefix.
+ self.vector_column_cache
+ .retain(|key, _| !key.starts_with(&fts_prefix));
let shadow_prefix = format!("{segment_id}\u{1}");
self.sort_shadow_cache
.retain(|key, _| !key.starts_with(&shadow_prefix));
@@ -34823,11 +34992,7 @@ fn raw_similarity_to_score(sim: &str, raw: f32) -> f32 {
fn compute_vector_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {
debug_assert_eq!(a.len(), b.len());
- let dot: f64 = a
- .iter()
- .zip(b.iter())
- .map(|(x, y)| (*x as f64) * (*y as f64))
- .sum();
+ let dot = crate::vector_column::dot_f64(a, b);
let result: f64 = match sim {
"l2_norm" => {
let sq: f64 = a
@@ -34848,25 +35013,74 @@ fn compute_vector_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {
dot + 1.0
}
}
- _ => {
- let na: f64 = a
- .iter()
- .map(|x| (*x as f64) * (*x as f64))
- .sum::<f64>()
- .sqrt();
- let nb: f64 = b
- .iter()
- .map(|x| (*x as f64) * (*x as f64))
- .sum::<f64>()
- .sqrt();
- let denom = na * nb;
- let cos = if denom > 0.0 { dot / denom } else { 0.0 };
- (1.0 + cos) / 2.0
- }
+ _ => cosine_score(
+ dot,
+ crate::vector_column::norm_f64(a),
+ crate::vector_column::norm_f64(b),
+ ),
};
result as f32
}
+/// The cosine arm of [`compute_vector_similarity`], given the dot product and
+/// both norms. Split out so a scan can supply norms it already has.
+#[inline]
+fn cosine_score(dot: f64, na: f64, nb: f64) -> f64 {
+ let denom = na * nb;
+ let cos = if denom > 0.0 { dot / denom } else { 0.0 };
+ (1.0 + cos) / 2.0
+}
+
+/// One query vector scored against many stored vectors (#939).
+///
+/// Bit-identical to calling [`compute_vector_similarity`] per vector — pinned
+/// by `query_scorer_is_bit_identical_to_compute_vector_similarity`. For cosine
+/// that function recomputes BOTH norms on every call; the query's is the same
+/// number every time and the document's is a property of the stored vector,
+/// so here the first is computed once per query and the second once per
+/// segment. Every value still comes out of the same expression
+/// (`vector_column::norm_f64`, `vector_column::dot_f64`, [`cosine_score`]),
+/// which is what makes the result exact rather than close. The other metrics
+/// have no norm to hoist and go through `compute_vector_similarity` unchanged.
+struct QueryScorer<'a> {
+ similarity: &'a str,
+ query: &'a [f32],
+ query_norm: f64,
+ cosine: bool,
+}
+
+impl<'a> QueryScorer<'a> {
+ fn new(similarity: &'a str, query: &'a [f32]) -> Self {
+ // Mirrors the `_` arm of `compute_vector_similarity`: anything that is
+ // not one of the three named metrics is cosine.
+ let cosine = !matches!(similarity, "l2_norm" | "dot_product" | "max_inner_product");
+ Self {
+ similarity,
+ query,
+ query_norm: if cosine {
+ crate::vector_column::norm_f64(query)
+ } else {
+ 0.0
+ },
+ cosine,
+ }
+ }
+
+ /// `vector.values.len()` must equal the query's; callers skip the rest.
+ #[inline]
+ fn score(&self, vector: crate::vector_column::StoredVector<'_>) -> f32 {
+ if self.cosine {
+ cosine_score(
+ crate::vector_column::dot_f64(self.query, vector.values),
+ self.query_norm,
+ vector.norm,
+ ) as f32
+ } else {
+ compute_vector_similarity(self.similarity, self.query, vector.values)
+ }
+ }
+}
+
/// Minimum live doc count before the HNSW query path may serve a kNN
/// request. Below this the exact brute-force scan is already fast and — more
/// importantly — every ES-YAML vector test (≤ a few hundred docs) keeps
@@ -35489,13 +35703,49 @@ fn knn_result_from_scored(
started: std::time::Instant,
generated_companion_fields: &HashSet<String>,
) -> SearchResult {
- scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
+ rank_knn_pool(&mut scored, k, |entry| entry.1);
+ knn_result_from_ranked(
+ index,
+ request,
+ vector_field,
+ scored,
+ started,
+ generated_companion_fields,
+ )
+}
+
+/// Rank a kNN candidate pool by score, descending, and cap it at `k` — the one
+/// definition of "top-k" every kNN executor shares.
+///
+/// The sort is STABLE and ties compare `Equal`, so documents with the same
+/// score keep the order they were collected in (memtable, then segments in
+/// snapshot order, then stored position). Generic over the entry so the exact
+/// scan can rank 16-byte `(candidate, score, ordinal)` tuples and clone a
+/// source only for what survives (#939); same comparator, so same order.
+fn rank_knn_pool<T>(pool: &mut Vec<T>, k: usize, score: impl Fn(&T) -> f32) {
+ pool.sort_by(|a, b| {
+ score(b)
+ .partial_cmp(&score(a))
+ .unwrap_or(std::cmp::Ordering::Equal)
+ });
// `k` bounds the kNN candidate pool; `from`/`size` then window into
// it, exactly like an ES top-level knn (returned hits are the
// size-slice of the top-k, NOT k itself). Without this, a
// `{"knn"/"semantic": {k: 5}}` with `"size": 3` returned 5 hits.
// `size == 0` is a count-only request (total only, no hits).
- scored.truncate(k.max(1));
+ pool.truncate(k.max(1));
+}
+
+/// Shape an already ranked, already `k`-capped pool (see [`rank_knn_pool`])
+/// into the response.
+fn knn_result_from_ranked(
+ index: &Index,
+ request: &SearchRequest,
+ vector_field: &str,
+ scored: Vec<(String, f32, Value, Option<u32>)>,
+ started: std::time::Instant,
+ generated_companion_fields: &HashSet<String>,
+) -> SearchResult {
// ES reports `hits.total.value` for a knn/semantic query as the size
// of the retrieved neighbor pool (min(k, matches)), NOT the number of
// docs that merely have a vector — so compute it AFTER the truncate.
@@ -35581,6 +35831,206 @@ fn knn_result_from_scored(
}
}
+/// Where a live kNN candidate sits — never a copy of the document (#939).
+#[derive(Clone, Copy, Debug)]
+enum KnnCandidateAt {
+ /// Slot in this query's memtable snapshot.
+ Memtable(usize),
+ /// `position` within the stored documents of `segment_views[view]`.
+ Segment { view: usize, position: usize },
+}
+
+/// One segment as the exact scan sees it: the shared parsed documents and the
+/// flat vectors derived from them, aligned by position.
+struct KnnSegmentView {
+ docs: Resident<Vec<Value>>,
+ column: Resident<crate::vector_column::SegmentVectorColumn>,
+}
+
+impl KnnCandidateAt {
+ /// The candidate's id and source, BORROWED — what the filter reads.
+ fn id_and_source<'a>(
+ &self,
+ mem_docs: &'a [(String, Arc<Value>)],
+ segment_views: &'a [KnnSegmentView],
+ ) -> (&'a str, &'a Value) {
+ match *self {
+ Self::Memtable(slot) => {
+ let (id, source) = &mem_docs[slot];
+ (id.as_str(), source.as_ref())
+ }
+ Self::Segment { view, position } => {
+ let doc = &segment_views[view].docs[position];
+ // A segment candidate is only ever created for a document
+ // with a string `_id`.
+ let id = doc.get("_id").and_then(Value::as_str).unwrap_or_default();
+ (id, crate::vector_column::stored_source_view(doc))
+ }
+ }
+ }
+
+ /// The candidate's id and an OWNED source — paid for returned hits only.
+ fn hydrate(
+ &self,
+ mem_docs: &[(String, Arc<Value>)],
+ segment_views: &[KnnSegmentView],
+ ) -> (String, Value) {
+ match *self {
+ Self::Memtable(slot) => {
+ let (id, source) = &mem_docs[slot];
+ (id.clone(), (**source).clone())
+ }
+ Self::Segment { view, position } => {
+ let doc = &segment_views[view].docs[position];
+ let id = doc
+ .get("_id")
+ .and_then(Value::as_str)
+ .unwrap_or_default()
+ .to_string();
+ // Reassembled segment docs have shape
+ // { "_id":..., "_seq_no":..., "_source": {...} }
+ // and a hit carries the `_source`; legacy pre-M7 segments
+ // without `_source` fall through to the wrapper minus `_id`.
+ let source = doc.get("_source").cloned().unwrap_or_else(|| {
+ let mut d = doc.clone();
+ if let Some(obj) = d.as_object_mut() {
+ obj.remove("_id");
+ }
+ d
+ });
+ (id, source)
+ }
+ }
+ }
+}
+
+/// How the exact kNN scan shows one document to its `filter` (#939).
+///
+/// `doc_matches_query_typed` reads `_id` out of the document it is handed, so
+/// the scan used to clone each candidate's WHOLE source — every passage vector
+/// included, ~1,900 JSON numbers a document on a multi-passage corpus — only
+/// to insert `_id` into the copy, and did so for every candidate whether or
+/// not the filter could ever match it. An `ids` filter naming 30 documents
+/// cost as much as no filter at all.
+enum KnnFilterPlan {
+ /// Every node of the filter reads the document ONLY through
+ /// `get_field_value(source, <a field it names>)` or `source.get("_id")`.
+ /// `get_field_value` looks at the source root under dotted prefixes of the
+ /// field name and nowhere else (its literal-key fast path, the
+ /// longest-dotted-key walk, and the multi-field parent fallback all do),
+ /// so the matcher cannot tell the source from a copy holding just those
+ /// root keys. It gets that copy: a few scalars instead of the vectors.
+ Projected(Vec<String>),
+ /// Anything else sees the whole source plus `_id`, exactly as before.
+ FullSource,
+}
+
+impl KnnFilterPlan {
+ fn for_filter(filter: &QueryNode) -> Self {
+ let mut root_keys = Vec::new();
+ if collect_filter_root_keys(filter, &mut root_keys) {
+ Self::Projected(root_keys)
+ } else {
+ Self::FullSource
+ }
+ }
+
+ fn matches(&self, filter: &QueryNode, id: &str, source: &Value, schema: &Schema) -> bool {
+ // A non-object source never had `_id` inserted — there is nowhere to
+ // put it — so the matcher always saw it as it is.
+ let Value::Object(fields) = source else {
+ return doc_matches_query_typed(filter, source, schema);
+ };
+ let mut view = match self {
+ Self::FullSource => fields.clone(),
+ Self::Projected(root_keys) => {
+ let mut view = serde_json::Map::new();
+ for key in root_keys {
+ if let Some(value) = fields.get(key) {
+ view.insert(key.clone(), value.clone());
+ }
+ }
+ view
+ }
+ };
+ view.insert("_id".to_string(), Value::String(id.to_string()));
+ doc_matches_query_typed(filter, &Value::Object(view), schema)
+ }
+}
+
+/// Collect the source-root keys `filter` can read into `out`; `false` when
+/// some node is not on the whitelist below and the filter must see the whole
+/// source.
+///
+/// WHITELIST DISCIPLINE: a node belongs here only if its arm in
+/// `doc_matches_query_typed` touches the document through
+/// `get_field_value(source, field)` / `source.get("_id")` and nothing else.
+/// Arms that iterate the source (`match` on a wildcard field, `multi_match`,
+/// `query_string`), hand it to a script (`script`, a scripted
+/// `minimum_should_match`), or walk nested objects are deliberately absent:
+/// declining costs a clone, admitting one wrongly changes which documents
+/// match. `knn_filter_projection_agrees_with_the_full_source` pins the
+/// equivalence for everything admitted.
+fn collect_filter_root_keys(filter: &QueryNode, out: &mut Vec<String>) -> bool {
+ match filter {
+ QueryNode::MatchAll | QueryNode::MatchNone | QueryNode::Ids { .. } => true,
+ QueryNode::Term { field, .. }
+ | QueryNode::Terms { field, .. }
+ | QueryNode::Range { field, .. }
+ | QueryNode::Prefix { field, .. }
+ | QueryNode::Exists { field } => push_field_root_keys(field, out),
+ QueryNode::Constant { query, .. }
+ | QueryNode::Boosted { query, .. }
+ | QueryNode::Named { query, .. } => collect_filter_root_keys(query, out),
+ QueryNode::Bool {
+ must,
+ should,
+ must_not,
+ filter,
+ minimum_should_match,
+ ..
+ } => {
+ match minimum_should_match {
+ // Runs Painless over the whole document.
+ Some(MinShouldMatch::Script { .. }) => return false,
+ // `terms_set`: the required count is read from this field.
+ Some(MinShouldMatch::Field(name)) => {
+ if !push_field_root_keys(name, out) {
+ return false;
+ }
+ }
+ _ => {}
+ }
+ must.iter()
+ .chain(should)
+ .chain(must_not)
+ .chain(filter)
+ .all(|clause| collect_filter_root_keys(clause, out))
+ }
+ _ => false,
+ }
+}
+
+/// Every dotted prefix of `field`, itself included — the only root keys
+/// `get_field_value(source, field)` can look up. `false` for a pattern.
+fn push_field_root_keys(field: &str, out: &mut Vec<String>) -> bool {
+ if field.contains('*') {
+ return false;
+ }
+ let mut push = |key: &str| {
+ if !out.iter().any(|existing| existing == key) {
+ out.push(key.to_string());
+ }
+ };
+ for (at, ch) in field.char_indices() {
+ if ch == '.' {
+ push(&field[..at]);
+ }
+ }
+ push(field);
+ true
+}
+
/// Extract `field` from `source` as an all-numbers f32 vector.
/// Returns `None` when the field is absent, not an array, empty, or holds
/// any non-number element.
@@ -52715,3 +53165,829 @@ mod pinned_knn_fts_892_tests {
);
}
}
+
+/// #939 — the exact kNN scan ranks addresses and hydrates only the winners.
+///
+/// The rewrite is meant to be invisible: same documents, same order, same
+/// scores to the bit. These tests pin that against `legacy_scan`, a transcript
+/// of the scan as it stood before the change (clone every live source, clone
+/// it again for the filter, re-walk the JSON numbers per passage), so the
+/// equivalence is checked in CI and not only by a one-off benchmark.
+#[cfg(test)]
+mod exact_scan_hydration_tests {
+ use super::*;
+ use crate::Engine;
+ use tempfile::TempDir;
+
+ const DIM: usize = 8;
+
+ fn engine(dir: &TempDir) -> Engine {
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ Engine::new(config).expect("engine")
+ }
+
+ /// `compute_vector_similarity` as it stood before #939, verbatim. The
+ /// oracle must not share code with what it checks.
+ fn legacy_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {
+ let dot: f64 = a
+ .iter()
+ .zip(b.iter())
+ .map(|(x, y)| (*x as f64) * (*y as f64))
+ .sum();
+ let result: f64 = match sim {
+ "l2_norm" => {
+ let sq: f64 = a
+ .iter()
+ .zip(b.iter())
+ .map(|(x, y)| {
+ let d = (*x as f64) - (*y as f64);
+ d * d
+ })
+ .sum();
+ 1.0 / (1.0 + sq)
+ }
+ "dot_product" => (1.0 + dot) / 2.0,
+ "max_inner_product" => {
+ if dot < 0.0 {
+ 1.0 / (1.0 - dot)
+ } else {
+ dot + 1.0
+ }
+ }
+ _ => {
+ let na: f64 = a
+ .iter()
+ .map(|x| (*x as f64) * (*x as f64))
+ .sum::<f64>()
+ .sqrt();
+ let nb: f64 = b
+ .iter()
+ .map(|x| (*x as f64) * (*x as f64))
+ .sum::<f64>()
+ .sqrt();
+ let denom = na * nb;
+ let cos = if denom > 0.0 { dot / denom } else { 0.0 };
+ (1.0 + cos) / 2.0
+ }
+ };
+ result as f32
+ }
+
+ fn json_vector(value: &Value) -> Option<Vec<f32>> {
+ match value {
+ Value::Array(a) => Some(
+ a.iter()
+ .filter_map(|v| v.as_f64().map(|f| f as f32))
+ .collect(),
+ ),
+ _ => None,
+ }
+ }
+
+ /// The exact scan as it stood before #939. Returns the ranked, `k`-capped
+ /// pool as `(id, score bits, passage ordinal, source)`.
+ #[allow(clippy::too_many_arguments)]
+ async fn legacy_scan(
+ idx: &Index,
+ field: &str,
+ query_vec: &[f32],
+ k: usize,
+ filter: Option<&QueryNode>,
+ similarity: &str,
+ boost: Option<f32>,
+ min_similarity: Option<f32>,
+ ) -> Vec<(String, u32, Option<u32>, Value)> {
+ let schema = idx.schema().await;
+ let mut candidates: Vec<(String, Value)> = idx.memtable.all_docs_with_sources();
+ let snap = idx.store.snapshot();
+ let mut seen: HashSet<String> = candidates.iter().map(|(id, _)| id.clone()).collect();
+ for meta in snap.segments.iter() {
+ let Some(docs) = idx.stored_values_for(&meta.id) else {
+ continue;
+ };
+ for doc in docs.iter() {
+ let Some(id) = doc.get("_id").and_then(Value::as_str).map(str::to_string) else {
+ continue;
+ };
+ if let Some(ver) = idx.store.version_map.get(&id) {
+ if ver.deleted {
+ continue;
+ }
+ if let Some(doc_seq) = doc.get("_seq_no").and_then(Value::as_u64) {
+ if doc_seq < ver.seq_no {
+ continue;
+ }
+ }
+ }
+ if !seen.insert(id.clone()) {
+ continue;
+ }
+ let src = doc.get("_source").cloned().unwrap_or_else(|| {
+ let mut d = doc.clone();
+ if let Some(obj) = d.as_object_mut() {
+ obj.remove("_id");
+ }
+ d
+ });
+ candidates.push((id, src));
+ }
+ }
+ let passes = |id: &str, src: &Value| -> bool {
+ let Some(f) = filter else {
+ return true;
+ };
+ let mut src_with_id = src.clone();
+ if let Some(obj) = src_with_id.as_object_mut() {
+ obj.insert("_id".to_string(), Value::String(id.to_string()));
+ }
+ doc_matches_query_typed(f, &src_with_id, &schema)
+ };
+ let use_sq8 = lookup_vector_quantization(&schema, field).as_deref() == Some("scalar8");
+ let mut scored: Vec<(String, f32, Value, Option<u32>)> = Vec::new();
+ if use_sq8 {
+ let normalize = !matches!(similarity, "l2_norm" | "dot_product" | "max_inner_product");
+ let dim = query_vec.len();
+ let mut cand: Vec<(String, Value, Vec<f32>)> = Vec::new();
+ for (id, src) in candidates {
+ if !passes(&id, &src) {
+ continue;
+ }
+ let Some(mut doc_vec) = get_field_value(&src, field).as_ref().and_then(json_vector)
+ else {
+ continue;
+ };
+ if doc_vec.len() != dim {
+ continue;
+ }
+ if normalize {
+ l2_normalize_vec(&mut doc_vec);
+ }
+ cand.push((id, src, doc_vec));
+ }
+ let params = Sq8Params::fit_borrowed(cand.iter().map(|(_, _, v)| v.as_slice()), dim);
+ let mut codes = vec![0u8; dim];
+ let mut decoded = vec![0.0f32; dim];
+ for (id, src, v) in cand {
+ params.encode_into(&v, &mut codes);
+ params.decode_into(&codes, &mut decoded);
+ scored.push((
+ id,
+ legacy_similarity(similarity, query_vec, &decoded),
+ src,
+ None,
+ ));
+ }
+ } else {
+ let chunk_field = format!("{field}_chunks");
+ for (id, src) in candidates {
+ if !passes(&id, &src) {
+ continue;
+ }
+ let (score, ordinal) = if let Some(Value::Array(chunks)) =
+ get_field_value(&src, &chunk_field)
+ {
+ let mut best: Option<(f32, u32)> = None;
+ for (chunk_position, cv) in chunks.iter().enumerate() {
+ let Some(dv) = json_vector(cv) else {
+ continue;
+ };
+ if dv.len() != query_vec.len() {
+ continue;
+ }
+ let s = legacy_similarity(similarity, query_vec, &dv);
+ if best.is_none_or(|(current, _)| s > current) {
+ best = Some((s, chunk_position as u32));
+ }
+ }
+ match best {
+ Some((score, ordinal)) => (score, Some(ordinal)),
+ None => continue,
+ }
+ } else {
+ let Some(doc_vec) = get_field_value(&src, field).as_ref().and_then(json_vector)
+ else {
+ continue;
+ };
+ if doc_vec.len() != query_vec.len() {
+ continue;
+ }
+ (legacy_similarity(similarity, query_vec, &doc_vec), Some(0))
+ };
+ scored.push((id, score, src, ordinal));
+ }
+ }
+ if let Some(raw) = min_similarity {
+ let cut = raw_similarity_to_score(similarity, raw);
+ scored.retain(|(_, score, _, _)| *score >= cut);
+ }
+ if let Some(b) = boost {
+ if (b - 1.0).abs() > f32::EPSILON {
+ for (_, score, _, _) in scored.iter_mut() {
+ *score *= b;
+ }
+ }
+ }
+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
+ scored.truncate(k.max(1));
+ scored
+ .into_iter()
+ .map(|(id, score, src, ordinal)| (id, score.to_bits(), ordinal, src))
+ .collect()
+ }
+
+ /// Deterministic pseudo-random unit-ish vectors; no `rand` dependency.
+ struct Lcg(u64);
+ impl Lcg {
+ fn next_f32(&mut self) -> f32 {
+ self.0 = self
+ .0
+ .wrapping_mul(6364136223846793005)
+ .wrapping_add(1442695040888963407);
+ (((self.0 >> 33) as f64) / ((1u64 << 31) as f64) - 1.0) as f32
+ }
+ fn vector(&mut self) -> Vec<f32> {
+ (0..DIM).map(|_| self.next_f32()).collect()
+ }
+ }
+
+ fn as_json(vector: &[f32]) -> Value {
+ Value::Array(vector.iter().map(|v| Value::from(*v as f64)).collect())
+ }
+
+ fn vector_schema(quantization: Option<&str>) -> Schema {
+ let mut schema = Schema::empty();
+ let mut body = FieldConfig::new("body", FieldType::Text);
+ body.options.dimensions = Some(DIM);
+ body.options.similarity = Some("cosine".into());
+ body.embedding = Some(xerj_common::types::EmbeddingConfig {
+ endpoint: None,
+ model: None,
+ target_field: Some("body_vector".into()),
+ });
+ schema.add_field(body).unwrap();
+ let mut companion = FieldConfig::new("body_vector", FieldType::Vector);
+ companion.options.dimensions = Some(DIM);
+ companion.options.similarity = Some("cosine".into());
+ companion.options.quantization = quantization.map(str::to_string);
+ schema.add_field(companion).unwrap();
+ schema
+ .add_field(FieldConfig::new("tag", FieldType::Keyword))
+ .unwrap();
+ schema
+ .add_field(FieldConfig::new("n", FieldType::Long))
+ .unwrap();
+ schema
+ .add_field(FieldConfig::new("title", FieldType::Text))
+ .unwrap();
+ schema
+ }
+
+ /// A document. Every third is multi-passage; a run of ids shares one
+ /// vector so the rank has exact score ties to break.
+ fn document(id: usize, rng: &mut Lcg, shared: &[f32]) -> Value {
+ let pooled = if (10..16).contains(&id) {
+ shared.to_vec()
+ } else {
+ rng.vector()
+ };
+ let mut doc = serde_json::json!({
+ "title": format!("doc {id} {}", if id % 4 == 0 { "cell" } else { "other" }),
+ "tag": format!("t{}", id % 5),
+ "n": (id * 37) % 1000,
+ "body": format!("body text {id}"),
+ "body_vector": as_json(&pooled),
+ });
+ if id % 3 == 0 {
+ let passages: Vec<Value> = (0..2 + id % 4).map(|_| as_json(&rng.vector())).collect();
+ doc["body_vector_chunks"] = Value::Array(passages);
+ }
+ doc
+ }
+
+ /// Two segments (the second superseding and deleting part of the first)
+ /// plus a live memtable, with every awkward document shape the scan has a
+ /// rule for.
+ async fn seed(idx: &Arc<Index>) {
+ let mut rng = Lcg(0x939);
+ let shared = rng.vector();
+ for id in 0..150 {
+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))
+ .await
+ .unwrap();
+ }
+ // Shapes with a rule of their own.
+ idx.index_document_prepared(
+ Some("malformed-passages".into()),
+ serde_json::json!({
+ "title": "cell malformed", "tag": "t1", "n": 5, "body": "x",
+ "body_vector": as_json(&rng.vector()),
+ // ordinal 1 is not a vector, ordinal 3 has the wrong length:
+ // both skipped, neither renumbers ordinal 2.
+ "body_vector_chunks": [as_json(&rng.vector()), "junk", as_json(&rng.vector()), [1.0, 2.0]],
+ }),
+ )
+ .await
+ .unwrap();
+ idx.index_document_prepared(
+ Some("empty-passages".into()),
+ serde_json::json!({
+ "title": "cell empty", "tag": "t1", "n": 6, "body": "x",
+ "body_vector": as_json(&shared),
+ "body_vector_chunks": [],
+ }),
+ )
+ .await
+ .unwrap();
+ idx.index_document_prepared(
+ Some("no-vector".into()),
+ serde_json::json!({"title": "cell none", "tag": "t1", "n": 7, "body": "x"}),
+ )
+ .await
+ .unwrap();
+ idx.flush().await.unwrap();
+
+ // Second segment: supersede some of the first, delete some, add more.
+ for id in (0..150).step_by(9) {
+ idx.index_document_prepared(
+ Some(format!("d{id}")),
+ document(id + 1000, &mut rng, &shared),
+ )
+ .await
+ .unwrap();
+ }
+ for id in (5..150).step_by(17) {
+ idx.delete_document(&format!("d{id}")).await.unwrap();
+ }
+ for id in 150..260 {
+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+
+ // Live memtable over both: updates of flushed documents and new ones.
+ for id in (3..260).step_by(31) {
+ idx.index_document_prepared(
+ Some(format!("d{id}")),
+ document(id + 2000, &mut rng, &shared),
+ )
+ .await
+ .unwrap();
+ }
+ for id in 260..290 {
+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))
+ .await
+ .unwrap();
+ }
+ assert!(
+ idx.store.snapshot().segments.len() >= 2,
+ "the fixture needs flushed segments beside the memtable"
+ );
+ }
+
+ fn filters() -> Vec<(&'static str, Option<QueryNode>)> {
+ let term = QueryNode::Term {
+ field: "tag".into(),
+ value: Value::from("t3"),
+ boost: None,
+ };
+ let range = QueryNode::Range {
+ field: "n".into(),
+ gte: Some(Value::from(100)),
+ gt: None,
+ lte: None,
+ lt: Some(Value::from(800)),
+ boost: None,
+ };
+ vec![
+ ("none", None),
+ (
+ "ids",
+ Some(QueryNode::Ids {
+ values: (0..290).step_by(7).map(|id| format!("d{id}")).collect(),
+ }),
+ ),
+ ("term", Some(term.clone())),
+ (
+ "bool(term AND range, NOT exists)",
+ Some(QueryNode::Bool {
+ must: Vec::new(),
+ should: Vec::new(),
+ filter: vec![term.clone(), range.clone()],
+ must_not: vec![QueryNode::Exists {
+ field: "body_vector_chunks".into(),
+ }],
+ minimum_should_match: None,
+ }),
+ ),
+ (
+ // Reads the VECTOR fields themselves: the projection must
+ // carry them rather than assume a filter never looks.
+ "exists(vector)",
+ Some(QueryNode::Exists {
+ field: "body_vector".into(),
+ }),
+ ),
+ (
+ // Not on the projection whitelist: takes the full-source path.
+ "match (full source)",
+ Some(QueryNode::Match {
+ field: "title".into(),
+ query: "cell".into(),
+ operator: xerj_query::ast::BoolOperator::Or,
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ }),
+ ),
+ ]
+ }
+
+ async fn assert_scan_matches_legacy(idx: &Arc<Index>, field: &str, sq8: bool) {
+ let mut rng = Lcg(0xBEEF);
+ let mut compared = 0usize;
+ let mut saw_later_passage = false;
+ let similarities: &[&str] = if sq8 {
+ &["cosine", "l2_norm"]
+ } else {
+ &["cosine", "dot_product", "l2_norm", "max_inner_product"]
+ };
+ for _ in 0..4 {
+ let query = rng.vector();
+ for similarity in similarities {
+ for (filter_name, filter) in filters() {
+ for (k, boost, min_similarity) in [
+ (1usize, None, None),
+ (10, Some(2.5f32), None),
+ (25, None, Some(0.1f32)),
+ (1000, None, None),
+ ] {
+ let expected = legacy_scan(
+ idx,
+ field,
+ &query,
+ k,
+ filter.as_ref(),
+ similarity,
+ boost,
+ min_similarity,
+ )
+ .await;
+ let request = SearchRequest {
+ size: k,
+ fields: vec![PASSAGE_RESPONSE_FIELD.into()],
+ ..SearchRequest::default()
+ };
+ let got = idx
+ .run_knn_brute_force(
+ &request,
+ field,
+ &query,
+ k,
+ filter.clone().map(Box::new),
+ similarity,
+ boost,
+ min_similarity,
+ )
+ .await
+ .unwrap();
+ let context = format!("similarity={similarity} filter={filter_name} k={k}");
+ assert_eq!(got.total.value, expected.len() as u64, "total: {context}");
+ assert_eq!(
+ got.hits
+ .iter()
+ .map(|h| (h.id.clone(), h.score.to_bits()))
+ .collect::<Vec<_>>(),
+ expected
+ .iter()
+ .map(|(id, bits, _, _)| (id.clone(), *bits))
+ .collect::<Vec<_>>(),
+ "ids, order and score BITS: {context}"
+ );
+ for (hit, (_, _, ordinal, source)) in got.hits.iter().zip(&expected) {
+ // Hydrated from the right document: the user
+ // fields are the legacy scan's, value for value.
+ for key in ["title", "tag", "n", "body"] {
+ assert_eq!(
+ hit.source.get(key),
+ source.get(key),
+ "{key}: {context}"
+ );
+ }
+ if let Some(ordinal) = ordinal {
+ saw_later_passage |= *ordinal > 0;
+ }
+ }
+ compared += got.hits.len();
+ }
+ }
+ }
+ }
+ assert!(
+ compared > 1000,
+ "the comparison must have teeth: {compared}"
+ );
+ if !sq8 {
+ assert!(
+ saw_later_passage,
+ "no query was won by a passage past the first"
+ );
+ }
+ }
+
+ #[tokio::test]
+ async fn exact_scan_is_bit_identical_to_the_clone_everything_scan() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("scan-939", vector_schema(None))
+ .unwrap();
+ let idx = engine.get_index("scan-939").unwrap();
+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;
+ seed(&idx).await;
+ // Cold (columns derived in the scan) and warm (columns cached) must
+ // agree with the oracle and therefore with each other.
+ assert_scan_matches_legacy(&idx, "body_vector", false).await;
+ assert!(
+ !idx.vector_column_cache.is_empty(),
+ "columns were never published"
+ );
+ assert_scan_matches_legacy(&idx, "body_vector", false).await;
+ }
+
+ #[tokio::test]
+ async fn scalar8_scan_is_bit_identical_to_the_clone_everything_scan() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("scan-939-sq8", vector_schema(Some("scalar8")))
+ .unwrap();
+ let idx = engine.get_index("scan-939-sq8").unwrap();
+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;
+ seed(&idx).await;
+ assert_scan_matches_legacy(&idx, "body_vector", true).await;
+ assert_scan_matches_legacy(&idx, "body_vector", true).await;
+ }
+
+ #[tokio::test]
+ async fn vector_columns_are_per_segment_budgeted_and_retired_by_merge() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("scan-939-cache", vector_schema(None))
+ .unwrap();
+ let idx = engine.get_index("scan-939-cache").unwrap();
+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;
+ seed(&idx).await;
+ let query = Lcg(7).vector();
+ let request = SearchRequest {
+ size: 10,
+ ..SearchRequest::default()
+ };
+ let scan = || {
+ idx.run_knn_brute_force(
+ &request,
+ "body_vector",
+ &query,
+ 10,
+ None,
+ "cosine",
+ None,
+ None,
+ )
+ };
+
+ let before = scan().await.unwrap();
+ let segments: Vec<String> = idx
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|m| m.id.clone())
+ .collect();
+ // One column per segment that HAS stored documents. A segment holding
+ // only tombstones has no stored section; the scan skips it, so there
+ // is nothing to derive a column from.
+ let mut with_stored = 0usize;
+ for id in &segments {
+ let has_stored = idx.stored_values_for(id).is_some();
+ with_stored += usize::from(has_stored);
+ assert_eq!(
+ idx.vector_column_cache
+ .contains_key(&format!("{id}\u{1}body_vector\u{1}c")),
+ has_stored,
+ "segment {id}: a column exists exactly when stored documents do"
+ );
+ }
+ assert!(with_stored >= 2, "the fixture needs several real segments");
+ assert_eq!(idx.vector_column_cache.len(), with_stored);
+ let charged = idx.segment_hydration_budget.snapshot().category_current
+ [SegmentCacheCategory::VectorColumn as usize];
+ assert!(
+ charged > 0,
+ "a published column must be charged to the hydration budget"
+ );
+
+ // A merge retires the segments; their columns must go with them, and
+ // the answer must not change.
+ idx.flush().await.unwrap();
+ idx.force_merge(1).await.unwrap();
+ let live: HashSet<String> = idx
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|m| m.id.clone())
+ .collect();
+ for entry in idx.vector_column_cache.iter() {
+ let segment = entry.key().split('\u{1}').next().unwrap();
+ assert!(
+ live.contains(segment),
+ "column outlived its segment: {}",
+ entry.key()
+ );
+ }
+ let after = scan().await.unwrap();
+ let rows = |r: &SearchResult| {
+ r.hits
+ .iter()
+ .map(|h| (h.id.clone(), h.score.to_bits()))
+ .collect::<Vec<_>>()
+ };
+ assert_eq!(
+ rows(&before),
+ rows(&after),
+ "a merge changed an exact answer"
+ );
+
+ idx.release_memory();
+ assert!(idx.vector_column_cache.is_empty());
+ assert_eq!(rows(&before), rows(&scan().await.unwrap()));
+ }
+
+ #[test]
+ fn query_scorer_is_bit_identical_to_compute_vector_similarity() {
+ let mut rng = Lcg(42);
+ let zero = vec![0.0f32; DIM];
+ for similarity in [
+ "cosine",
+ "dot_product",
+ "l2_norm",
+ "max_inner_product",
+ "anything-else",
+ ] {
+ for round in 0..500 {
+ let query = if round == 0 {
+ zero.clone()
+ } else {
+ rng.vector()
+ };
+ let stored = if round == 1 {
+ zero.clone()
+ } else {
+ rng.vector()
+ };
+ let scorer = QueryScorer::new(similarity, &query);
+ let got = scorer.score(crate::vector_column::StoredVector {
+ values: &stored,
+ ordinal: 0,
+ norm: crate::vector_column::norm_f64(&stored),
+ });
+ let legacy = legacy_similarity(similarity, &query, &stored);
+ assert_eq!(
+ got.to_bits(),
+ legacy.to_bits(),
+ "{similarity} round {round}"
+ );
+ assert_eq!(
+ compute_vector_similarity(similarity, &query, &stored).to_bits(),
+ legacy.to_bits(),
+ "refactored compute_vector_similarity drifted: {similarity} round {round}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn knn_filter_projection_agrees_with_the_full_source() {
+ let schema = vector_schema(None);
+ let docs = [
+ serde_json::json!({"tag": "t3", "n": 150, "a": {"b": {"c": 1}}, "a.b": {"c": 2}, "body_vector": [1.0], "_id": "shadowed"}),
+ serde_json::json!({"tag": ["t1", "t3"], "n": 900, "a.b.c": 3, "_routing": "r"}),
+ serde_json::json!({"tag": null, "n": "150", "a": [{"b": {"c": 4}}, {"b": {"c": 5}}]}),
+ serde_json::json!({"title": "cell", "count": 2}),
+ serde_json::json!({}),
+ Value::Null,
+ ];
+ let term = |field: &str, value: Value| QueryNode::Term {
+ field: field.into(),
+ value,
+ boost: None,
+ };
+ let exists = |field: &str| QueryNode::Exists {
+ field: field.into(),
+ };
+ let projected = vec![
+ QueryNode::MatchAll,
+ QueryNode::MatchNone,
+ QueryNode::Ids {
+ values: vec!["id-0".into(), "id-2".into()],
+ },
+ term("tag", Value::from("t3")),
+ term("_id", Value::from("id-1")),
+ term("a.b.c", Value::from(1)),
+ term("a.b.c", Value::from(3)),
+ term("a.b.c", Value::from(4)),
+ term("tag.keyword", Value::from("t3")),
+ exists("a.b"),
+ exists("a.b.c"),
+ exists("_routing"),
+ exists("_id"),
+ exists("body_vector"),
+ exists("missing"),
+ QueryNode::Terms {
+ field: "tag".into(),
+ values: vec![Value::from("t1"), Value::from("zz")],
+ boost: None,
+ },
+ QueryNode::Range {
+ field: "n".into(),
+ gte: Some(Value::from(100)),
+ gt: None,
+ lte: None,
+ lt: Some(Value::from(800)),
+ boost: None,
+ },
+ QueryNode::Prefix {
+ field: "tag".into(),
+ value: "t".into(),
+ boost: None,
+ constant_score: false,
+ },
+ QueryNode::Bool {
+ must: vec![exists("tag")],
+ should: vec![term("tag", Value::from("t3")), term("n", Value::from(900))],
+ filter: Vec::new(),
+ must_not: vec![term("_id", Value::from("id-0"))],
+ minimum_should_match: Some(MinShouldMatch::Field("count".into())),
+ },
+ QueryNode::Constant {
+ score: 1.0,
+ query: Box::new(term("tag", Value::from("t3"))),
+ },
+ ];
+ for filter in &projected {
+ let plan = KnnFilterPlan::for_filter(filter);
+ assert!(
+ matches!(plan, KnnFilterPlan::Projected(_)),
+ "should project: {filter:?}"
+ );
+ for (n, doc) in docs.iter().enumerate() {
+ let id = format!("id-{n}");
+ assert_eq!(
+ plan.matches(filter, &id, doc, &schema),
+ KnnFilterPlan::FullSource.matches(filter, &id, doc, &schema),
+ "projection changed the answer: {filter:?} on {doc}"
+ );
+ }
+ }
+ // Declined: these read the document some other way.
+ let declined = vec![
+ term("ta*", Value::from("t3")),
+ QueryNode::Script {
+ source: "true".into(),
+ params: None,
+ },
+ QueryNode::Bool {
+ must: Vec::new(),
+ should: vec![term("tag", Value::from("t3"))],
+ filter: Vec::new(),
+ must_not: Vec::new(),
+ minimum_should_match: Some(MinShouldMatch::Script {
+ source: "1".into(),
+ params: None,
+ }),
+ },
+ ];
+ for filter in &declined {
+ assert!(
+ matches!(KnnFilterPlan::for_filter(filter), KnnFilterPlan::FullSource),
+ "must not project: {filter:?}"
+ );
+ }
+ }
+
+ /// The full-source plan IS the old behaviour: the whole source with `_id`
+ /// inserted over whatever the document carried under that key.
+ #[test]
+ fn full_source_plan_inserts_the_id_like_the_old_scan_did() {
+ let schema = vector_schema(None);
+ let filter = QueryNode::Ids {
+ values: vec!["real".into()],
+ };
+ let doc = serde_json::json!({"_id": "shadowed", "tag": "t"});
+ assert!(KnnFilterPlan::FullSource.matches(&filter, "real", &doc, &schema));
+ assert!(!KnnFilterPlan::FullSource.matches(&filter, "other", &doc, &schema));
+ }
+}
diff --git a/engine/crates/xerj-engine/src/ingest_memory.rs b/engine/crates/xerj-engine/src/ingest_memory.rs
index ea58c6374..64ba4e522 100644
--- a/engine/crates/xerj-engine/src/ingest_memory.rs
+++ b/engine/crates/xerj-engine/src/ingest_memory.rs
@@ -121,10 +121,11 @@ pub enum Category {
CacheRowSequences,
CacheDecodedStored,
CacheFtsReader,
+ CacheVectorColumn,
}
impl Category {
- pub const ALL: [Self; 22] = [
+ pub const ALL: [Self; 23] = [
Self::HttpBody,
Self::HttpRewriteBuffer,
Self::RawSource,
@@ -147,6 +148,7 @@ impl Category {
Self::CacheRowSequences,
Self::CacheDecodedStored,
Self::CacheFtsReader,
+ Self::CacheVectorColumn,
];
const COUNT: usize = Self::ALL.len();
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 20e7e3ca6..3a9c42afd 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -38,6 +38,7 @@ pub mod segment_cache_estimates;
pub mod slow_query;
pub mod sql;
pub mod turbo_ingest;
+mod vector_column;
pub mod wal_tap;
mod write_publication;
diff --git a/engine/crates/xerj-engine/src/segment_cache_budget.rs b/engine/crates/xerj-engine/src/segment_cache_budget.rs
index 4d938872c..b2e06e719 100644
--- a/engine/crates/xerj-engine/src/segment_cache_budget.rs
+++ b/engine/crates/xerj-engine/src/segment_cache_budget.rs
@@ -10,7 +10,7 @@ use std::sync::Arc;
use crate::ingest_memory::{Category, Retained};
-pub const CATEGORY_COUNT: usize = 8;
+pub const CATEGORY_COUNT: usize = 9;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[repr(usize)]
@@ -24,6 +24,9 @@ pub enum SegmentCacheCategory {
DecodedStored,
/// Decompressed `.post` + `.meta` blobs behind a cached `FtsIndexReader`.
FtsReader,
+ /// Flat `f32` vectors of one field over one segment, derived from its
+ /// stored documents for the exact kNN scan (#939).
+ VectorColumn,
}
impl SegmentCacheCategory {
@@ -36,6 +39,7 @@ impl SegmentCacheCategory {
Self::RowSequences,
Self::DecodedStored,
Self::FtsReader,
+ Self::VectorColumn,
];
fn trace_category(self) -> Category {
@@ -48,6 +52,7 @@ impl SegmentCacheCategory {
Self::RowSequences => Category::CacheRowSequences,
Self::DecodedStored => Category::CacheDecodedStored,
Self::FtsReader => Category::CacheFtsReader,
+ Self::VectorColumn => Category::CacheVectorColumn,
}
}
}
diff --git a/engine/crates/xerj-engine/src/vector_column.rs b/engine/crates/xerj-engine/src/vector_column.rs
new file mode 100644
index 000000000..b99d3cdd9
--- /dev/null
+++ b/engine/crates/xerj-engine/src/vector_column.rs
@@ -0,0 +1,446 @@
+//! Flat `f32` view of one vector field over one immutable segment (#939).
+//!
+//! The exact kNN scan used to deep-clone every stored `_source` — vectors
+//! included, as `serde_json::Value::Number`s — into its candidate list, clone
+//! it a second time to evaluate the filter, clone the passage-vector array a
+//! third time through `get_field_value`, and then re-walk those JSON numbers
+//! into a fresh `Vec<f32>` per passage. All of that ran per document, per
+//! query, and none of it was arithmetic: on 5,183 multi-passage documents it
+//! was ~400 ms around ~14 ms of model time.
+//!
+//! This module holds the vectors once, as plain `f32`s, per segment. Segments
+//! are immutable, so the column is derived exactly once and then shared; which
+//! documents are *live* is still decided per query, by the caller.
+//!
+//! # Reference designs (approach adapted, no code copied)
+//!
+//! * **Flattened multi-vector + `dim`.** qdrant stores a multi-vector as one
+//! contiguous buffer and slices it per sub-vector
+//! (`lib/segment/src/data_types/vectors.rs:271`, max-sim at
+//! `lib/segment/src/vector_storage/query_scorer/mod.rs:77`; Apache-2.0,
+//! 74f3e85). Ours differs where XERJ's contract differs: one query vector,
+//! best passage wins, lowest ordinal on an exact tie, and passages may be
+//! skipped without renumbering the ones after them.
+//! * **Score an address, hydrate the winners.** qdrant ranks
+//! `ScoredPointOffset { idx, score }` and only then retrieves payloads
+//! (`lib/segment/src/segment/read_view/search.rs:222`); tantivy ranks
+//! `DocAddress { segment_ord, doc_id }` and fetches stored documents by
+//! address (`src/lib.rs:338`, `src/core/searcher.rs:88`; MIT, 3a55bc9).
+//!
+//! # Exactness contract
+//!
+//! Scores produced from this column are **bit-identical** to the scan it
+//! replaces. That is a property of construction, not of tolerance:
+//!
+//! * elements are converted with the same expression the scan used
+//! (`as_f64()` then `as f32`, non-numbers dropped);
+//! * a vector whose converted length differs from the query's is skipped at
+//! query time, exactly as before, and keeps its original passage ordinal;
+//! * a document whose `<field>_chunks` is an array with no usable passage is
+//! skipped — it does **not** fall back to the pooled vector, because the old
+//! scan did not;
+//! * the dot product and the norms are the same strictly sequential `f64`
+//! reductions ([`dot_f64`], [`norm_f64`]), shared with
+//! `compute_vector_similarity` so the two cannot drift. Precomputing a
+//! document vector's norm stores the value the scan would have recomputed.
+
+use serde_json::Value;
+use std::borrow::Cow;
+
+use crate::index::get_field_value;
+
+/// Which stored vectors a column holds for each document.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub(crate) enum ColumnMode {
+ /// `<field>_chunks` passage vectors when that companion is an array, the
+ /// pooled `<field>` vector otherwise. The default exact scan.
+ BestPassage,
+ /// Always the pooled `<field>` vector. The `scalar8` scan reads only that.
+ PooledOnly,
+}
+
+impl ColumnMode {
+ /// Distinguishes the two shapes in a cache key.
+ pub(crate) fn key_tag(self) -> char {
+ match self {
+ Self::BestPassage => 'c',
+ Self::PooledOnly => 'p',
+ }
+ }
+}
+
+/// One stored vector: where it lives in the flat buffer, which passage it was,
+/// and its Euclidean norm.
+#[derive(Clone, Copy, Debug)]
+struct VectorEntry {
+ start: usize,
+ len: usize,
+ ordinal: u32,
+ norm: f64,
+}
+
+/// The vectors one document contributes.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum DocVectors {
+ /// Nothing usable: the scan skips the document.
+ Absent,
+ /// The pooled vector, reported as passage ordinal 0.
+ Pooled(usize),
+ /// `count` passage vectors starting at entry `first`. `count == 0` is a
+ /// passage array with no usable element, which also skips the document.
+ Chunks { first: usize, count: usize },
+}
+
+/// A borrowed stored vector handed to the scorer.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct StoredVector<'a> {
+ pub(crate) values: &'a [f32],
+ pub(crate) ordinal: u32,
+ pub(crate) norm: f64,
+}
+
+/// What a document offers the scorer.
+pub(crate) enum DocVectorsView<'a> {
+ Absent,
+ Pooled(StoredVector<'a>),
+ Chunks(ChunkIter<'a>),
+}
+
+pub(crate) struct ChunkIter<'a> {
+ column: &'a SegmentVectorColumn,
+ next: usize,
+ end: usize,
+}
+
+impl<'a> Iterator for ChunkIter<'a> {
+ type Item = StoredVector<'a>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.next >= self.end {
+ return None;
+ }
+ let vector = self.column.stored(self.next);
+ self.next += 1;
+ Some(vector)
+ }
+}
+
+/// Flat vectors for one field over a run of documents, addressed by position.
+#[derive(Debug, Default)]
+pub(crate) struct SegmentVectorColumn {
+ data: Vec<f32>,
+ entries: Vec<VectorEntry>,
+ docs: Vec<DocVectors>,
+}
+
+impl SegmentVectorColumn {
+ /// Number of document positions covered. A column abandoned part-way (the
+ /// request's deadline passed) covers a prefix of its segment.
+ #[cfg(test)]
+ pub(crate) fn len(&self) -> usize {
+ self.docs.len()
+ }
+
+ fn stored(&self, entry: usize) -> StoredVector<'_> {
+ let e = &self.entries[entry];
+ StoredVector {
+ values: &self.data[e.start..e.start + e.len],
+ ordinal: e.ordinal,
+ norm: e.norm,
+ }
+ }
+
+ /// The vectors of the document at `position`; `Absent` past the end.
+ pub(crate) fn doc(&self, position: usize) -> DocVectorsView<'_> {
+ match self.docs.get(position) {
+ None | Some(DocVectors::Absent) => DocVectorsView::Absent,
+ Some(DocVectors::Pooled(entry)) => DocVectorsView::Pooled(self.stored(*entry)),
+ Some(DocVectors::Chunks { first, count }) => DocVectorsView::Chunks(ChunkIter {
+ column: self,
+ next: *first,
+ end: first + count,
+ }),
+ }
+ }
+
+ /// Bytes this column keeps alive, for the segment hydration budget.
+ pub(crate) fn retained_bytes(&self) -> u64 {
+ let bytes = self.data.capacity() * std::mem::size_of::<f32>()
+ + self.entries.capacity() * std::mem::size_of::<VectorEntry>()
+ + self.docs.capacity() * std::mem::size_of::<DocVectors>()
+ + std::mem::size_of::<Self>();
+ u64::try_from(bytes).unwrap_or(u64::MAX)
+ }
+}
+
+/// Builds a [`SegmentVectorColumn`] one document at a time, in position order.
+pub(crate) struct ColumnBuilder {
+ field: String,
+ chunk_field: String,
+ mode: ColumnMode,
+ column: SegmentVectorColumn,
+}
+
+impl ColumnBuilder {
+ pub(crate) fn new(field: &str, mode: ColumnMode) -> Self {
+ Self {
+ field: field.to_string(),
+ chunk_field: format!("{field}_chunks"),
+ mode,
+ column: SegmentVectorColumn::default(),
+ }
+ }
+
+ /// Forget every document but keep the allocations — the memtable path
+ /// reuses one builder as per-document scratch.
+ pub(crate) fn clear(&mut self) {
+ self.column.data.clear();
+ self.column.entries.clear();
+ self.column.docs.clear();
+ }
+
+ /// The column built so far.
+ pub(crate) fn column(&self) -> &SegmentVectorColumn {
+ &self.column
+ }
+
+ pub(crate) fn finish(mut self) -> SegmentVectorColumn {
+ self.column.data.shrink_to_fit();
+ self.column.entries.shrink_to_fit();
+ self.column.docs.shrink_to_fit();
+ self.column
+ }
+
+ /// Append a segment's stored document: `{"_id", "_seq_no", "_source"}`,
+ /// or a legacy pre-`_source` document whose fields sit at the top level.
+ pub(crate) fn push_stored_doc(&mut self, stored: &Value) {
+ self.push_source(stored_source_view(stored));
+ }
+
+ /// Append a document given its source object.
+ pub(crate) fn push_source(&mut self, source: &Value) {
+ let doc = match self.mode {
+ ColumnMode::PooledOnly => self.push_pooled(source),
+ ColumnMode::BestPassage => match field_value_cow(source, &self.chunk_field) {
+ Some(chunks) => match chunks.as_ref() {
+ Value::Array(chunks) => self.push_chunks(chunks),
+ _ => self.push_pooled(source),
+ },
+ None => self.push_pooled(source),
+ },
+ };
+ self.column.docs.push(doc);
+ }
+
+ fn push_chunks(&mut self, chunks: &[Value]) -> DocVectors {
+ let first = self.column.entries.len();
+ for (position, chunk) in chunks.iter().enumerate() {
+ let Value::Array(elements) = chunk else {
+ continue;
+ };
+ // The ordinal is the passage's position in the ORIGINAL array, so
+ // a skipped passage never renumbers the ones after it.
+ let Ok(ordinal) = u32::try_from(position) else {
+ continue;
+ };
+ self.push_vector(elements, ordinal);
+ }
+ DocVectors::Chunks {
+ first,
+ count: self.column.entries.len() - first,
+ }
+ }
+
+ fn push_pooled(&mut self, source: &Value) -> DocVectors {
+ match field_value_cow(source, &self.field).as_deref() {
+ Some(Value::Array(elements)) => DocVectors::Pooled(self.push_vector(elements, 0)),
+ _ => DocVectors::Absent,
+ }
+ }
+
+ fn push_vector(&mut self, elements: &[Value], ordinal: u32) -> usize {
+ let start = self.column.data.len();
+ self.column
+ .data
+ .extend(elements.iter().filter_map(|v| v.as_f64().map(|f| f as f32)));
+ let len = self.column.data.len() - start;
+ let norm = norm_f64(&self.column.data[start..]);
+ self.column.entries.push(VectorEntry {
+ start,
+ len,
+ ordinal,
+ norm,
+ });
+ self.column.entries.len() - 1
+ }
+}
+
+/// The source object of a segment's stored document, without cloning it.
+///
+/// Reassembled segment documents are `{"_id", "_seq_no", "_source": {...}}`;
+/// legacy pre-M7 segments keep the fields at the top level, beside `_id`.
+pub(crate) fn stored_source_view(stored: &Value) -> &Value {
+ stored.get("_source").unwrap_or(stored)
+}
+
+/// [`get_field_value`] without the clone when the field is a literal key at
+/// the source root — which is where a vector field and its generated
+/// companions live. Anything else (a dotted path, an array of objects, the
+/// multi-field fallback) is answered by `get_field_value` itself, so the two
+/// resolve identically by construction.
+fn field_value_cow<'a>(source: &'a Value, field: &str) -> Option<Cow<'a, Value>> {
+ if let Value::Object(map) = source {
+ if let Some(value) = map.get(field) {
+ return Some(Cow::Borrowed(value));
+ }
+ }
+ get_field_value(source, field).map(Cow::Owned)
+}
+
+/// Strictly sequential `f64` dot product. Shared with
+/// `compute_vector_similarity`; do not reorder or widen this reduction — a
+/// different summation order changes the low bits of every score.
+#[inline]
+pub(crate) fn dot_f64(a: &[f32], b: &[f32]) -> f64 {
+ a.iter()
+ .zip(b.iter())
+ .map(|(x, y)| (*x as f64) * (*y as f64))
+ .sum()
+}
+
+/// Strictly sequential `f64` Euclidean norm. Same warning as [`dot_f64`].
+#[inline]
+pub(crate) fn norm_f64(v: &[f32]) -> f64 {
+ v.iter()
+ .map(|x| (*x as f64) * (*x as f64))
+ .sum::<f64>()
+ .sqrt()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn views(column: &SegmentVectorColumn, position: usize) -> Vec<(u32, Vec<f32>)> {
+ match column.doc(position) {
+ DocVectorsView::Absent => Vec::new(),
+ DocVectorsView::Pooled(v) => vec![(v.ordinal, v.values.to_vec())],
+ DocVectorsView::Chunks(iter) => iter.map(|v| (v.ordinal, v.values.to_vec())).collect(),
+ }
+ }
+
+ #[test]
+ fn best_passage_prefers_the_chunk_companion_over_the_pooled_vector() {
+ let mut b = ColumnBuilder::new("body_vector", ColumnMode::BestPassage);
+ b.push_stored_doc(&json!({"_id": "a", "_source": {
+ "body_vector": [9.0, 9.0],
+ "body_vector_chunks": [[1.0, 0.0], [0.0, 1.0]],
+ }}));
+ let column = b.finish();
+ assert_eq!(
+ views(&column, 0),
+ vec![(0, vec![1.0, 0.0]), (1, vec![0.0, 1.0])]
+ );
+ }
+
+ #[test]
+ fn skipped_passages_keep_the_ordinals_of_the_ones_after_them() {
+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);
+ b.push_source(&json!({"v_chunks": [[1.0, 0.0], "not a vector", [0.0, 1.0]]}));
+ let column = b.finish();
+ assert_eq!(
+ views(&column, 0),
+ vec![(0, vec![1.0, 0.0]), (2, vec![0.0, 1.0])]
+ );
+ }
+
+ #[test]
+ fn an_empty_passage_array_skips_the_document_instead_of_falling_back() {
+ // The scan this replaces `continue`d here; falling back to the pooled
+ // vector would surface a document it never returned.
+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);
+ b.push_source(&json!({"v": [1.0, 0.0], "v_chunks": ["x", 3]}));
+ b.push_source(&json!({"v": [1.0, 0.0], "v_chunks": []}));
+ let column = b.finish();
+ assert!(views(&column, 0).is_empty());
+ assert!(views(&column, 1).is_empty());
+ }
+
+ #[test]
+ fn a_non_array_chunk_companion_falls_back_to_the_pooled_vector() {
+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);
+ b.push_source(&json!({"v": [0.5, 0.25], "v_chunks": "nope"}));
+ let column = b.finish();
+ assert_eq!(views(&column, 0), vec![(0, vec![0.5, 0.25])]);
+ }
+
+ #[test]
+ fn pooled_only_ignores_the_chunk_companion() {
+ let mut b = ColumnBuilder::new("v", ColumnMode::PooledOnly);
+ b.push_source(&json!({"v": [0.5, 0.25], "v_chunks": [[1.0, 0.0]]}));
+ let column = b.finish();
+ assert_eq!(views(&column, 0), vec![(0, vec![0.5, 0.25])]);
+ }
+
+ #[test]
+ fn non_numeric_elements_are_dropped_like_the_scan_dropped_them() {
+ // `filter_map(as_f64)` shortens the vector; the length check against
+ // the query then rejects it at query time, as it always did.
+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);
+ b.push_source(&json!({"v": [1.0, "x", 2.0]}));
+ let column = b.finish();
+ assert_eq!(views(&column, 0), vec![(0, vec![1.0, 2.0])]);
+ }
+
+ #[test]
+ fn missing_and_non_array_vectors_are_absent_and_positions_stay_aligned() {
+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);
+ b.push_source(&json!({"other": 1}));
+ b.push_source(&json!({"v": "text"}));
+ b.push_source(&json!({"v": [3.0]}));
+ let column = b.finish();
+ assert_eq!(column.len(), 3);
+ assert!(views(&column, 0).is_empty());
+ assert!(views(&column, 1).is_empty());
+ assert_eq!(views(&column, 2), vec![(0, vec![3.0])]);
+ assert!(views(&column, 99).is_empty(), "past the end is absent");
+ }
+
+ #[test]
+ fn legacy_documents_without_source_are_read_at_the_top_level() {
+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);
+ b.push_stored_doc(&json!({"_id": "legacy", "v": [1.0, 2.0]}));
+ let column = b.finish();
+ assert_eq!(views(&column, 0), vec![(0, vec![1.0, 2.0])]);
+ }
+
+ #[test]
+ fn dotted_vector_fields_resolve_through_get_field_value() {
+ let mut b = ColumnBuilder::new("emb.v", ColumnMode::BestPassage);
+ b.push_source(&json!({"emb": {"v": [1.0, 2.0]}}));
+ let column = b.finish();
+ assert_eq!(views(&column, 0), vec![(0, vec![1.0, 2.0])]);
+ }
+
+ #[test]
+ fn stored_norm_is_the_value_the_scan_would_recompute() {
+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);
+ b.push_source(&json!({"v": [0.1, -0.7, 0.33, 1e-3]}));
+ let column = b.finish();
+ let DocVectorsView::Pooled(v) = column.doc(0) else {
+ panic!("expected a pooled vector");
+ };
+ assert_eq!(v.norm.to_bits(), norm_f64(v.values).to_bits());
+ }
+
+ #[test]
+ fn clear_reuses_the_builder_as_scratch() {
+ let mut b = ColumnBuilder::new("v", ColumnMode::BestPassage);
+ b.push_source(&json!({"v": [1.0]}));
+ b.clear();
+ b.push_source(&json!({"v": [2.0]}));
+ assert_eq!(b.column().len(), 1);
+ assert_eq!(views(b.column(), 0), vec![(0, vec![2.0])]);
+ }
+}