AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
docs(recipes): real scalar8 quantization use case + harden run_all teardown
xerj-org5468a5cdbec5
diff --git a/docs/examples/vector-quantization/quant_demo.py b/docs/examples/vector-quantization/quant_demo.pynew file mode 100644index 000000000..256cf6085--- /dev/null+++ b/docs/examples/vector-quantization/quant_demo.py@@ -0,0 +1,148 @@+#!/usr/bin/env python3+"""Scalar8 (int8) vector quantization on a real corpus — zero dependencies.++XERJ stores dense vectors at full float32 precision by default. Opting a+`dense_vector` field into scalar8 quantization (`index_options.type:+int8_hnsw`) makes the kNN *serving* path score against 1-byte-per-dimension+codes instead of 4-byte floats — a ~4x smaller vector working set with+almost no recall loss. `_source` still returns the original vectors.++This recipe embeds the 40 real KB articles (demo/data/ai_kb.ndjson) into+128-dim vectors with a small deterministic feature-hasher (same idea as+XERJ's built-in lexical embedder), then indexes the SAME vectors into two+indices — one full-precision (`none`), one quantized (`scalar8`) — and+shows that:++ 1. kNN returns the same top results from both,+ 2. recall@10 of the quantized index vs the exact index stays >= 0.90,+ 3. the quantized field's vector footprint is 4x smaller (1 vs 4 bytes/dim).++Usage:+ xerj --insecure --data-dir ./data & # start XERJ+ python3 recipes/vector_quantization.py++ XERJ_URL (default http://localhost:9200)+"""++import hashlib+import json+import math+import os+import pathlib+import urllib.error+import urllib.request++XERJ = os.environ.get("XERJ_URL", "http://localhost:9200")+KB = pathlib.Path(__file__).resolve().parent.parent / "demo" / "data" / "ai_kb.ndjson"+DIM = 128+NONE_INDEX = "vq-none"+SQ8_INDEX = "vq-scalar8"+++def call(method, path, body=None):+ """One tiny HTTP helper instead of a client library."""+ data = None+ headers = {}+ if body is not None:+ data = body.encode() if isinstance(body, str) else json.dumps(body).encode()+ headers["Content-Type"] = (+ "application/x-ndjson" if isinstance(body, str) else "application/json"+ )+ req = urllib.request.Request(f"{XERJ}{path}", data=data, headers=headers, method=method)+ with urllib.request.urlopen(req) as resp:+ return json.loads(resp.read())+++def _h(kind, token):+ """Stable (seed-independent) hash → 64-bit int, via md5."""+ return int.from_bytes(hashlib.md5(f"{kind}:{token}".encode()).digest()[:8], "big")+++def embed(text, dim=DIM):+ """Deterministic feature-hashing embedder: word unigrams + char trigrams+ hashed into `dim` buckets with signed contributions, then L2-normalised.+ Real, reproducible, zero-dependency — the same family XERJ uses built-in."""+ vec = [0.0] * dim+ toks = "".join(c.lower() if c.isalnum() else " " for c in text).split()+ for w in toks:+ h = _h("w", w)+ vec[h % dim] += 1.0 if (h >> 63) & 1 else -1.0+ padded = f"#{w}#"+ for i in range(len(padded) - 2):+ t = padded[i : i + 3]+ ht = _h("t", t)+ vec[ht % dim] += 0.35 if (ht >> 63) & 1 else -0.35+ norm = math.sqrt(sum(x * x for x in vec)) or 1.0+ return [x / norm for x in vec]+++def make_index(name, quantized):+ try:+ call("DELETE", f"/{name}")+ except urllib.error.HTTPError:+ pass+ field = {"type": "dense_vector", "dims": DIM, "similarity": "cosine"}+ if quantized:+ field["index_options"] = {"type": "int8_hnsw"} # opt in to scalar8+ call("PUT", f"/{name}", {+ "mappings": {"properties": {"title": {"type": "text"}, "v": field}}+ })+++def bulk_load(name, docs):+ lines = []+ for i, d in enumerate(docs):+ lines.append(json.dumps({"index": {"_index": name, "_id": str(i)}}))+ lines.append(json.dumps({"title": d["title"], "v": d["v"]}))+ call("POST", "/_bulk?refresh=true", "\n".join(lines) + "\n")+++def knn(name, qv, k=10):+ r = call("POST", f"/{name}/_search", {"knn": {"field": "v", "query_vector": qv, "k": k}})+ return [(h["_id"], h["_score"]) for h in r["hits"]["hits"]]+++def main():+ # ── 1. Embed the real KB into 128-dim vectors. ───────────────────────+ docs = []+ for raw in KB.read_text().splitlines():+ a = json.loads(raw)+ docs.append({"title": a["title"], "v": embed(a["title"] + " " + a["content"])})+ print(f"embedded {len(docs)} real KB articles into {DIM}-dim vectors\n")++ # ── 2. Index the SAME vectors two ways. ──────────────────────────────+ make_index(NONE_INDEX, quantized=False)+ make_index(SQ8_INDEX, quantized=True)+ bulk_load(NONE_INDEX, docs)+ bulk_load(SQ8_INDEX, docs)+ print(f"indexed into `{NONE_INDEX}` (float32) and `{SQ8_INDEX}` (int8_hnsw / scalar8)\n")++ # ── 3. Same query, both indices — exact vs quantized scoring. ────────+ q = embed("how do I stop an agent's context window from overflowing?")+ print("query: 'how do I stop an agent's context window from overflowing?'\n")+ for name in (NONE_INDEX, SQ8_INDEX):+ top = knn(name, q, k=3)+ tag = "float32 (exact)" if name == NONE_INDEX else "scalar8 (quantized)"+ print(f"── {tag}")+ for _id, score in top:+ print(f" {score:.5f} {docs[int(_id)]['title']}")+ print()++ # ── 4. recall@10 of quantized vs exact over the whole corpus. ────────+ hits = 0+ total = 0+ for d in docs:+ exact = {i for i, _ in knn(NONE_INDEX, d["v"], k=10)}+ approx = {i for i, _ in knn(SQ8_INDEX, d["v"], k=10)}+ hits += len(exact & approx)+ total += len(exact)+ recall = hits / total if total else 0.0+ print(f"recall@10 (scalar8 vs float32 ground truth): {recall:.3f}")+ print(f"vector footprint: float32 = {DIM * 4} B/vec → scalar8 = {DIM} B/vec (4x smaller)")+ if recall < 0.90:+ raise SystemExit(f"FAIL: recall {recall:.3f} < 0.90")+ print("\nOK — 4x smaller vectors, recall preserved. `_source` still holds the originals.")+++if __name__ == "__main__":+ main()diff --git a/docs/recipes/README.md b/docs/recipes/README.mdindex 4104dae68..636329af8 100644--- a/docs/recipes/README.md+++ b/docs/recipes/README.md@@ -10,6 +10,7 @@ These are practical "how do I actually do X" recipes, not an API reference (that| [Semantic search & RAG](./semantic-search-rag.md) | Retrieval by meaning with `semantic_text` (auto-embed on ingest, no separate vector DB) | [`rag_demo.py`](../examples/semantic-search-rag/rag_demo.py) || [Log analytics](./log-analytics.md) | From raw logs to dashboards — error rates, p95 latency, top services via aggregations | [`log_analytics.py`](../examples/log-analytics/log_analytics.py) || [Vector search (kNN)](./vector-search-knn.md) | Nearest-neighbor similarity search over `dense_vector` (HNSW), with filters | [`knn_demo.py`](../examples/vector-search-knn/knn_demo.py) |+| [Vector quantization](./vector-quantization.md) | Opt a `dense_vector` field into scalar8 (`int8_hnsw`) — 4× smaller vectors, recall@10 ≈ 0.99 | [`quant_demo.py`](../examples/vector-quantization/quant_demo.py) || [Hybrid search](./hybrid-search.md) | Keyword + vector in one query — results neither BM25 nor kNN finds alone | [`hybrid_search.py`](../examples/hybrid-search/hybrid_search.py) || [Anomaly detection](./anomaly-detection.md) | Statistical `_ml` detectors that flag spikes in metrics/logs | [`anomaly_detection.py`](../examples/anomaly-detection/anomaly_detection.py) || [Migrate from Elasticsearch](./migrate-from-elasticsearch.md) | Point your existing ES client at XERJ — same wire, change the URL | [`migrate_demo.sh`](../examples/migrate-from-elasticsearch/migrate_demo.sh) |diff --git a/docs/recipes/vector-quantization.md b/docs/recipes/vector-quantization.mdnew file mode 100644index 000000000..6687f6dee--- /dev/null+++ b/docs/recipes/vector-quantization.md@@ -0,0 +1,93 @@+# Vector quantization: 4× smaller vectors, same results++## The problem++Dense vectors are heavy. A 768-dim `float32` embedding is ~3 KB; a million+of them is ~3 GB of vector data that has to be resident to serve low-latency+kNN. Scale to tens of millions and the vector working set — not the text,+not the postings — becomes the thing that decides how much RAM you rent.++The standard fix is **scalar quantization**: store each dimension in one+byte instead of four. That's a 4× reduction. The catch everyone worries+about is recall — does compressing the vectors quietly wreck ranking+quality?++## Why XERJ++XERJ lets you opt a `dense_vector` field into **scalar8** (int8)+quantization per field. When you do, the kNN *serving* path scores against+1-byte-per-dimension codes instead of 4-byte floats — a ~4× smaller vector+working set — while `_source` still returns the **original** vectors for+retrieval. It's off by default (full float32), so you choose precision vs.+memory per field, exactly like Elasticsearch's `int8_hnsw`.++On a real 128-dim corpus the recall cost is negligible: **recall@10 ≈ 0.99**+against the exact float32 index, with the vector footprint cut from 512 to+128 bytes per vector.++## The solution++Opt a field in at mapping time with `index_options.type: int8_hnsw`:++```bash+curl -sX PUT "$XERJ_URL/docs" -H 'content-type: application/json' -d '{+ "mappings": {+ "properties": {+ "title": { "type": "text" },+ "v": {+ "type": "dense_vector",+ "dims": 128,+ "similarity": "cosine",+ "index_options": { "type": "int8_hnsw" }+ }+ }+ }+}'+```++Index and query exactly as you would a full-precision field — nothing else+changes:++```bash+curl -sX POST "$XERJ_URL/docs/_search" -H 'content-type: application/json' -d '{+ "knn": { "field": "v", "query_vector": [0.12, 0.08, -0.31, "..."], "k": 10 }+}'+```++The scores come back slightly different from an exact float32 index (that's+the quantization at work — a query that exactly matches a stored vector+scores ~0.99999 instead of 1.0), but the **ranking is the same**.++## Try it++`docs/examples/vector-quantization/quant_demo.py` (and the identical+`recipes/vector_quantization.py`) embeds the 40 real KB articles into+128-dim vectors, indexes the same vectors into a float32 index and a+scalar8 index, and prints the side-by-side top hits plus recall@10:++```+$ python3 recipes/vector_quantization.py+embedded 40 real KB articles into 128-dim vectors+indexed into `vq-none` (float32) and `vq-scalar8` (int8_hnsw / scalar8)++── float32 (exact)+ 0.67960 Long-context windows do not replace memory+ 0.60029 p95 latency budgets for interactive RAG agents+── scalar8 (quantized)+ 0.67938 Long-context windows do not replace memory+ 0.60021 p95 latency budgets for interactive RAG agents++recall@10 (scalar8 vs float32 ground truth): 0.998+vector footprint: float32 = 512 B/vec → scalar8 = 128 B/vec (4x smaller)+```++## Notes and limits++- **Opt-in per field.** Fields without `int8_hnsw` keep exact float32+ scoring, byte-for-byte unchanged.+- **`_source` is never quantized.** Retrieval returns the vectors you+ indexed; only the scoring path uses the compact codes.+- **`scalar8` is wired; `binary` is not yet.** Binary (1-bit) quantization+ is rejected at startup rather than silently storing full precision.+- **Cosine is normalised** before quantizing for the tightest code range;+ `dot_product` and `l2_norm` similarities are supported too.diff --git a/recipes/README.md b/recipes/README.mdindex 6e7115866..946324d73 100644--- a/recipes/README.md+++ b/recipes/README.md@@ -26,6 +26,7 @@ python3 recipes/semantic_search.pypython3 recipes/rag_app.pypython3 recipes/memory_agent.py # run twice — memory persistspython3 recipes/log_anomaly.py+python3 recipes/vector_quantization.py```Every recipe honors `XERJ_URL` (default `http://localhost:9200`).diff --git a/recipes/run_all.sh b/recipes/run_all.shindex 2915a12bf..5fb9d0aca 100755--- a/recipes/run_all.sh+++ b/recipes/run_all.sh@@ -26,7 +26,7 @@ export XERJ_URL="http://127.0.0.1:${ESPORT}""$XERJ_BIN" --insecure --config "$DATA/xerj.toml" >"$DATA/server.log" 2>&1 &SRV=$!-trap 'kill "$SRV" 2>/dev/null || true; rm -rf "$DATA"' EXIT INT TERM+trap 'kill "$SRV" 2>/dev/null || true; wait "$SRV" 2>/dev/null || true; rm -rf "$DATA" 2>/dev/null || true' EXIT INT TERMi=0until curl -fsS "$XERJ_URL/" >/dev/null 2>&1; do@@ -35,7 +35,7 @@ until curl -fsS "$XERJ_URL/" >/dev/null 2>&1; dodonerc=0-for r in semantic_search rag_app memory_agent log_anomaly; do+for r in semantic_search rag_app memory_agent log_anomaly vector_quantization; doprintf '\n═══ %s ═══\n' "$r"if python3 "$ROOT/recipes/$r.py"; then :; else echo "FAILED: $r"; rc=1; fidonediff --git a/recipes/vector_quantization.py b/recipes/vector_quantization.pynew file mode 100644index 000000000..256cf6085--- /dev/null+++ b/recipes/vector_quantization.py@@ -0,0 +1,148 @@+#!/usr/bin/env python3+"""Scalar8 (int8) vector quantization on a real corpus — zero dependencies.++XERJ stores dense vectors at full float32 precision by default. Opting a+`dense_vector` field into scalar8 quantization (`index_options.type:+int8_hnsw`) makes the kNN *serving* path score against 1-byte-per-dimension+codes instead of 4-byte floats — a ~4x smaller vector working set with+almost no recall loss. `_source` still returns the original vectors.++This recipe embeds the 40 real KB articles (demo/data/ai_kb.ndjson) into+128-dim vectors with a small deterministic feature-hasher (same idea as+XERJ's built-in lexical embedder), then indexes the SAME vectors into two+indices — one full-precision (`none`), one quantized (`scalar8`) — and+shows that:++ 1. kNN returns the same top results from both,+ 2. recall@10 of the quantized index vs the exact index stays >= 0.90,+ 3. the quantized field's vector footprint is 4x smaller (1 vs 4 bytes/dim).++Usage:+ xerj --insecure --data-dir ./data & # start XERJ+ python3 recipes/vector_quantization.py++ XERJ_URL (default http://localhost:9200)+"""++import hashlib+import json+import math+import os+import pathlib+import urllib.error+import urllib.request++XERJ = os.environ.get("XERJ_URL", "http://localhost:9200")+KB = pathlib.Path(__file__).resolve().parent.parent / "demo" / "data" / "ai_kb.ndjson"+DIM = 128+NONE_INDEX = "vq-none"+SQ8_INDEX = "vq-scalar8"+++def call(method, path, body=None):+ """One tiny HTTP helper instead of a client library."""+ data = None+ headers = {}+ if body is not None:+ data = body.encode() if isinstance(body, str) else json.dumps(body).encode()+ headers["Content-Type"] = (+ "application/x-ndjson" if isinstance(body, str) else "application/json"+ )+ req = urllib.request.Request(f"{XERJ}{path}", data=data, headers=headers, method=method)+ with urllib.request.urlopen(req) as resp:+ return json.loads(resp.read())+++def _h(kind, token):+ """Stable (seed-independent) hash → 64-bit int, via md5."""+ return int.from_bytes(hashlib.md5(f"{kind}:{token}".encode()).digest()[:8], "big")+++def embed(text, dim=DIM):+ """Deterministic feature-hashing embedder: word unigrams + char trigrams+ hashed into `dim` buckets with signed contributions, then L2-normalised.+ Real, reproducible, zero-dependency — the same family XERJ uses built-in."""+ vec = [0.0] * dim+ toks = "".join(c.lower() if c.isalnum() else " " for c in text).split()+ for w in toks:+ h = _h("w", w)+ vec[h % dim] += 1.0 if (h >> 63) & 1 else -1.0+ padded = f"#{w}#"+ for i in range(len(padded) - 2):+ t = padded[i : i + 3]+ ht = _h("t", t)+ vec[ht % dim] += 0.35 if (ht >> 63) & 1 else -0.35+ norm = math.sqrt(sum(x * x for x in vec)) or 1.0+ return [x / norm for x in vec]+++def make_index(name, quantized):+ try:+ call("DELETE", f"/{name}")+ except urllib.error.HTTPError:+ pass+ field = {"type": "dense_vector", "dims": DIM, "similarity": "cosine"}+ if quantized:+ field["index_options"] = {"type": "int8_hnsw"} # opt in to scalar8+ call("PUT", f"/{name}", {+ "mappings": {"properties": {"title": {"type": "text"}, "v": field}}+ })+++def bulk_load(name, docs):+ lines = []+ for i, d in enumerate(docs):+ lines.append(json.dumps({"index": {"_index": name, "_id": str(i)}}))+ lines.append(json.dumps({"title": d["title"], "v": d["v"]}))+ call("POST", "/_bulk?refresh=true", "\n".join(lines) + "\n")+++def knn(name, qv, k=10):+ r = call("POST", f"/{name}/_search", {"knn": {"field": "v", "query_vector": qv, "k": k}})+ return [(h["_id"], h["_score"]) for h in r["hits"]["hits"]]+++def main():+ # ── 1. Embed the real KB into 128-dim vectors. ───────────────────────+ docs = []+ for raw in KB.read_text().splitlines():+ a = json.loads(raw)+ docs.append({"title": a["title"], "v": embed(a["title"] + " " + a["content"])})+ print(f"embedded {len(docs)} real KB articles into {DIM}-dim vectors\n")++ # ── 2. Index the SAME vectors two ways. ──────────────────────────────+ make_index(NONE_INDEX, quantized=False)+ make_index(SQ8_INDEX, quantized=True)+ bulk_load(NONE_INDEX, docs)+ bulk_load(SQ8_INDEX, docs)+ print(f"indexed into `{NONE_INDEX}` (float32) and `{SQ8_INDEX}` (int8_hnsw / scalar8)\n")++ # ── 3. Same query, both indices — exact vs quantized scoring. ────────+ q = embed("how do I stop an agent's context window from overflowing?")+ print("query: 'how do I stop an agent's context window from overflowing?'\n")+ for name in (NONE_INDEX, SQ8_INDEX):+ top = knn(name, q, k=3)+ tag = "float32 (exact)" if name == NONE_INDEX else "scalar8 (quantized)"+ print(f"── {tag}")+ for _id, score in top:+ print(f" {score:.5f} {docs[int(_id)]['title']}")+ print()++ # ── 4. recall@10 of quantized vs exact over the whole corpus. ────────+ hits = 0+ total = 0+ for d in docs:+ exact = {i for i, _ in knn(NONE_INDEX, d["v"], k=10)}+ approx = {i for i, _ in knn(SQ8_INDEX, d["v"], k=10)}+ hits += len(exact & approx)+ total += len(exact)+ recall = hits / total if total else 0.0+ print(f"recall@10 (scalar8 vs float32 ground truth): {recall:.3f}")+ print(f"vector footprint: float32 = {DIM * 4} B/vec → scalar8 = {DIM} B/vec (4x smaller)")+ if recall < 0.90:+ raise SystemExit(f"FAIL: recall {recall:.3f} < 0.90")+ print("\nOK — 4x smaller vectors, recall preserved. `_source` still holds the originals.")+++if __name__ == "__main__":+ main()