Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

docs(recipes): real scalar8 quantization use case + harden run_all teardown

xerj-org5468a5cdbec5
diff --git a/docs/examples/vector-quantization/quant_demo.py b/docs/examples/vector-quantization/quant_demo.py
new file mode 100644
index 000000000..256cf6085
--- /dev/null
+++ b/docs/examples/vector-quantization/quant_demo.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Scalar8 (int8) vector quantization on a real corpus — zero dependencies.
+
+XERJ stores dense vectors at full float32 precision by default. Opting a
+`dense_vector` field into scalar8 quantization (`index_options.type:
+int8_hnsw`) makes the kNN *serving* path score against 1-byte-per-dimension
+codes instead of 4-byte floats — a ~4x smaller vector working set with
+almost no recall loss. `_source` still returns the original vectors.
+
+This recipe embeds the 40 real KB articles (demo/data/ai_kb.ndjson) into
+128-dim vectors with a small deterministic feature-hasher (same idea as
+XERJ's built-in lexical embedder), then indexes the SAME vectors into two
+indices — one full-precision (`none`), one quantized (`scalar8`) — and
+shows that:
+
+ 1. kNN returns the same top results from both,
+ 2. recall@10 of the quantized index vs the exact index stays >= 0.90,
+ 3. the quantized field's vector footprint is 4x smaller (1 vs 4 bytes/dim).
+
+Usage:
+ xerj --insecure --data-dir ./data & # start XERJ
+ python3 recipes/vector_quantization.py
+
+ XERJ_URL (default http://localhost:9200)
+"""
+
+import hashlib
+import json
+import math
+import os
+import pathlib
+import urllib.error
+import urllib.request
+
+XERJ = os.environ.get("XERJ_URL", "http://localhost:9200")
+KB = pathlib.Path(__file__).resolve().parent.parent / "demo" / "data" / "ai_kb.ndjson"
+DIM = 128
+NONE_INDEX = "vq-none"
+SQ8_INDEX = "vq-scalar8"
+
+
+def call(method, path, body=None):
+ """One tiny HTTP helper instead of a client library."""
+ data = None
+ headers = {}
+ if body is not None:
+ data = body.encode() if isinstance(body, str) else json.dumps(body).encode()
+ headers["Content-Type"] = (
+ "application/x-ndjson" if isinstance(body, str) else "application/json"
+ )
+ req = urllib.request.Request(f"{XERJ}{path}", data=data, headers=headers, method=method)
+ with urllib.request.urlopen(req) as resp:
+ return json.loads(resp.read())
+
+
+def _h(kind, token):
+ """Stable (seed-independent) hash → 64-bit int, via md5."""
+ return int.from_bytes(hashlib.md5(f"{kind}:{token}".encode()).digest()[:8], "big")
+
+
+def embed(text, dim=DIM):
+ """Deterministic feature-hashing embedder: word unigrams + char trigrams
+ hashed into `dim` buckets with signed contributions, then L2-normalised.
+ Real, reproducible, zero-dependency — the same family XERJ uses built-in."""
+ vec = [0.0] * dim
+ toks = "".join(c.lower() if c.isalnum() else " " for c in text).split()
+ for w in toks:
+ h = _h("w", w)
+ vec[h % dim] += 1.0 if (h >> 63) & 1 else -1.0
+ padded = f"#{w}#"
+ for i in range(len(padded) - 2):
+ t = padded[i : i + 3]
+ ht = _h("t", t)
+ vec[ht % dim] += 0.35 if (ht >> 63) & 1 else -0.35
+ norm = math.sqrt(sum(x * x for x in vec)) or 1.0
+ return [x / norm for x in vec]
+
+
+def make_index(name, quantized):
+ try:
+ call("DELETE", f"/{name}")
+ except urllib.error.HTTPError:
+ pass
+ field = {"type": "dense_vector", "dims": DIM, "similarity": "cosine"}
+ if quantized:
+ field["index_options"] = {"type": "int8_hnsw"} # opt in to scalar8
+ call("PUT", f"/{name}", {
+ "mappings": {"properties": {"title": {"type": "text"}, "v": field}}
+ })
+
+
+def bulk_load(name, docs):
+ lines = []
+ for i, d in enumerate(docs):
+ lines.append(json.dumps({"index": {"_index": name, "_id": str(i)}}))
+ lines.append(json.dumps({"title": d["title"], "v": d["v"]}))
+ call("POST", "/_bulk?refresh=true", "\n".join(lines) + "\n")
+
+
+def knn(name, qv, k=10):
+ r = call("POST", f"/{name}/_search", {"knn": {"field": "v", "query_vector": qv, "k": k}})
+ return [(h["_id"], h["_score"]) for h in r["hits"]["hits"]]
+
+
+def main():
+ # ── 1. Embed the real KB into 128-dim vectors. ───────────────────────
+ docs = []
+ for raw in KB.read_text().splitlines():
+ a = json.loads(raw)
+ docs.append({"title": a["title"], "v": embed(a["title"] + " " + a["content"])})
+ print(f"embedded {len(docs)} real KB articles into {DIM}-dim vectors\n")
+
+ # ── 2. Index the SAME vectors two ways. ──────────────────────────────
+ make_index(NONE_INDEX, quantized=False)
+ make_index(SQ8_INDEX, quantized=True)
+ bulk_load(NONE_INDEX, docs)
+ bulk_load(SQ8_INDEX, docs)
+ print(f"indexed into `{NONE_INDEX}` (float32) and `{SQ8_INDEX}` (int8_hnsw / scalar8)\n")
+
+ # ── 3. Same query, both indices — exact vs quantized scoring. ────────
+ q = embed("how do I stop an agent's context window from overflowing?")
+ print("query: 'how do I stop an agent's context window from overflowing?'\n")
+ for name in (NONE_INDEX, SQ8_INDEX):
+ top = knn(name, q, k=3)
+ tag = "float32 (exact)" if name == NONE_INDEX else "scalar8 (quantized)"
+ print(f"── {tag}")
+ for _id, score in top:
+ print(f" {score:.5f} {docs[int(_id)]['title']}")
+ print()
+
+ # ── 4. recall@10 of quantized vs exact over the whole corpus. ────────
+ hits = 0
+ total = 0
+ for d in docs:
+ exact = {i for i, _ in knn(NONE_INDEX, d["v"], k=10)}
+ approx = {i for i, _ in knn(SQ8_INDEX, d["v"], k=10)}
+ hits += len(exact & approx)
+ total += len(exact)
+ recall = hits / total if total else 0.0
+ print(f"recall@10 (scalar8 vs float32 ground truth): {recall:.3f}")
+ print(f"vector footprint: float32 = {DIM * 4} B/vec → scalar8 = {DIM} B/vec (4x smaller)")
+ if recall < 0.90:
+ raise SystemExit(f"FAIL: recall {recall:.3f} < 0.90")
+ print("\nOK — 4x smaller vectors, recall preserved. `_source` still holds the originals.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/recipes/README.md b/docs/recipes/README.md
index 4104dae68..636329af8 100644
--- a/docs/recipes/README.md
+++ b/docs/recipes/README.md
@@ -10,6 +10,7 @@ These are practical "how do I actually do X" recipes, not an API reference (that
| [Semantic search & RAG](./semantic-search-rag.md) | Retrieval by meaning with `semantic_text` (auto-embed on ingest, no separate vector DB) | [`rag_demo.py`](../examples/semantic-search-rag/rag_demo.py) |
| [Log analytics](./log-analytics.md) | From raw logs to dashboards — error rates, p95 latency, top services via aggregations | [`log_analytics.py`](../examples/log-analytics/log_analytics.py) |
| [Vector search (kNN)](./vector-search-knn.md) | Nearest-neighbor similarity search over `dense_vector` (HNSW), with filters | [`knn_demo.py`](../examples/vector-search-knn/knn_demo.py) |
+| [Vector quantization](./vector-quantization.md) | Opt a `dense_vector` field into scalar8 (`int8_hnsw`) — 4× smaller vectors, recall@10 ≈ 0.99 | [`quant_demo.py`](../examples/vector-quantization/quant_demo.py) |
| [Hybrid search](./hybrid-search.md) | Keyword + vector in one query — results neither BM25 nor kNN finds alone | [`hybrid_search.py`](../examples/hybrid-search/hybrid_search.py) |
| [Anomaly detection](./anomaly-detection.md) | Statistical `_ml` detectors that flag spikes in metrics/logs | [`anomaly_detection.py`](../examples/anomaly-detection/anomaly_detection.py) |
| [Migrate from Elasticsearch](./migrate-from-elasticsearch.md) | Point your existing ES client at XERJ — same wire, change the URL | [`migrate_demo.sh`](../examples/migrate-from-elasticsearch/migrate_demo.sh) |
diff --git a/docs/recipes/vector-quantization.md b/docs/recipes/vector-quantization.md
new file mode 100644
index 000000000..6687f6dee
--- /dev/null
+++ b/docs/recipes/vector-quantization.md
@@ -0,0 +1,93 @@
+# Vector quantization: 4× smaller vectors, same results
+
+## The problem
+
+Dense vectors are heavy. A 768-dim `float32` embedding is ~3 KB; a million
+of them is ~3 GB of vector data that has to be resident to serve low-latency
+kNN. Scale to tens of millions and the vector working set — not the text,
+not the postings — becomes the thing that decides how much RAM you rent.
+
+The standard fix is **scalar quantization**: store each dimension in one
+byte instead of four. That's a 4× reduction. The catch everyone worries
+about is recall — does compressing the vectors quietly wreck ranking
+quality?
+
+## Why XERJ
+
+XERJ lets you opt a `dense_vector` field into **scalar8** (int8)
+quantization per field. When you do, the kNN *serving* path scores against
+1-byte-per-dimension codes instead of 4-byte floats — a ~4× smaller vector
+working set — while `_source` still returns the **original** vectors for
+retrieval. It's off by default (full float32), so you choose precision vs.
+memory per field, exactly like Elasticsearch's `int8_hnsw`.
+
+On a real 128-dim corpus the recall cost is negligible: **recall@10 ≈ 0.99**
+against the exact float32 index, with the vector footprint cut from 512 to
+128 bytes per vector.
+
+## The solution
+
+Opt a field in at mapping time with `index_options.type: int8_hnsw`:
+
+```bash
+curl -sX PUT "$XERJ_URL/docs" -H 'content-type: application/json' -d '{
+ "mappings": {
+ "properties": {
+ "title": { "type": "text" },
+ "v": {
+ "type": "dense_vector",
+ "dims": 128,
+ "similarity": "cosine",
+ "index_options": { "type": "int8_hnsw" }
+ }
+ }
+ }
+}'
+```
+
+Index and query exactly as you would a full-precision field — nothing else
+changes:
+
+```bash
+curl -sX POST "$XERJ_URL/docs/_search" -H 'content-type: application/json' -d '{
+ "knn": { "field": "v", "query_vector": [0.12, 0.08, -0.31, "..."], "k": 10 }
+}'
+```
+
+The scores come back slightly different from an exact float32 index (that's
+the quantization at work — a query that exactly matches a stored vector
+scores ~0.99999 instead of 1.0), but the **ranking is the same**.
+
+## Try it
+
+`docs/examples/vector-quantization/quant_demo.py` (and the identical
+`recipes/vector_quantization.py`) embeds the 40 real KB articles into
+128-dim vectors, indexes the same vectors into a float32 index and a
+scalar8 index, and prints the side-by-side top hits plus recall@10:
+
+```
+$ python3 recipes/vector_quantization.py
+embedded 40 real KB articles into 128-dim vectors
+indexed into `vq-none` (float32) and `vq-scalar8` (int8_hnsw / scalar8)
+
+── float32 (exact)
+ 0.67960 Long-context windows do not replace memory
+ 0.60029 p95 latency budgets for interactive RAG agents
+── scalar8 (quantized)
+ 0.67938 Long-context windows do not replace memory
+ 0.60021 p95 latency budgets for interactive RAG agents
+
+recall@10 (scalar8 vs float32 ground truth): 0.998
+vector footprint: float32 = 512 B/vec → scalar8 = 128 B/vec (4x smaller)
+```
+
+## Notes and limits
+
+- **Opt-in per field.** Fields without `int8_hnsw` keep exact float32
+ scoring, byte-for-byte unchanged.
+- **`_source` is never quantized.** Retrieval returns the vectors you
+ indexed; only the scoring path uses the compact codes.
+- **`scalar8` is wired; `binary` is not yet.** Binary (1-bit) quantization
+ is rejected at startup rather than silently storing full precision.
+- **Cosine is normalised** before quantizing for the tightest code range;
+ `dot_product` and `l2_norm` similarities are supported too.
diff --git a/recipes/README.md b/recipes/README.md
index 6e7115866..946324d73 100644
--- a/recipes/README.md
+++ b/recipes/README.md
@@ -26,6 +26,7 @@ python3 recipes/semantic_search.py
python3 recipes/rag_app.py
python3 recipes/memory_agent.py # run twice — memory persists
python3 recipes/log_anomaly.py
+python3 recipes/vector_quantization.py
```
Every recipe honors `XERJ_URL` (default `http://localhost:9200`).
diff --git a/recipes/run_all.sh b/recipes/run_all.sh
index 2915a12bf..5fb9d0aca 100755
--- a/recipes/run_all.sh
+++ b/recipes/run_all.sh
@@ -26,7 +26,7 @@ export XERJ_URL="http://127.0.0.1:${ESPORT}"
"$XERJ_BIN" --insecure --config "$DATA/xerj.toml" >"$DATA/server.log" 2>&1 &
SRV=$!
-trap 'kill "$SRV" 2>/dev/null || true; rm -rf "$DATA"' EXIT INT TERM
+trap 'kill "$SRV" 2>/dev/null || true; wait "$SRV" 2>/dev/null || true; rm -rf "$DATA" 2>/dev/null || true' EXIT INT TERM
i=0
until curl -fsS "$XERJ_URL/" >/dev/null 2>&1; do
@@ -35,7 +35,7 @@ until curl -fsS "$XERJ_URL/" >/dev/null 2>&1; do
done
rc=0
-for r in semantic_search rag_app memory_agent log_anomaly; do
+for r in semantic_search rag_app memory_agent log_anomaly vector_quantization; do
printf '\n═══ %s ═══\n' "$r"
if python3 "$ROOT/recipes/$r.py"; then :; else echo "FAILED: $r"; rc=1; fi
done
diff --git a/recipes/vector_quantization.py b/recipes/vector_quantization.py
new file mode 100644
index 000000000..256cf6085
--- /dev/null
+++ b/recipes/vector_quantization.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Scalar8 (int8) vector quantization on a real corpus — zero dependencies.
+
+XERJ stores dense vectors at full float32 precision by default. Opting a
+`dense_vector` field into scalar8 quantization (`index_options.type:
+int8_hnsw`) makes the kNN *serving* path score against 1-byte-per-dimension
+codes instead of 4-byte floats — a ~4x smaller vector working set with
+almost no recall loss. `_source` still returns the original vectors.
+
+This recipe embeds the 40 real KB articles (demo/data/ai_kb.ndjson) into
+128-dim vectors with a small deterministic feature-hasher (same idea as
+XERJ's built-in lexical embedder), then indexes the SAME vectors into two
+indices — one full-precision (`none`), one quantized (`scalar8`) — and
+shows that:
+
+ 1. kNN returns the same top results from both,
+ 2. recall@10 of the quantized index vs the exact index stays >= 0.90,
+ 3. the quantized field's vector footprint is 4x smaller (1 vs 4 bytes/dim).
+
+Usage:
+ xerj --insecure --data-dir ./data & # start XERJ
+ python3 recipes/vector_quantization.py
+
+ XERJ_URL (default http://localhost:9200)
+"""
+
+import hashlib
+import json
+import math
+import os
+import pathlib
+import urllib.error
+import urllib.request
+
+XERJ = os.environ.get("XERJ_URL", "http://localhost:9200")
+KB = pathlib.Path(__file__).resolve().parent.parent / "demo" / "data" / "ai_kb.ndjson"
+DIM = 128
+NONE_INDEX = "vq-none"
+SQ8_INDEX = "vq-scalar8"
+
+
+def call(method, path, body=None):
+ """One tiny HTTP helper instead of a client library."""
+ data = None
+ headers = {}
+ if body is not None:
+ data = body.encode() if isinstance(body, str) else json.dumps(body).encode()
+ headers["Content-Type"] = (
+ "application/x-ndjson" if isinstance(body, str) else "application/json"
+ )
+ req = urllib.request.Request(f"{XERJ}{path}", data=data, headers=headers, method=method)
+ with urllib.request.urlopen(req) as resp:
+ return json.loads(resp.read())
+
+
+def _h(kind, token):
+ """Stable (seed-independent) hash → 64-bit int, via md5."""
+ return int.from_bytes(hashlib.md5(f"{kind}:{token}".encode()).digest()[:8], "big")
+
+
+def embed(text, dim=DIM):
+ """Deterministic feature-hashing embedder: word unigrams + char trigrams
+ hashed into `dim` buckets with signed contributions, then L2-normalised.
+ Real, reproducible, zero-dependency — the same family XERJ uses built-in."""
+ vec = [0.0] * dim
+ toks = "".join(c.lower() if c.isalnum() else " " for c in text).split()
+ for w in toks:
+ h = _h("w", w)
+ vec[h % dim] += 1.0 if (h >> 63) & 1 else -1.0
+ padded = f"#{w}#"
+ for i in range(len(padded) - 2):
+ t = padded[i : i + 3]
+ ht = _h("t", t)
+ vec[ht % dim] += 0.35 if (ht >> 63) & 1 else -0.35
+ norm = math.sqrt(sum(x * x for x in vec)) or 1.0
+ return [x / norm for x in vec]
+
+
+def make_index(name, quantized):
+ try:
+ call("DELETE", f"/{name}")
+ except urllib.error.HTTPError:
+ pass
+ field = {"type": "dense_vector", "dims": DIM, "similarity": "cosine"}
+ if quantized:
+ field["index_options"] = {"type": "int8_hnsw"} # opt in to scalar8
+ call("PUT", f"/{name}", {
+ "mappings": {"properties": {"title": {"type": "text"}, "v": field}}
+ })
+
+
+def bulk_load(name, docs):
+ lines = []
+ for i, d in enumerate(docs):
+ lines.append(json.dumps({"index": {"_index": name, "_id": str(i)}}))
+ lines.append(json.dumps({"title": d["title"], "v": d["v"]}))
+ call("POST", "/_bulk?refresh=true", "\n".join(lines) + "\n")
+
+
+def knn(name, qv, k=10):
+ r = call("POST", f"/{name}/_search", {"knn": {"field": "v", "query_vector": qv, "k": k}})
+ return [(h["_id"], h["_score"]) for h in r["hits"]["hits"]]
+
+
+def main():
+ # ── 1. Embed the real KB into 128-dim vectors. ───────────────────────
+ docs = []
+ for raw in KB.read_text().splitlines():
+ a = json.loads(raw)
+ docs.append({"title": a["title"], "v": embed(a["title"] + " " + a["content"])})
+ print(f"embedded {len(docs)} real KB articles into {DIM}-dim vectors\n")
+
+ # ── 2. Index the SAME vectors two ways. ──────────────────────────────
+ make_index(NONE_INDEX, quantized=False)
+ make_index(SQ8_INDEX, quantized=True)
+ bulk_load(NONE_INDEX, docs)
+ bulk_load(SQ8_INDEX, docs)
+ print(f"indexed into `{NONE_INDEX}` (float32) and `{SQ8_INDEX}` (int8_hnsw / scalar8)\n")
+
+ # ── 3. Same query, both indices — exact vs quantized scoring. ────────
+ q = embed("how do I stop an agent's context window from overflowing?")
+ print("query: 'how do I stop an agent's context window from overflowing?'\n")
+ for name in (NONE_INDEX, SQ8_INDEX):
+ top = knn(name, q, k=3)
+ tag = "float32 (exact)" if name == NONE_INDEX else "scalar8 (quantized)"
+ print(f"── {tag}")
+ for _id, score in top:
+ print(f" {score:.5f} {docs[int(_id)]['title']}")
+ print()
+
+ # ── 4. recall@10 of quantized vs exact over the whole corpus. ────────
+ hits = 0
+ total = 0
+ for d in docs:
+ exact = {i for i, _ in knn(NONE_INDEX, d["v"], k=10)}
+ approx = {i for i, _ in knn(SQ8_INDEX, d["v"], k=10)}
+ hits += len(exact & approx)
+ total += len(exact)
+ recall = hits / total if total else 0.0
+ print(f"recall@10 (scalar8 vs float32 ground truth): {recall:.3f}")
+ print(f"vector footprint: float32 = {DIM * 4} B/vec → scalar8 = {DIM} B/vec (4x smaller)")
+ if recall < 0.90:
+ raise SystemExit(f"FAIL: recall {recall:.3f} < 0.90")
+ print("\nOK — 4x smaller vectors, recall preserved. `_source` still holds the originals.")
+
+
+if __name__ == "__main__":
+ main()