Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

chore(playbooks): reproducible neural-embedder verification harness

xerj-org87dcf3311a0e
diff --git a/demo/playbooks/neural-embedder-verification/ingest_slice.py b/demo/playbooks/neural-embedder-verification/ingest_slice.py
new file mode 100644
index 000000000..073138995
--- /dev/null
+++ b/demo/playbooks/neural-embedder-verification/ingest_slice.py
@@ -0,0 +1,100 @@
+"""Index a small slice of the rust-text corpus into a XERJ index whose `body`
+field is `semantic_text`, so the server's configured embedding backend embeds
+it on ingest.
+
+usage: ingest_slice.py <base_url> <index_name>
+"""
+
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+
+SLICE_ROOT = "/home/claude/.xerj-code/corpora/rust-text/regex/regex-automata"
+
+MAPPING = {
+ "mappings": {
+ "properties": {
+ "title": {"type": "keyword"},
+ "path": {"type": "keyword"},
+ "language": {"type": "keyword"},
+ "body": {"type": "semantic_text"},
+ }
+ }
+}
+
+
+def req(method, url, payload=None, ctype="application/json"):
+ data = payload.encode() if isinstance(payload, str) else payload
+ r = urllib.request.Request(url, data=data, method=method,
+ headers={"Content-Type": ctype})
+ try:
+ with urllib.request.urlopen(r, timeout=1800) as resp:
+ return resp.status, resp.read().decode()
+ except urllib.error.HTTPError as e:
+ return e.code, e.read().decode()
+
+
+def main():
+ base, index = sys.argv[1], sys.argv[2]
+
+ print(f"DELETE {index}: {req('DELETE', f'{base}/{index}')[0]}")
+ status, body = req("PUT", f"{base}/{index}", json.dumps(MAPPING))
+ print(f"PUT {index}: {status} {body[:300]}")
+ if status >= 300:
+ sys.exit(1)
+
+ files = []
+ for root, _dirs, names in os.walk(SLICE_ROOT):
+ for name in sorted(names):
+ if name.endswith(".rs"):
+ files.append(os.path.join(root, name))
+ files.sort()
+ print(f"slice: {len(files)} .rs files under {SLICE_ROOT}")
+
+ lines = []
+ total_bytes = 0
+ for path in files:
+ with open(path, "r", encoding="utf-8", errors="replace") as fh:
+ text = fh.read()
+ total_bytes += len(text)
+ doc = {
+ "title": os.path.basename(path),
+ "path": os.path.relpath(path, SLICE_ROOT),
+ "language": "rust",
+ "body": text,
+ }
+ lines.append(json.dumps({"index": {"_index": index}}))
+ lines.append(json.dumps(doc))
+ print(f"slice bytes: {total_bytes}")
+
+ # Bulk in small batches so a slow CPU model does not blow the HTTP timeout.
+ batch, sent, t0 = 8, 0, time.time()
+ for i in range(0, len(lines), batch * 2):
+ payload = "\n".join(lines[i:i + batch * 2]) + "\n"
+ status, body = req("POST", f"{base}/_bulk", payload,
+ ctype="application/x-ndjson")
+ if status >= 300:
+ print(f"bulk FAILED {status}: {body[:600]}")
+ sys.exit(1)
+ parsed = json.loads(body)
+ if parsed.get("errors"):
+ for item in parsed.get("items", []):
+ err = list(item.values())[0].get("error")
+ if err:
+ print(f"item error: {json.dumps(err)[:500]}")
+ sys.exit(1)
+ sent += batch
+ print(f" indexed ~{min(sent, len(files))}/{len(files)} "
+ f"({time.time() - t0:.1f}s)", flush=True)
+
+ req("POST", f"{base}/{index}/_refresh")
+ status, body = req("GET", f"{base}/{index}/_count")
+ print(f"count: {body}")
+ print(f"total ingest wall time: {time.time() - t0:.1f}s")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/demo/playbooks/neural-embedder-verification/lexical.toml b/demo/playbooks/neural-embedder-verification/lexical.toml
new file mode 100644
index 000000000..dbc51d6f2
--- /dev/null
+++ b/demo/playbooks/neural-embedder-verification/lexical.toml
@@ -0,0 +1,14 @@
+[server]
+rest_port = 9540
+grpc_port = 9541
+es_compat_port = 9542
+data_dir = "/tmp/xerj-lex-data"
+
+[tls]
+enabled = false
+
+[auth]
+enabled = false
+
+[embedding]
+mode = "lexical"
diff --git a/demo/playbooks/neural-embedder-verification/neural.toml b/demo/playbooks/neural-embedder-verification/neural.toml
new file mode 100644
index 000000000..0a342f0db
--- /dev/null
+++ b/demo/playbooks/neural-embedder-verification/neural.toml
@@ -0,0 +1,16 @@
+[server]
+rest_port = 9530
+grpc_port = 9531
+es_compat_port = 9532
+data_dir = "/tmp/xerj-neural-data"
+
+[tls]
+enabled = false
+
+[auth]
+enabled = false
+
+[embedding]
+mode = "neural"
+neural_model = "sentence-transformers/all-MiniLM-L6-v2"
+model_cache_dir = "/tmp/xerj-neural-model-cache"
diff --git a/demo/playbooks/neural-embedder-verification/paraphrase.py b/demo/playbooks/neural-embedder-verification/paraphrase.py
new file mode 100644
index 000000000..bc5684547
--- /dev/null
+++ b/demo/playbooks/neural-embedder-verification/paraphrase.py
@@ -0,0 +1,58 @@
+"""Paraphrase-retrieval probe: queries deliberately worded WITHOUT the target
+file's identifying vocabulary. Reports the rank of the ground-truth file and
+the top-hit / 10th-hit score spread, for whichever backend the server runs."""
+
+import json
+import sys
+import urllib.request
+
+# (query, expected filename substring). Each query avoids the target's own
+# distinctive tokens so lexical overlap cannot carry the retrieval.
+PROBES = [
+ ("bounded backtracking visited set", "backtrack.rs"),
+ ("avoid retrying the same position twice", "backtrack.rs"),
+ ("build the state machine lazily while searching instead of ahead of time",
+ "hybrid"),
+ ("reuse scratch allocations across calls instead of allocating each time",
+ "pool.rs"),
+ ("a machine that never has to try an alternative path", "onepass.rs"),
+]
+
+
+def search(base, index, field, query, k=10):
+ body = json.dumps({
+ "query": {"semantic": {"field": field, "query": query, "k": k}},
+ "size": k,
+ "_source": ["title", "path"],
+ }).encode()
+ req = urllib.request.Request(f"{base}/{index}/_search", data=body,
+ headers={"Content-Type": "application/json"})
+ with urllib.request.urlopen(req, timeout=300) as r:
+ return json.load(r)
+
+
+def main():
+ base, index, field, label = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
+ print(f"\n=== {label} ===")
+ found, spreads = 0, []
+ for query, expect in PROBES:
+ res = search(base, index, field, query)
+ hits = res["hits"]["hits"]
+ paths = [h["_source"].get("path", "") for h in hits]
+ titles = [h["_source"].get("title", "") for h in hits]
+ rank = next((i + 1 for i, (p, t) in enumerate(zip(paths, titles))
+ if expect in p or expect in t), None)
+ scores = [h["_score"] for h in hits]
+ spread = scores[0] / scores[min(9, len(scores) - 1)]
+ spreads.append(spread)
+ if rank:
+ found += 1
+ print(f" {query!r}")
+ print(f" expect~{expect} rank={rank} spread={spread:.4f} "
+ f"top1={titles[0]}")
+ print(f" --> found in top-10: {found}/{len(PROBES)} "
+ f"mean spread={sum(spreads) / len(spreads):.4f}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/demo/playbooks/neural-embedder-verification/qlatency.py b/demo/playbooks/neural-embedder-verification/qlatency.py
new file mode 100644
index 000000000..2068e2c59
--- /dev/null
+++ b/demo/playbooks/neural-embedder-verification/qlatency.py
@@ -0,0 +1,37 @@
+"""Per-query wall time for a `semantic` query, which includes embedding the
+query text with whichever backend the server runs. Every iteration uses a
+DISTINCT query string so no result/query cache can serve the measurement."""
+
+import json
+import statistics
+import sys
+import time
+import urllib.request
+
+QUERY = "avoid retrying the same position twice"
+
+
+def one(base, index, nonce=""):
+ body = json.dumps({
+ "query": {"semantic": {"field": "body", "query": QUERY + nonce, "k": 10}},
+ "size": 10, "_source": ["title"],
+ }).encode()
+ req = urllib.request.Request(f"{base}/{index}/_search", data=body,
+ headers={"Content-Type": "application/json"})
+ t0 = time.time()
+ with urllib.request.urlopen(req, timeout=300) as r:
+ r.read()
+ return (time.time() - t0) * 1000
+
+
+def main():
+ base, index, label = sys.argv[1], sys.argv[2], sys.argv[3]
+ one(base, index, " warmup") # warm the model + page cache
+ samples = [one(base, index, f" variant {i}") for i in range(5)]
+ print(f"{label}: median={statistics.median(samples):.1f}ms "
+ f"min={min(samples):.1f}ms max={max(samples):.1f}ms "
+ f"samples={[round(s, 1) for s in samples]}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/demo/playbooks/neural-embedder-verification/rc10-neural.toml b/demo/playbooks/neural-embedder-verification/rc10-neural.toml
new file mode 100644
index 000000000..b3d244fc1
--- /dev/null
+++ b/demo/playbooks/neural-embedder-verification/rc10-neural.toml
@@ -0,0 +1,15 @@
+[server]
+rest_port = 9550
+grpc_port = 9551
+es_compat_port = 9552
+data_dir = "/tmp/xerj-rc10-neural-data"
+
+[tls]
+enabled = false
+
+[auth]
+enabled = false
+
+[embedding]
+mode = "neural"
+model_cache_dir = "/tmp/xerj-neural-model-cache"
diff --git a/demo/playbooks/neural-embedder-verification/sanity.py b/demo/playbooks/neural-embedder-verification/sanity.py
new file mode 100644
index 000000000..355b9e5f3
--- /dev/null
+++ b/demo/playbooks/neural-embedder-verification/sanity.py
@@ -0,0 +1,60 @@
+"""Minimal semantic sanity check: three short sentences, one paraphrase pair
+with almost no shared vocabulary, one unrelated. A backend that models meaning
+must rank the paraphrase above the unrelated sentence."""
+
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+
+DOCS = [
+ ("guitar", "A man is playing a guitar on stage."),
+ ("musician", "A performer strums six strings in front of a crowd."),
+ ("finance", "The quarterly financial report shows rising interest rates."),
+]
+QUERY = "someone entertaining an audience with a stringed instrument"
+
+
+def req(method, url, payload=None):
+ data = payload.encode() if isinstance(payload, str) else payload
+ r = urllib.request.Request(url, data=data, method=method,
+ headers={"Content-Type": "application/json"})
+ try:
+ with urllib.request.urlopen(r, timeout=600) as resp:
+ return resp.status, resp.read().decode()
+ except urllib.error.HTTPError as e:
+ return e.code, e.read().decode()
+
+
+def main():
+ base, index = sys.argv[1], sys.argv[2]
+ req("DELETE", f"{base}/{index}")
+ status, body = req("PUT", f"{base}/{index}", json.dumps(
+ {"mappings": {"properties": {"title": {"type": "keyword"},
+ "body": {"type": "semantic_text"}}}}))
+ print(f"PUT {index}: {status} {body[:200]}")
+ t0 = time.time()
+ for title, text in DOCS:
+ status, body = req("POST", f"{base}/{index}/_doc?refresh=true",
+ json.dumps({"title": title, "body": text}))
+ if status >= 300:
+ print(f"index {title} FAILED {status}: {body[:400]}")
+ sys.exit(1)
+ print(f"indexed {len(DOCS)} docs in {time.time() - t0:.1f}s")
+
+ status, body = req("POST", f"{base}/{index}/_search", json.dumps(
+ {"query": {"semantic": {"field": "body", "query": QUERY, "k": 3}},
+ "size": 3, "_source": ["title"]}))
+ hits = json.loads(body)["hits"]["hits"]
+ print(f"query: {QUERY!r}")
+ for h in hits:
+ print(f" {h['_source']['title']:10s} score={h['_score']:.5f}")
+ names = [h["_source"]["title"] for h in hits]
+ top = hits[0]["_score"]
+ bottom = hits[-1]["_score"]
+ print(f" ranking={names} top/bottom spread={top / bottom:.4f}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/demo/playbooks/neural-embedder-verification/spread.py b/demo/playbooks/neural-embedder-verification/spread.py
new file mode 100644
index 000000000..1f8cd9fd0
--- /dev/null
+++ b/demo/playbooks/neural-embedder-verification/spread.py
@@ -0,0 +1,64 @@
+"""Score-spread diagnostic: top-hit / 10th-hit ratio on a semantic query,
+plus the rank of the ground-truth file. Same shape for lexical and neural."""
+
+import json
+import sys
+import urllib.request
+
+QUERIES = [
+ ("bounded backtracking visited set", "backtrack"),
+ ("avoid retrying the same position twice", "backtrack"),
+]
+
+
+def search(base, index, field, query, k=10):
+ body = json.dumps(
+ {
+ "query": {"semantic": {"field": field, "query": query, "k": k}},
+ "size": k,
+ "_source": ["title", "ax_path", "path"],
+ }
+ ).encode()
+ req = urllib.request.Request(
+ f"{base}/{index}/_search",
+ data=body,
+ headers={"Content-Type": "application/json"},
+ )
+ with urllib.request.urlopen(req, timeout=300) as r:
+ return json.load(r)
+
+
+def report(base, index, field, label, queries):
+ print(f"\n=== {label} ({base} / {index}) ===")
+ for q, expect in queries:
+ try:
+ res = search(base, index, field, q)
+ except Exception as e:
+ print(f" {q!r}: ERROR {e}")
+ continue
+ hits = res["hits"]["hits"]
+ if not hits:
+ print(f" {q!r}: no hits")
+ continue
+ scores = [h["_score"] for h in hits]
+ top = scores[0]
+ tenth = scores[min(9, len(scores) - 1)]
+ spread = top / tenth if tenth else float("inf")
+ names = [
+ h["_source"].get("title")
+ or h["_source"].get("ax_path")
+ or h["_source"].get("path")
+ for h in hits
+ ]
+ rank = next((i + 1 for i, n in enumerate(names) if n and expect in n), None)
+ print(f" query: {q!r}")
+ print(
+ f" expect~{expect} rank={rank} spread(top/10th)={spread:.4f} "
+ f"top={top:.5f} 10th={tenth:.5f} n={len(hits)}"
+ )
+ print(f" top5: {names[:5]}")
+
+
+if __name__ == "__main__":
+ base, index, field, label = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
+ report(base, index, field, label, QUERIES)