AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
chore(playbooks): reproducible neural-embedder verification harness
xerj-org87dcf3311a0e
diff --git a/demo/playbooks/neural-embedder-verification/ingest_slice.py b/demo/playbooks/neural-embedder-verification/ingest_slice.pynew file mode 100644index 000000000..073138995--- /dev/null+++ b/demo/playbooks/neural-embedder-verification/ingest_slice.py@@ -0,0 +1,100 @@+"""Index a small slice of the rust-text corpus into a XERJ index whose `body`+field is `semantic_text`, so the server's configured embedding backend embeds+it on ingest.++usage: ingest_slice.py <base_url> <index_name>+"""++import json+import os+import sys+import time+import urllib.error+import urllib.request++SLICE_ROOT = "/home/claude/.xerj-code/corpora/rust-text/regex/regex-automata"++MAPPING = {+ "mappings": {+ "properties": {+ "title": {"type": "keyword"},+ "path": {"type": "keyword"},+ "language": {"type": "keyword"},+ "body": {"type": "semantic_text"},+ }+ }+}+++def req(method, url, payload=None, ctype="application/json"):+ data = payload.encode() if isinstance(payload, str) else payload+ r = urllib.request.Request(url, data=data, method=method,+ headers={"Content-Type": ctype})+ try:+ with urllib.request.urlopen(r, timeout=1800) as resp:+ return resp.status, resp.read().decode()+ except urllib.error.HTTPError as e:+ return e.code, e.read().decode()+++def main():+ base, index = sys.argv[1], sys.argv[2]++ print(f"DELETE {index}: {req('DELETE', f'{base}/{index}')[0]}")+ status, body = req("PUT", f"{base}/{index}", json.dumps(MAPPING))+ print(f"PUT {index}: {status} {body[:300]}")+ if status >= 300:+ sys.exit(1)++ files = []+ for root, _dirs, names in os.walk(SLICE_ROOT):+ for name in sorted(names):+ if name.endswith(".rs"):+ files.append(os.path.join(root, name))+ files.sort()+ print(f"slice: {len(files)} .rs files under {SLICE_ROOT}")++ lines = []+ total_bytes = 0+ for path in files:+ with open(path, "r", encoding="utf-8", errors="replace") as fh:+ text = fh.read()+ total_bytes += len(text)+ doc = {+ "title": os.path.basename(path),+ "path": os.path.relpath(path, SLICE_ROOT),+ "language": "rust",+ "body": text,+ }+ lines.append(json.dumps({"index": {"_index": index}}))+ lines.append(json.dumps(doc))+ print(f"slice bytes: {total_bytes}")++ # Bulk in small batches so a slow CPU model does not blow the HTTP timeout.+ batch, sent, t0 = 8, 0, time.time()+ for i in range(0, len(lines), batch * 2):+ payload = "\n".join(lines[i:i + batch * 2]) + "\n"+ status, body = req("POST", f"{base}/_bulk", payload,+ ctype="application/x-ndjson")+ if status >= 300:+ print(f"bulk FAILED {status}: {body[:600]}")+ sys.exit(1)+ parsed = json.loads(body)+ if parsed.get("errors"):+ for item in parsed.get("items", []):+ err = list(item.values())[0].get("error")+ if err:+ print(f"item error: {json.dumps(err)[:500]}")+ sys.exit(1)+ sent += batch+ print(f" indexed ~{min(sent, len(files))}/{len(files)} "+ f"({time.time() - t0:.1f}s)", flush=True)++ req("POST", f"{base}/{index}/_refresh")+ status, body = req("GET", f"{base}/{index}/_count")+ print(f"count: {body}")+ print(f"total ingest wall time: {time.time() - t0:.1f}s")+++if __name__ == "__main__":+ main()diff --git a/demo/playbooks/neural-embedder-verification/lexical.toml b/demo/playbooks/neural-embedder-verification/lexical.tomlnew file mode 100644index 000000000..dbc51d6f2--- /dev/null+++ b/demo/playbooks/neural-embedder-verification/lexical.toml@@ -0,0 +1,14 @@+[server]+rest_port = 9540+grpc_port = 9541+es_compat_port = 9542+data_dir = "/tmp/xerj-lex-data"++[tls]+enabled = false++[auth]+enabled = false++[embedding]+mode = "lexical"diff --git a/demo/playbooks/neural-embedder-verification/neural.toml b/demo/playbooks/neural-embedder-verification/neural.tomlnew file mode 100644index 000000000..0a342f0db--- /dev/null+++ b/demo/playbooks/neural-embedder-verification/neural.toml@@ -0,0 +1,16 @@+[server]+rest_port = 9530+grpc_port = 9531+es_compat_port = 9532+data_dir = "/tmp/xerj-neural-data"++[tls]+enabled = false++[auth]+enabled = false++[embedding]+mode = "neural"+neural_model = "sentence-transformers/all-MiniLM-L6-v2"+model_cache_dir = "/tmp/xerj-neural-model-cache"diff --git a/demo/playbooks/neural-embedder-verification/paraphrase.py b/demo/playbooks/neural-embedder-verification/paraphrase.pynew file mode 100644index 000000000..bc5684547--- /dev/null+++ b/demo/playbooks/neural-embedder-verification/paraphrase.py@@ -0,0 +1,58 @@+"""Paraphrase-retrieval probe: queries deliberately worded WITHOUT the target+file's identifying vocabulary. Reports the rank of the ground-truth file and+the top-hit / 10th-hit score spread, for whichever backend the server runs."""++import json+import sys+import urllib.request++# (query, expected filename substring). Each query avoids the target's own+# distinctive tokens so lexical overlap cannot carry the retrieval.+PROBES = [+ ("bounded backtracking visited set", "backtrack.rs"),+ ("avoid retrying the same position twice", "backtrack.rs"),+ ("build the state machine lazily while searching instead of ahead of time",+ "hybrid"),+ ("reuse scratch allocations across calls instead of allocating each time",+ "pool.rs"),+ ("a machine that never has to try an alternative path", "onepass.rs"),+]+++def search(base, index, field, query, k=10):+ body = json.dumps({+ "query": {"semantic": {"field": field, "query": query, "k": k}},+ "size": k,+ "_source": ["title", "path"],+ }).encode()+ req = urllib.request.Request(f"{base}/{index}/_search", data=body,+ headers={"Content-Type": "application/json"})+ with urllib.request.urlopen(req, timeout=300) as r:+ return json.load(r)+++def main():+ base, index, field, label = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]+ print(f"\n=== {label} ===")+ found, spreads = 0, []+ for query, expect in PROBES:+ res = search(base, index, field, query)+ hits = res["hits"]["hits"]+ paths = [h["_source"].get("path", "") for h in hits]+ titles = [h["_source"].get("title", "") for h in hits]+ rank = next((i + 1 for i, (p, t) in enumerate(zip(paths, titles))+ if expect in p or expect in t), None)+ scores = [h["_score"] for h in hits]+ spread = scores[0] / scores[min(9, len(scores) - 1)]+ spreads.append(spread)+ if rank:+ found += 1+ print(f" {query!r}")+ print(f" expect~{expect} rank={rank} spread={spread:.4f} "+ f"top1={titles[0]}")+ print(f" --> found in top-10: {found}/{len(PROBES)} "+ f"mean spread={sum(spreads) / len(spreads):.4f}")+++if __name__ == "__main__":+ main()diff --git a/demo/playbooks/neural-embedder-verification/qlatency.py b/demo/playbooks/neural-embedder-verification/qlatency.pynew file mode 100644index 000000000..2068e2c59--- /dev/null+++ b/demo/playbooks/neural-embedder-verification/qlatency.py@@ -0,0 +1,37 @@+"""Per-query wall time for a `semantic` query, which includes embedding the+query text with whichever backend the server runs. Every iteration uses a+DISTINCT query string so no result/query cache can serve the measurement."""++import json+import statistics+import sys+import time+import urllib.request++QUERY = "avoid retrying the same position twice"+++def one(base, index, nonce=""):+ body = json.dumps({+ "query": {"semantic": {"field": "body", "query": QUERY + nonce, "k": 10}},+ "size": 10, "_source": ["title"],+ }).encode()+ req = urllib.request.Request(f"{base}/{index}/_search", data=body,+ headers={"Content-Type": "application/json"})+ t0 = time.time()+ with urllib.request.urlopen(req, timeout=300) as r:+ r.read()+ return (time.time() - t0) * 1000+++def main():+ base, index, label = sys.argv[1], sys.argv[2], sys.argv[3]+ one(base, index, " warmup") # warm the model + page cache+ samples = [one(base, index, f" variant {i}") for i in range(5)]+ print(f"{label}: median={statistics.median(samples):.1f}ms "+ f"min={min(samples):.1f}ms max={max(samples):.1f}ms "+ f"samples={[round(s, 1) for s in samples]}")+++if __name__ == "__main__":+ main()diff --git a/demo/playbooks/neural-embedder-verification/rc10-neural.toml b/demo/playbooks/neural-embedder-verification/rc10-neural.tomlnew file mode 100644index 000000000..b3d244fc1--- /dev/null+++ b/demo/playbooks/neural-embedder-verification/rc10-neural.toml@@ -0,0 +1,15 @@+[server]+rest_port = 9550+grpc_port = 9551+es_compat_port = 9552+data_dir = "/tmp/xerj-rc10-neural-data"++[tls]+enabled = false++[auth]+enabled = false++[embedding]+mode = "neural"+model_cache_dir = "/tmp/xerj-neural-model-cache"diff --git a/demo/playbooks/neural-embedder-verification/sanity.py b/demo/playbooks/neural-embedder-verification/sanity.pynew file mode 100644index 000000000..355b9e5f3--- /dev/null+++ b/demo/playbooks/neural-embedder-verification/sanity.py@@ -0,0 +1,60 @@+"""Minimal semantic sanity check: three short sentences, one paraphrase pair+with almost no shared vocabulary, one unrelated. A backend that models meaning+must rank the paraphrase above the unrelated sentence."""++import json+import sys+import time+import urllib.error+import urllib.request++DOCS = [+ ("guitar", "A man is playing a guitar on stage."),+ ("musician", "A performer strums six strings in front of a crowd."),+ ("finance", "The quarterly financial report shows rising interest rates."),+]+QUERY = "someone entertaining an audience with a stringed instrument"+++def req(method, url, payload=None):+ data = payload.encode() if isinstance(payload, str) else payload+ r = urllib.request.Request(url, data=data, method=method,+ headers={"Content-Type": "application/json"})+ try:+ with urllib.request.urlopen(r, timeout=600) as resp:+ return resp.status, resp.read().decode()+ except urllib.error.HTTPError as e:+ return e.code, e.read().decode()+++def main():+ base, index = sys.argv[1], sys.argv[2]+ req("DELETE", f"{base}/{index}")+ status, body = req("PUT", f"{base}/{index}", json.dumps(+ {"mappings": {"properties": {"title": {"type": "keyword"},+ "body": {"type": "semantic_text"}}}}))+ print(f"PUT {index}: {status} {body[:200]}")+ t0 = time.time()+ for title, text in DOCS:+ status, body = req("POST", f"{base}/{index}/_doc?refresh=true",+ json.dumps({"title": title, "body": text}))+ if status >= 300:+ print(f"index {title} FAILED {status}: {body[:400]}")+ sys.exit(1)+ print(f"indexed {len(DOCS)} docs in {time.time() - t0:.1f}s")++ status, body = req("POST", f"{base}/{index}/_search", json.dumps(+ {"query": {"semantic": {"field": "body", "query": QUERY, "k": 3}},+ "size": 3, "_source": ["title"]}))+ hits = json.loads(body)["hits"]["hits"]+ print(f"query: {QUERY!r}")+ for h in hits:+ print(f" {h['_source']['title']:10s} score={h['_score']:.5f}")+ names = [h["_source"]["title"] for h in hits]+ top = hits[0]["_score"]+ bottom = hits[-1]["_score"]+ print(f" ranking={names} top/bottom spread={top / bottom:.4f}")+++if __name__ == "__main__":+ main()diff --git a/demo/playbooks/neural-embedder-verification/spread.py b/demo/playbooks/neural-embedder-verification/spread.pynew file mode 100644index 000000000..1f8cd9fd0--- /dev/null+++ b/demo/playbooks/neural-embedder-verification/spread.py@@ -0,0 +1,64 @@+"""Score-spread diagnostic: top-hit / 10th-hit ratio on a semantic query,+plus the rank of the ground-truth file. Same shape for lexical and neural."""++import json+import sys+import urllib.request++QUERIES = [+ ("bounded backtracking visited set", "backtrack"),+ ("avoid retrying the same position twice", "backtrack"),+]+++def search(base, index, field, query, k=10):+ body = json.dumps(+ {+ "query": {"semantic": {"field": field, "query": query, "k": k}},+ "size": k,+ "_source": ["title", "ax_path", "path"],+ }+ ).encode()+ req = urllib.request.Request(+ f"{base}/{index}/_search",+ data=body,+ headers={"Content-Type": "application/json"},+ )+ with urllib.request.urlopen(req, timeout=300) as r:+ return json.load(r)+++def report(base, index, field, label, queries):+ print(f"\n=== {label} ({base} / {index}) ===")+ for q, expect in queries:+ try:+ res = search(base, index, field, q)+ except Exception as e:+ print(f" {q!r}: ERROR {e}")+ continue+ hits = res["hits"]["hits"]+ if not hits:+ print(f" {q!r}: no hits")+ continue+ scores = [h["_score"] for h in hits]+ top = scores[0]+ tenth = scores[min(9, len(scores) - 1)]+ spread = top / tenth if tenth else float("inf")+ names = [+ h["_source"].get("title")+ or h["_source"].get("ax_path")+ or h["_source"].get("path")+ for h in hits+ ]+ rank = next((i + 1 for i, n in enumerate(names) if n and expect in n), None)+ print(f" query: {q!r}")+ print(+ f" expect~{expect} rank={rank} spread(top/10th)={spread:.4f} "+ f"top={top:.5f} 10th={tenth:.5f} n={len(hits)}"+ )+ print(f" top5: {names[:5]}")+++if __name__ == "__main__":+ base, index, field, label = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]+ report(base, index, field, label, QUERIES)