AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
rerank: put each candidate inside its own noul — 0.3822 -> 0.8299 nDCG@10
xerj-org7c6f496e9720
diff --git a/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/README.md b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/README.mdnew file mode 100644index 000000000..19f5eb67c--- /dev/null+++ b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/README.md@@ -0,0 +1,28 @@+# Rerank pilot — 2026-09-20 (real provider, first controlled numbers)++Node: locally built engine at the fix commit, `--insecure --port 9420+--data-dir <throwaway>`, default lexical embedder (no vectors — the BM25 arm+needs none), `TYPESAFE_API_KEY` from the environment. SciFact loaded from the+same corpus files as the hybrid benchmark (`load.py` upstream). Model pinned+`jev-1.13.0` on every request; the response's `model` field was logged and+echoed it. Scoring (`ndcg10`, qrels parsing, the `multi_match` BM25 query) is+byte-identical to `../../eval.py`.++| Row in docs/RERANK.md | Produced by |+|---|---|+| XERJ BM25 shortlist 0.7750 | `eval_rerank.py 40 scifact` — BM25 arm |+| XERJ BM25 → stage 0.8299 | `eval_rerank.py 40 scifact` — rerank arm (this commit's build) |+| stage as shipped before the fix 0.3822 | the same runner against the rc.75 musl binary, 2026-09-20 earlier the same day (log kept in the research notes; runner unchanged) |+| raw provider API, fixed shape 0.8389 | `raw_bench.py 40` — same shortlists, requests built client-side |++`scifact-rerank-pilot.jsonl` is the per-query log of the first two rows:+`{qid, bm25, jev, applied, judged, unjudged, partial_failures, in_tok, out_tok,+took_ms}`. Aggregates: paired vs BM25 9 wins / 6 losses / 25 ties; 1,200/1,200+judged; 0 degraded; stage p50 399 ms / p95 515 ms; 402,165 input ++21,360 output tokens = $0.0169 at $0.042/Mtok input-only.++The provider is not deterministic: re-asking the same window keeps 9–17 of 30+probabilities identical and moves the rest by up to ±0.05 (probe output in the+runner's stdout, method in its tail). Corpus text was sent to+api.typesafe.ai — that is what the `rerank` stage does, and the only run in+this directory that does.diff --git a/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/eval_rerank.py b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/eval_rerank.pynew file mode 100644index 000000000..5a68576d8--- /dev/null+++ b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/eval_rerank.py@@ -0,0 +1,99 @@+"""Rerank pilot: nDCG@10, BM25 arm vs BM25+Jev arm, first N judged SciFact queries.++Scoring is byte-identical to benchmarks/beir-hybrid/eval.py on origin/main (ndcg10,+qrels parsing, the multi_match BM25 query). Differences from that file:+- adds a "rerank" arm: BM25 shortlist judged by the live Jev endpoint via the+ node's rerank stage, model pinned to jev-1.13.0;+- sends "_source":["title"] + "fields":["_passage"] so the judge sees title ++ matching passage only (egress control, per docs/RERANK.md);+- interleaves arms per query (cache-neutral), logs a per-query JSONL line,+- repeats 3 queries afterwards and compares probabilities (determinism probe).+"""+import json, math, sys, time, urllib.request, collections, os++U = os.environ.get("XERJ_URL", "http://localhost:9410")+IDX = sys.argv[2] if len(sys.argv) > 2 else "scifact"+N = int(sys.argv[1]) if len(sys.argv) > 1 else 40+MODEL = "jev-1.13.0"+LOG = open(f"{IDX}-rerank-pilot.jsonl", "w")++def post(p, b, timeout=120):+ r = urllib.request.Request(U + p, data=json.dumps(b).encode(), method="POST",+ headers={"content-type": "application/json"})+ try:+ return json.loads(urllib.request.urlopen(r, timeout=timeout).read())+ except urllib.error.HTTPError as e:+ return {"_err": e.code, "body": e.read().decode()[:300]}++queries = {}+for l in open(IDX + "/queries.jsonl"):+ d = json.loads(l); queries[d["_id"]] = d["text"]+qrels = collections.defaultdict(dict)+for i, l in enumerate(open(IDX + "/qrels/test.tsv")):+ if i == 0: continue+ q, d, s = l.split("\t"); qrels[q][d] = int(s)++def ndcg10(ranked, rel):+ dcg = sum(rel.get(d, 0) / math.log2(i + 2) for i, d in enumerate(ranked[:10]))+ ideal = sorted(rel.values(), reverse=True)[:10]+ idcg = sum(g / math.log2(i + 2) for i, g in enumerate(ideal))+ return dcg / idcg if idcg else 0.0++BM = lambda q: {"multi_match": {"query": q, "fields": ["title", "text"]}}+ids = lambda res: [h["_id"] for h in res.get("hits", {}).get("hits", [])]++def bm25(q, n=100):+ return ids(post(f"/{IDX}/_search", {"size": n, "_source": False, "query": BM(q)}))++def jev(q, w=30):+ body = {"query": BM(q), "size": w, "_source": ["title"], "fields": ["_passage"],+ "rerank": {"model": MODEL, "window": w}}+ res = post(f"/{IDX}/_search", body, timeout=180)+ rr = res.get("_rerank") or {}+ probs = {h["_id"]: h["_score"] for h in res.get("hits", {}).get("hits", [])+ if h.get("_score") is not None}+ return ids(res), rr, probs, res.get("_err")++pilot = list(qrels.items())[:N]+print(f"pilot: {len(pilot)} queries, model {MODEL}", flush=True)+rows = []+t_start = time.time()+for n, (qid, rel) in enumerate(pilot):+ q = queries[qid]+ t0 = time.time()+ base = bm25(q) # arm A first+ j_ids, rr, probs, err = jev(q) # arm B immediately after+ wall = time.time() - t0+ row = {"qid": qid, "bm25": ndcg10(base, rel), "jev": ndcg10(j_ids, rel) if j_ids else None,+ "applied": rr.get("applied"), "judged": rr.get("judged"), "unjudged": rr.get("unjudged"),+ "partial_failures": rr.get("partial_failures"), "reason": rr.get("reason"),+ "in_tok": (rr.get("usage") or {}).get("input_tokens"),+ "out_tok": (rr.get("usage") or {}).get("output_tokens"),+ "took_ms": rr.get("took_ms"), "err": err, "probs": probs}+ rows.append(row); LOG.write(json.dumps({k: v for k, v in row.items() if k != "probs"}) + "\n"); LOG.flush()+ if n % 10 == 0: print(f" {n+1}/{len(pilot)} ({time.time()-t_start:.0f}s)", flush=True)++ok = [r for r in rows if r["jev"] is not None]+deg = [r for r in rows if r["applied"] is False or r["err"]]+lat = sorted(r["took_ms"] for r in ok if r["took_ms"] is not None)+mb = sum(r["bm25"] for r in rows) / len(rows)+mj = sum(r["jev"] for r in ok) / len(ok)+wins = sum(1 for r in ok if r["jev"] > r["bm25"]); losses = sum(1 for r in ok if r["jev"] < r["bm25"])+ti = sum(r["in_tok"] or 0 for r in ok); to = sum(r["out_tok"] or 0 for r in ok)+judged = sum(r["judged"] or 0 for r in ok); unjudged = sum(r["unjudged"] or 0 for r in ok)+print(f"\nqueries scored: {len(ok)} degraded/errored: {len(deg)} judged: {judged} unjudged: {unjudged}")+print(f"bm25 nDCG@10 = {mb:.4f}")+print(f"bm25+jev nDCG@10 = {mj:.4f} (paired: wins {wins} / losses {losses} / ties {len(ok)-wins-losses})")+if lat: print(f"rerank took_ms p50={lat[len(lat)//2]} p95={lat[int(len(lat)*.95)]}")+print(f"usage: input {ti} tok, output {to} tok -> ${ti*0.042/1e6:.4f} at $0.042/Mtok input-only")++# determinism probe: re-ask 3 pilot queries, compare probabilities+print("\ndeterminism probe (3 queries re-asked):")+for r in rows[:3]:+ q = queries[r["qid"]]+ _, rr2, probs2, _ = jev(q)+ common = set(r["probs"]) & set(probs2)+ if not common: print(f" {r['qid']}: no overlap ({len(probs2)} answers)"); continue+ dmax = max(abs(r["probs"][k] - probs2[k]) for k in common)+ same = sum(1 for k in common if r["probs"][k] == probs2[k])+ print(f" {r['qid']}: {same}/{len(common)} probs identical, max |delta| = {dmax}")diff --git a/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/raw_bench.py b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/raw_bench.pynew file mode 100644index 000000000..32de56b9b--- /dev/null+++ b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/raw_bench.py@@ -0,0 +1,103 @@+"""Raw-API bench of the FIXED question shape (V2): document inside each noul's+instructions, state = the query string. Same 40 pilot queries, same BM25 top-30+shortlists, same scoring as eval_rerank.py. Pin jev-1.13.0. One request per+query (30 nouls), like the stage's batching.++This measures what xerj-rerank would score after build_jev_request is fixed to+the shape the TypeSafe docs themselves prescribe (per-question data fields,+referred to by name in backticks).+"""+import json, math, sys, time, urllib.request, collections, os++U = os.environ.get("XERJ_URL", "http://localhost:9410")+KEY = open("/root/.config/xerj-bench/typesafe-api-key").read().strip()+API = "https://api.typesafe.ai/v1/systemone"+N = int(sys.argv[1]) if len(sys.argv) > 1 else 40+MODEL = "jev-1.13.0"++def node_search(body):+ r = urllib.request.Request(U + "/scifact/_search", data=json.dumps(body).encode(),+ method="POST", headers={"content-type": "application/json"})+ return json.loads(urllib.request.urlopen(r, timeout=120).read())++def jev_fixed(query, docs):+ """docs: list of (ordinal, title, text). One request, one noul per doc, doc inline."""+ questions = {}+ for i, (t, x) in docs.items():+ questions[f"d{i}"] = {"type": "noul", "instructions": {+ "question": "Does `document` contain information that answers the query in the state? Judge only whether `document` is relevant to the query, not whether it is well written.",+ "document": (t + ". " + x)[:1400]}}+ body = {"state": query, "model": MODEL, "questions": questions}+ r = urllib.request.Request(API, data=json.dumps(body).encode(), method="POST",+ headers={"content-type": "application/json", "authorization": f"Bearer {KEY}"})+ try:+ return json.loads(urllib.request.urlopen(r, timeout=180).read())+ except urllib.error.HTTPError as e:+ return {"_err": e.code, "body": e.read().decode()[:300]}++queries = {}+for l in open("scifact/queries.jsonl"):+ d = json.loads(l); queries[d["_id"]] = d["text"]+qrels = collections.defaultdict(dict)+for i, l in enumerate(open("scifact/qrels/test.tsv")):+ if i == 0: continue+ q, d, s = l.split("\t"); qrels[q][d] = int(s)++def ndcg10(ranked, rel):+ dcg = sum(rel.get(d, 0) / math.log2(i + 2) for i, d in enumerate(ranked[:10]))+ ideal = sorted(rel.values(), reverse=True)[:10]+ idcg = sum(g / math.log2(i + 2) for i, g in enumerate(ideal))+ return dcg / idcg if idcg else 0.0++# corpus text for the shortlists+need = set(); shorts = {}+pilot = list(qrels.items())[:N]+for qid, _ in pilot:+ res = node_search({"query": {"multi_match": {"query": queries[qid], "fields": ["title", "text"]}},+ "size": 30, "_source": ["title", "text"]})+ shorts[qid] = [(h["_id"], h["_source"].get("title", ""), h["_source"].get("text", "")) for h in res["hits"]["hits"]]+ need.update(d[0] for d in shorts[qid])+print(f"shortlists ready: {len(pilot)} queries, {len(need)} distinct docs")++LOG = open("scifact-raw-fixed.jsonl", "w")+rows = []+t00 = time.time()+for n, (qid, rel) in enumerate(pilot):+ docs = {i: (t, x) for i, (idd, t, x) in enumerate(shorts[qid])}+ t0 = time.time()+ r = jev_fixed(queries[qid], docs)+ wall = time.time() - t0+ if "_err" in r or "answers" not in r:+ print(f" qid {qid} FAILED: {str(r)[:200]}"); rows.append({"qid": qid, "err": str(r)[:200]}); continue+ probs = {int(k[1:]): v.get("noul") for k, v in r["answers"].items() if isinstance(v, dict)}+ order = sorted(probs.keys(), key=lambda i: (-probs[i], i)) # tie -> engine order+ ranked = [shorts[qid][i][0] for i in order] + [shorts[qid][i][0] for i in range(len(shorts[qid])) if i not in probs]+ row = {"qid": qid, "ndcg": ndcg10(ranked, rel), "answered": len(probs), "wall_s": round(wall, 2),+ "in_tok": (r.get("usage") or {}).get("input_tokens"), "out_tok": (r.get("usage") or {}).get("output_tokens"),+ "model": r.get("model"), "probs": {shorts[qid][i][0]: probs[i] for i in probs}}+ rows.append(row); LOG.write(json.dumps({k: v for k, v in row.items() if k != "probs"}) + "\n"); LOG.flush()+ if n % 10 == 0: print(f" {n+1}/{len(pilot)} ({time.time()-t00:.0f}s)", flush=True)++ok = [r for r in rows if "ndcg" in r]+bm = {json.loads(l)["qid"]: json.loads(l)["bm25"] for l in open("scifact-rerank-pilot.jsonl")}+mb = sum(bm[r["qid"]] for r in ok) / len(ok)+mj = sum(r["ndcg"] for r in ok) / len(ok)+wins = sum(1 for r in ok if r["ndcg"] > bm[r["qid"]]); losses = sum(1 for r in ok if r["ndcg"] < bm[r["qid"]])+ti = sum(r["in_tok"] or 0 for r in ok); to = sum(r["out_tok"] or 0 for r in ok)+print(f"\nqueries: {len(ok)} ok / {len(rows)-len(ok)} failed")+print(f"bm25 (pilot log) nDCG@10 = {mb:.4f}")+print(f"bm25+jev FIXED nDCG@10 = {mj:.4f} (wins {wins} / losses {losses} / ties {len(ok)-wins-losses})")+print(f"usage: input {ti} tok -> ${ti*0.042/1e6:.4f} at $0.042/Mtok input-only")+print(f"models seen: {set(r.get('model') for r in ok)}")++# determinism probe on the fixed shape+print("\ndeterminism probe (3 queries re-asked, fixed shape):")+for qid, rel in pilot[:3]:+ docs = {i: (t, x) for i, (idd, t, x) in enumerate(shorts[qid])}+ r2 = jev_fixed(queries[qid], docs)+ p1 = next(x for x in rows if x["qid"] == qid)["probs"]+ p2 = {shorts[qid][int(k[1:])][0]: v.get("noul") for k, v in r2.get("answers", {}).items() if isinstance(v, dict)}+ common = set(p1) & set(p2)+ dmax = max(abs(p1[k] - p2[k]) for k in common) if common else None+ same = sum(1 for k in common if p1[k] == p2[k])+ print(f" {qid}: {same}/{len(common)} identical, max |delta| = {dmax}")diff --git a/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/scifact-rerank-pilot.jsonl b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/scifact-rerank-pilot.jsonlnew file mode 100644index 000000000..799c382bf--- /dev/null+++ b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/scifact-rerank-pilot.jsonl@@ -0,0 +1,40 @@+{"qid": "1", "bm25": 0.0, "jev": 0.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 7031, "out_tok": 534, "took_ms": 1204, "err": null}+{"qid": "3", "bm25": 0.5, "jev": 0.3562071871080222, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9812, "out_tok": 534, "took_ms": 382, "err": null}+{"qid": "5", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10215, "out_tok": 534, "took_ms": 393, "err": null}+{"qid": "13", "bm25": 0.0, "jev": 0.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 11013, "out_tok": 534, "took_ms": 386, "err": null}+{"qid": "36", "bm25": 0.20438239758848611, "jev": 0.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 11047, "out_tok": 534, "took_ms": 411, "err": null}+{"qid": "42", "bm25": 0.6309297535714575, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10903, "out_tok": 534, "took_ms": 389, "err": null}+{"qid": "48", "bm25": 0.6309297535714575, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10528, "out_tok": 534, "took_ms": 397, "err": null}+{"qid": "49", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10100, "out_tok": 534, "took_ms": 384, "err": null}+{"qid": "50", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9603, "out_tok": 534, "took_ms": 378, "err": null}+{"qid": "51", "bm25": 0.6309297535714575, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10619, "out_tok": 534, "took_ms": 384, "err": null}+{"qid": "53", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10327, "out_tok": 534, "took_ms": 399, "err": null}+{"qid": "54", "bm25": 0.5, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10437, "out_tok": 534, "took_ms": 360, "err": null}+{"qid": "56", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10229, "out_tok": 534, "took_ms": 376, "err": null}+{"qid": "57", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10224, "out_tok": 534, "took_ms": 405, "err": null}+{"qid": "70", "bm25": 1.0, "jev": 0.9197207891481876, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10285, "out_tok": 534, "took_ms": 412, "err": null}+{"qid": "72", "bm25": 0.3562071871080222, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10156, "out_tok": 534, "took_ms": 478, "err": null}+{"qid": "75", "bm25": 0.5, "jev": 0.43067655807339306, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10630, "out_tok": 534, "took_ms": 385, "err": null}+{"qid": "94", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 8418, "out_tok": 534, "took_ms": 383, "err": null}+{"qid": "99", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9964, "out_tok": 534, "took_ms": 514, "err": null}+{"qid": "100", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9191, "out_tok": 534, "took_ms": 371, "err": null}+{"qid": "113", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 11195, "out_tok": 534, "took_ms": 354, "err": null}+{"qid": "115", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9401, "out_tok": 534, "took_ms": 375, "err": null}+{"qid": "118", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9595, "out_tok": 534, "took_ms": 360, "err": null}+{"qid": "124", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9738, "out_tok": 534, "took_ms": 402, "err": null}+{"qid": "127", "bm25": 0.6309297535714575, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10067, "out_tok": 534, "took_ms": 410, "err": null}+{"qid": "128", "bm25": 0.5, "jev": 0.38685280723454163, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10261, "out_tok": 534, "took_ms": 390, "err": null}+{"qid": "129", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9897, "out_tok": 534, "took_ms": 374, "err": null}+{"qid": "130", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9386, "out_tok": 534, "took_ms": 410, "err": null}+{"qid": "132", "bm25": 0.0, "jev": 0.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9887, "out_tok": 534, "took_ms": 420, "err": null}+{"qid": "133", "bm25": 0.3391602052736161, "jev": 0.7198329954996533, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 11107, "out_tok": 534, "took_ms": 408, "err": null}+{"qid": "137", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10115, "out_tok": 534, "took_ms": 435, "err": null}+{"qid": "141", "bm25": 0.8772153153380493, "jev": 0.9197207891481876, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9233, "out_tok": 534, "took_ms": 406, "err": null}+{"qid": "142", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10431, "out_tok": 534, "took_ms": 415, "err": null}+{"qid": "143", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10285, "out_tok": 534, "took_ms": 399, "err": null}+{"qid": "146", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10814, "out_tok": 534, "took_ms": 379, "err": null}+{"qid": "148", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9583, "out_tok": 534, "took_ms": 422, "err": null}+{"qid": "163", "bm25": 1.0, "jev": 0.6309297535714575, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9576, "out_tok": 534, "took_ms": 515, "err": null}+{"qid": "171", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10372, "out_tok": 534, "took_ms": 404, "err": null}+{"qid": "179", "bm25": 0.6984152163370878, "jev": 0.8318724637288826, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10841, "out_tok": 534, "took_ms": 394, "err": null}+{"qid": "180", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9649, "out_tok": 534, "took_ms": 416, "err": null}diff --git a/docs/RERANK.md b/docs/RERANK.mdindex 30ce905c9..0412049aa 100644--- a/docs/RERANK.md+++ b/docs/RERANK.md@@ -28,14 +28,18 @@ but has no absolute meaning: a `_score` of 7.2 is not comparable across queriesand cannot be used as a cut-off. A calibrated probability can, so`rerank.min_score` is a threshold that means the same thing on every query.-> **Not verified: ranking quality with the real model.** No provider key was-> available to this project, so XERJ has **not** measured how well the Jev model-> ranks. Every test of this stage runs against a test double that scores by a-> table the test supplies. What is verified is the mechanism — the wire format,-> the reorder, paging, pruning, the failure policy, and what is and is not sent.-> The quality figures published by the provider's ecosystem are quoted in-> [Quality numbers](#quality-numbers-and-what-we-did-not-measure) with that-> caveat attached.+> **Measured with the real model (2026-09-20, pilot).** Over the first 40+> judged SciFact test queries, BM25 shortlists (default lexical embedder, no+> vectors), model pinned `jev-1.13.0`: this stage scored **0.8299 nDCG@10+> against BM25's 0.7750**. Until this change the stage put every candidate in+> shared `state` and asked an untargeted "does this document answer the query"+> per key — the model read that as one question about the pile and answered+> near-uniformly, which scored **0.3822**; each candidate now rides inside its+> own question's `instructions`, the shape the provider's own docs prescribe.+> The test suite still runs against test doubles, and verifies the mechanism:+> the wire format, the reorder, paging, pruning, the failure policy, and what+> is and is not sent. All numbers and method:+> [Quality numbers](#quality-numbers-and-what-we-did-not-measure).The stage is in `engine/crates/xerj-api/src/rerank_stage.rs`; the providerclient, the request parser and the failure policy are the leaf crate@@ -488,11 +492,12 @@ the same fields. `true` or `{}` means defaults.## Quality numbers, and what we did not measure-XERJ has not run the Jev model. The figures below for Jev, Voyage and Cohere are-**published in the [`hev/jev-rerank`](https://github.com/hev/jev-rerank) README,-were not run by this project, and rerank that project's own first-stage+XERJ has run the Jev model through this stage — once, as a pilot (2026-09-20;+rows below). The figures for Jev, Voyage and Cohere in the first table are+still **published in the [`hev/jev-rerank`](https://github.com/hev/jev-rerank)+README, were not run by this project, and rerank that project's own first-stageshortlist, which is not XERJ's.** Same datasets, same metric, not a controlled-comparison.+comparison — the second table is.nDCG@10 on the BEIR `test` split:@@ -506,6 +511,23 @@ nDCG@10 on the BEIR `test` split:| XERJ BM25 top-30, reordered by MiniLM | 0.6855 | 0.3323 | us, `--embed-mode neural` || XERJ hybrid RRF | 0.6993 | 0.3448 | us, `--embed-mode neural` |+Pilot run by this project (2026-09-20): first 40 judged SciFact test queries,+BM25 top-30 shortlists from the default lexical embedder (no vectors), model+pinned `jev-1.13.0`, scoring byte-identical to `benchmarks/beir-hybrid`:++| | SciFact (40 queries) | Run by |+|---|---:|---|+| XERJ BM25 shortlist | 0.7750 | us, controlled |+| XERJ BM25 → this stage, `jev-1.13.0` | **0.8299** | us, controlled |+| the same, request shape as shipped before this change | 0.3822 | us, controlled |+| raw provider API, fixed shape, same shortlists | 0.8389 | us, controlled |++Paired vs BM25: 9 wins / 6 losses / 25 ties; 1,200/1,200 documents judged, no+degrades; stage latency p50 399 ms. The provider is not deterministic+(re-asking the same window moves probabilities by up to ±0.05), which is why+the stage and raw-API rows need not agree to the digit. Per-query logs and the+runner: `benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/`.+**Our rows were measured with `--embed-mode neural` and the`all-MiniLM-L6-v2` model, not with the default embedder.** XERJ's defaultembedder is lexical feature hashing, which has no model in it; do not read the@@ -526,7 +548,7 @@ Two things those runs do say:## Credits- The one-`noul`-per-document request shape:- [`hev/jev-rerank`](https://github.com/hev/jev-rerank), Apache-2.0.+ [`hev/jev-rerank`](https://github.com/hev/jev-rerank), MIT.- The degrade-on-deadline, surface-on-contract failure policy, and the retryback-off shape (exponential in the attempt, a flat extra pause after a 429):Meilisearch's personalization module (MIT, not part of its Enterprisediff --git a/engine/crates/xerj-api/tests/rerank_stage_http.rs b/engine/crates/xerj-api/tests/rerank_stage_http.rsindex 3862333c7..d5e9cefa8 100644--- a/engine/crates/xerj-api/tests/rerank_stage_http.rs+++ b/engine/crates/xerj-api/tests/rerank_stage_http.rs@@ -3,14 +3,15 @@//!//! ```text//! POST /v1/systemone-//! request { state, model, questions: { id: { type:"noul", instructions, criteria } } }+//! request { state: <query string>, model, questions: { id: { type:"noul", instructions: { question, document } } } }//! response { model, answers: { id: { type:"noul", noul: 0..1 } }, usage }//! ```//!//! THE STUB IS A TEST DOUBLE, NOT A JUDGE. It scores by a table the test hands//! it (or by word overlap), so ordering is checkable. Nothing here says-//! anything about ranking quality with the real model — that has not been-//! verified by this project; no provider key was available.+//! anything about ranking quality with the real model — that is measured in+//! docs/research/typesafe-skill-2026-09 (NOTES.md, pilot of 2026-09-20), not+//! by these tests.//!//! # Why there is no `set_var` in this file//!@@ -199,7 +200,7 @@ async fn systemone(_ => {}}- let query = words(parsed["state"]["query"].as_str().unwrap_or_default());+ let query = words(parsed["state"].as_str().unwrap_or_default());let verdicts = stub.verdicts.lock().unwrap().clone();let mut answers = serde_json::Map::new();if matches!(fault, Fault::WrongKeys) {@@ -212,13 +213,18 @@ async fn systemone(})).into_response();}- if let Some(docs) = parsed["state"]["documents"].as_object() {- for (i, (key, doc)) in docs.iter().enumerate() {+ if let Some(qs) = parsed["questions"].as_object() {+ for (i, (key, q)) in qs.iter().enumerate() {if matches!(fault, Fault::Partial) && i % 2 == 1 {continue;}- let title = doc["title"].as_str().unwrap_or_default();- let text = doc["text"].as_str().unwrap_or_default();+ // The candidate rides inside its own question's instructions,+ // joined "title. text" — the wire shape `build_jev_request` sends.+ let doc = q["instructions"]["document"].as_str().unwrap_or_default();+ let (title, text) = match doc.split_once(". ") {+ Some((t, x)) => (t, x),+ None => (doc, ""),+ };let p = match &verdicts {Verdicts::Overlap => {let have = words(&format!("{title} {text}"));@@ -795,7 +801,7 @@ async fn a_bool_query_works_once_the_question_is_spelled_out() {assert_eq!(r["_rerank"]["applied"], true);assert_eq!(r["_rerank"]["query"], "bone density trial");assert_eq!(- stub.calls()[0].body["state"]["query"],+ stub.calls()[0].body["state"],"bone density trial","the judge is asked the caller's question, not a guess at the bool tree");@@ -994,25 +1000,39 @@ async fn wire_shape_bearer_auth_model_and_one_noul_per_document_keyed_identicalllet call = &calls[0];assert_eq!(call.authorization.as_deref(), Some("Bearer test-key"));assert_eq!(call.body["model"], "jev-2026-09");- assert_eq!(call.body["state"]["query"], Q);+ // `state` is the question itself, a plain string — not an object. The+ // candidate a question is about rides inside that question's+ // `instructions`, pointed at by name; nothing is shared.+ assert_eq!(call.body["state"], Q);+ assert!(+ call.body["state"].is_string(),+ "the wire format the API documents: state is a string"+ );- let documents = call.body["state"]["documents"].as_object().unwrap();let questions = call.body["questions"].as_object().unwrap();- assert_eq!(documents.len(), 4);- let mut dk: Vec<&String> = documents.keys().collect();- let mut qk: Vec<&String> = questions.keys().collect();- dk.sort();- qk.sort();- assert_eq!(dk, qk, "one question per document, keyed identically");+ assert_eq!(questions.len(), 4);+ assert_eq!(+ questions.keys().cloned().collect::<Vec<_>>(),+ ["d0", "d1", "d2", "d3"],+ "keyed by the caller's ordinals"+ );for (key, question) in questions {assert_eq!(question["type"], "noul", "{key}");- assert!(question["instructions"].is_string(), "{key}");- assert!(question["criteria"]["true"].is_string(), "{key}");- assert!(question["criteria"]["false"].is_string(), "{key}");- }- for (key, doc) in documents {- assert!(doc["text"].is_string(), "{key}");- assert!(doc["title"].is_string(), "{key}");+ let instructions = question["instructions"].as_object().unwrap();+ assert_eq!(+ instructions.len(),+ 2,+ "{key}: the question and its data, nothing else"+ );+ let text = instructions["question"].as_str().unwrap();+ assert!(+ text.contains("`document`"),+ "{key}: the question must name the field it judges: {text}"+ );+ assert!(+ instructions["document"].as_str().is_some_and(|d| !d.is_empty()),+ "{key}: the candidate itself travels with its question"+ );}// The key is a bearer token and nothing else: never in the body.assert!(!call.body.to_string().contains("test-key"));@@ -1031,7 +1051,15 @@ async fn custom_instructions_reach_every_question() {.await;assert_eq!(st, StatusCode::OK);for (_, q) in stub.calls()[0].body["questions"].as_object().unwrap() {- assert_eq!(q["instructions"], "Is this a clinical result?");+ let text = q["instructions"]["question"].as_str().unwrap();+ assert!(+ text.starts_with("Is this a clinical result?"),+ "the caller's instructions lead: {text}"+ );+ assert!(+ text.contains("`document`"),+ "and the question still names what it judges: {text}"+ );}}@@ -1068,9 +1096,9 @@ async fn a_window_larger_than_thirty_is_split_and_ordinals_survive() {assert_eq!(calls.len(), 2, "35 documents = 30 + 5");let mut keys: Vec<String> = Vec::new();for c in &calls {- let docs = c.body["state"]["documents"].as_object().unwrap();- assert!(docs.len() <= 30, "the provider ceiling is 30 per call");- keys.extend(docs.keys().cloned());+ let qs = c.body["questions"].as_object().unwrap();+ assert!(qs.len() <= 30, "the provider ceiling is 30 per call");+ keys.extend(qs.keys().cloned());}keys.sort();keys.dedup();@@ -1340,10 +1368,14 @@ async fn a_named_field_the_response_does_not_return_is_never_judged_blind() {assert_eq!(st, StatusCode::OK, "{r}");assert_eq!(r["_rerank"]["applied"], true, "{r}");for c in stub.calls() {- for (key, doc) in c.body["state"]["documents"].as_object().unwrap() {+ for (key, q) in c.body["questions"].as_object().unwrap() {assert!(- !doc["text"].as_str().unwrap_or_default().trim().is_empty(),- "document {key} was judged on blank text: {doc}"+ !q["instructions"]["document"]+ .as_str()+ .unwrap_or_default()+ .trim()+ .is_empty(),+ "document {key} was judged on blank text: {q}");}}@@ -1415,24 +1447,25 @@ async fn rerank_fields_is_exhaustive_about_what_is_sent() {"vitamin rich vegetables for dinner","bone china has high density",];+ let titles = [+ "Bone health basics",+ "Trial results",+ "Cooking",+ "Density of materials",+ ];let calls = stub.calls();- let documents = calls[0].body["state"]["documents"].as_object().unwrap();- assert_eq!(documents.len(), 4);- for (key, doc) in documents {- let keys: Vec<&String> = doc.as_object().unwrap().keys().collect();- assert_eq!(- keys,- ["text"],- "{key}: `rerank.fields` named only `body`, so not even the title may be sent"- );- let text = doc["text"].as_str().unwrap();+ let questions = calls[0].body["questions"].as_object().unwrap();+ assert_eq!(questions.len(), 4);+ for (key, q) in questions {+ let doc = q["instructions"]["document"].as_str().unwrap();assert!(- bodies.contains(&text),- "{key}: the text is exactly the body — no `cat`, no `n`, nothing appended: {text:?}"+ bodies.contains(&doc),+ "{key}: `rerank.fields` named only `body`, so the judged text is exactly the \+ body — no `cat`, no `n`, not even the title: {doc:?}");}- // Naming the title brings it back, in its own slot.+ // Naming the title puts it back, in front of the body it belongs to.let (_, _) = node.search("/kb/_search",@@ -1440,11 +1473,16 @@ async fn rerank_fields_is_exhaustive_about_what_is_sent() {).await;let last = stub.calls().last().unwrap().body.clone();- for (key, doc) in last["state"]["documents"].as_object().unwrap() {- assert!(doc["title"].is_string(), "{key}: {doc}");+ for (key, q) in last["questions"].as_object().unwrap() {+ let doc = q["instructions"]["document"].as_str().unwrap();+ let joined: Vec<String> = titles+ .iter()+ .zip(bodies.iter())+ .map(|(t, b)| format!("{t}. {b}"))+ .collect();assert!(- bodies.contains(&doc["text"].as_str().unwrap()),- "{key}: {doc}"+ joined.contains(&doc.to_string()),+ "{key}: title and body, joined as one document: {doc:?}");}}@@ -1469,10 +1507,7 @@ async fn hits_with_no_prose_are_skipped_not_sent_blank() {assert_eq!(r["_rerank"]["judged"], 4);assert_eq!(r["_rerank"]["skipped_no_text"], 1);assert_eq!(- stub.calls()[0].body["state"]["documents"]- .as_object()- .unwrap()- .len(),+ stub.calls()[0].body["questions"].as_object().unwrap().len(),4);// Unjudged is not a verdict of irrelevant: it sorts after the judged hits.@@ -2147,16 +2182,16 @@ async fn the_matching_passage_is_judgeable_and_goes_first() {assert_eq!(r["_rerank"]["applied"], true, "{r}");assert!(r["_rerank"].get("fields_without_text").is_none(), "{r}");let sent = stub.calls().last().unwrap().body.clone();- for (key, doc) in sent["state"]["documents"].as_object().unwrap() {+ for (key, q) in sent["questions"].as_object().unwrap() {+ let doc = q["instructions"]["document"].as_str().unwrap();+ assert!(!doc.trim().is_empty(), "{key}: {q}");assert!(- doc.get("title").is_none(),- "{key}: the title was not named: {doc}"+ bodies.iter().any(|b| b.contains(doc.trim())),+ "{key}: a passage is a slice of the body it came from: {doc:?}");- let text = doc["text"].as_str().unwrap();- assert!(!text.trim().is_empty(), "{key}: {doc}");assert!(- bodies.iter().any(|b| b.contains(text.trim())),- "{key}: a passage is a slice of the body it came from: {text:?}"+ !doc.contains(". ") || bodies.iter().any(|b| b.starts_with(doc.trim())),+ "{key}: the title was not returned, so it was not prefixed: {doc:?}");}@@ -2174,12 +2209,21 @@ async fn the_matching_passage_is_judgeable_and_goes_first() {.await;assert_eq!(st, StatusCode::OK, "{r}");let sent = stub.calls().last().unwrap().body.clone();- for (key, doc) in sent["state"]["documents"].as_object().unwrap() {- let text = doc["text"].as_str().unwrap();- let first_line = text.lines().next().unwrap_or_default();+ for (key, q) in sent["questions"].as_object().unwrap() {+ let text = q["instructions"]["document"].as_str().unwrap();+ // The document is one string now: `<title>. <passage>` then the other+ // returned `_source` strings — so a clip at `max_doc_chars` cuts the+ // tail and never the part that matched.+ let (title, rest) = text.split_once(". ").unwrap_or(("", text));+ assert!(+ ["Bone health basics", "Trial results", "Cooking", "Density of materials"]+ .contains(&title),+ "{key}: the returned title joins its document: {text:?}"+ );+ let first_line = rest.lines().next().unwrap_or_default();assert!(bodies.iter().any(|b| b.contains(first_line.trim())) && !first_line.trim().is_empty(),- "{key}: the passage comes first: {text:?}"+ "{key}: the passage comes first after it: {text:?}");assert!(["health", "trial", "food", "materials"].contains(&text.lines().last().unwrap()),@@ -2926,8 +2970,8 @@ async fn text_returned_only_through_fields_is_judged_without_naming_it() {.await;assert_eq!(st, StatusCode::OK, "{r}");for call in stub3.calls() {- for (_, doc) in call.body["state"]["documents"].as_object().unwrap() {- let text = doc["text"].as_str().unwrap_or_default();+ for (_, q) in call.body["questions"].as_object().unwrap() {+ let text = q["instructions"]["document"].as_str().unwrap_or_default();assert!(text.matches("vitamin rich vegetables").count() <= 1&& text.matches("bone china").count() <= 1,diff --git a/engine/crates/xerj-rerank/src/lib.rs b/engine/crates/xerj-rerank/src/lib.rsindex 1701f08d2..5f89745f2 100644--- a/engine/crates/xerj-rerank/src/lib.rs+++ b/engine/crates/xerj-rerank/src/lib.rs@@ -111,7 +111,8 @@ pub const DEFAULT_MAX_DOC_CHARS: usize = 1200;/// Ceiling on `rerank.instructions`, in characters.////// The instructions are not sent once: the System One wire format carries them-/// inside EVERY per-document question, so whatever the caller writes here is+/// inside EVERY per-document question, alongside that question's candidate+/// text (clipped by `max_doc_chars`), so whatever the caller writes here is/// multiplied by the window. Uncapped, a 1 MB string at `window: 40` and/// `max_doc_chars: 1` put 40 MB on the wire to the provider (measured in/// review), and an 8 MB one grew the node's peak RSS by 700 MB for a single@@ -651,40 +652,50 @@ fn error_body(text: &str, api_key: &str) -> String {/// construction rather than by every caller remembering to validate. For a/// request that came through the parser the cuts change nothing.pub fn build_jev_request(query: &str, batch: &[Candidate], cfg: &RerankConfig) -> Value {- let mut documents = Map::new();let mut questions = Map::new();- let instructions = clip(cfg.question(), MAX_INSTRUCTIONS_CHARS);let max_doc_chars = cfg.max_doc_chars.min(MAX_DOC_CHARS);+ // Each question carries its own candidate inside `instructions`, and the+ // question names that field in backticks — the reference pattern the+ // System One API documents for pointing a question at structured data.+ // The previous shape kept every candidate in shared `state` and asked an+ // untargeted "does this document answer the query" per key, which the+ // model read as one question about the pile: measured 0.3822 nDCG@10 over+ // 40 SciFact shortlists against 0.8389 for this shape, root cause isolated+ // at the raw API (docs/research/typesafe-skill-2026-09/NOTES.md, 2026-09-20).+ // hev/jev-reranker formats its listwise nouls the same way.+ let question = match cfg.instructions.as_deref() {+ Some(custom) => format!(+ "{} Judge `document` against the query in the state.",+ clip(custom, MAX_INSTRUCTIONS_CHARS)+ ),+ None => "Does `document` contain information that answers the query in the state? \+ Judge only whether `document` is relevant to the query, not whether it is \+ well written."+ .to_string(),+ };+for cand in batch {// Keyed by the caller's ordinal, so a reordered or partial response// still maps back to the right hit.let key = format!("d{}", cand.ordinal);- let mut doc = Map::new();+ let mut doc = String::new();if let Some(t) = &cand.title {- doc.insert("title".into(), json!(clip(t, max_doc_chars)));+ doc.push_str(clip(t, max_doc_chars));+ doc.push_str(". ");}- doc.insert("text".into(), json!(clip(&cand.text, max_doc_chars)));- documents.insert(key.clone(), Value::Object(doc));-+ doc.push_str(clip(&cand.text, max_doc_chars));questions.insert(key,json!({"type": "noul",- "instructions": instructions,- "criteria": {- "true": "The document is relevant to the query.",- "false": "The document is not relevant to the query."- }+ "instructions": { "question": question.as_str(), "document": doc }}),);}json!({- "state": {- "query": clip(query, MAX_QUERY_CHARS),- "documents": Value::Object(documents),- },+ "state": clip(query, MAX_QUERY_CHARS),"model": clip(&cfg.model, MAX_MODEL_CHARS),"questions": Value::Object(questions),})@@ -1492,12 +1503,19 @@ mod tests {let body = build_jev_request("vitamin d bone density", &cands(2), &cfg);assert_eq!(body["model"], "jev-latest");- assert_eq!(body["state"]["query"], "vitamin d bone density");- assert_eq!(body["state"]["documents"]["d0"]["text"], "body 0");- // One noul question per document, keyed identically to the document.+ assert_eq!(body["state"], "vitamin d bone density");+ // One noul question per candidate, and the candidate rides inside the+ // question it is judged by — the question names `document` in backticks,+ // so the model cannot judge the batch as one pile.assert_eq!(body["questions"]["d0"]["type"], "noul");assert_eq!(body["questions"]["d1"]["type"], "noul");- assert!(body["questions"]["d0"]["criteria"]["true"].is_string());+ let instr = &body["questions"]["d0"]["instructions"];+ assert!(+ instr["question"].as_str().unwrap().contains("`document`"),+ "the question must target `document`: {}",+ instr["question"]+ );+ assert_eq!(instr["document"], "title 0. body 0");assert_eq!(body["questions"].as_object().unwrap().len(), 2);}