Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

rerank: put each candidate inside its own noul — 0.3822 -> 0.8299 nDCG@10

xerj-org7c6f496e9720
diff --git a/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/README.md b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/README.md
new file mode 100644
index 000000000..19f5eb67c
--- /dev/null
+++ b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/README.md
@@ -0,0 +1,28 @@
+# Rerank pilot — 2026-09-20 (real provider, first controlled numbers)
+
+Node: locally built engine at the fix commit, `--insecure --port 9420
+--data-dir <throwaway>`, default lexical embedder (no vectors — the BM25 arm
+needs none), `TYPESAFE_API_KEY` from the environment. SciFact loaded from the
+same corpus files as the hybrid benchmark (`load.py` upstream). Model pinned
+`jev-1.13.0` on every request; the response's `model` field was logged and
+echoed it. Scoring (`ndcg10`, qrels parsing, the `multi_match` BM25 query) is
+byte-identical to `../../eval.py`.
+
+| Row in docs/RERANK.md | Produced by |
+|---|---|
+| XERJ BM25 shortlist 0.7750 | `eval_rerank.py 40 scifact` — BM25 arm |
+| XERJ BM25 → stage 0.8299 | `eval_rerank.py 40 scifact` — rerank arm (this commit's build) |
+| stage as shipped before the fix 0.3822 | the same runner against the rc.75 musl binary, 2026-09-20 earlier the same day (log kept in the research notes; runner unchanged) |
+| raw provider API, fixed shape 0.8389 | `raw_bench.py 40` — same shortlists, requests built client-side |
+
+`scifact-rerank-pilot.jsonl` is the per-query log of the first two rows:
+`{qid, bm25, jev, applied, judged, unjudged, partial_failures, in_tok, out_tok,
+took_ms}`. Aggregates: paired vs BM25 9 wins / 6 losses / 25 ties; 1,200/1,200
+judged; 0 degraded; stage p50 399 ms / p95 515 ms; 402,165 input +
+21,360 output tokens = $0.0169 at $0.042/Mtok input-only.
+
+The provider is not deterministic: re-asking the same window keeps 9–17 of 30
+probabilities identical and moves the rest by up to ±0.05 (probe output in the
+runner's stdout, method in its tail). Corpus text was sent to
+api.typesafe.ai — that is what the `rerank` stage does, and the only run in
+this directory that does.
diff --git a/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/eval_rerank.py b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/eval_rerank.py
new file mode 100644
index 000000000..5a68576d8
--- /dev/null
+++ b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/eval_rerank.py
@@ -0,0 +1,99 @@
+"""Rerank pilot: nDCG@10, BM25 arm vs BM25+Jev arm, first N judged SciFact queries.
+
+Scoring is byte-identical to benchmarks/beir-hybrid/eval.py on origin/main (ndcg10,
+qrels parsing, the multi_match BM25 query). Differences from that file:
+- adds a "rerank" arm: BM25 shortlist judged by the live Jev endpoint via the
+ node's rerank stage, model pinned to jev-1.13.0;
+- sends "_source":["title"] + "fields":["_passage"] so the judge sees title +
+ matching passage only (egress control, per docs/RERANK.md);
+- interleaves arms per query (cache-neutral), logs a per-query JSONL line,
+- repeats 3 queries afterwards and compares probabilities (determinism probe).
+"""
+import json, math, sys, time, urllib.request, collections, os
+
+U = os.environ.get("XERJ_URL", "http://localhost:9410")
+IDX = sys.argv[2] if len(sys.argv) > 2 else "scifact"
+N = int(sys.argv[1]) if len(sys.argv) > 1 else 40
+MODEL = "jev-1.13.0"
+LOG = open(f"{IDX}-rerank-pilot.jsonl", "w")
+
+def post(p, b, timeout=120):
+ r = urllib.request.Request(U + p, data=json.dumps(b).encode(), method="POST",
+ headers={"content-type": "application/json"})
+ try:
+ return json.loads(urllib.request.urlopen(r, timeout=timeout).read())
+ except urllib.error.HTTPError as e:
+ return {"_err": e.code, "body": e.read().decode()[:300]}
+
+queries = {}
+for l in open(IDX + "/queries.jsonl"):
+ d = json.loads(l); queries[d["_id"]] = d["text"]
+qrels = collections.defaultdict(dict)
+for i, l in enumerate(open(IDX + "/qrels/test.tsv")):
+ if i == 0: continue
+ q, d, s = l.split("\t"); qrels[q][d] = int(s)
+
+def ndcg10(ranked, rel):
+ dcg = sum(rel.get(d, 0) / math.log2(i + 2) for i, d in enumerate(ranked[:10]))
+ ideal = sorted(rel.values(), reverse=True)[:10]
+ idcg = sum(g / math.log2(i + 2) for i, g in enumerate(ideal))
+ return dcg / idcg if idcg else 0.0
+
+BM = lambda q: {"multi_match": {"query": q, "fields": ["title", "text"]}}
+ids = lambda res: [h["_id"] for h in res.get("hits", {}).get("hits", [])]
+
+def bm25(q, n=100):
+ return ids(post(f"/{IDX}/_search", {"size": n, "_source": False, "query": BM(q)}))
+
+def jev(q, w=30):
+ body = {"query": BM(q), "size": w, "_source": ["title"], "fields": ["_passage"],
+ "rerank": {"model": MODEL, "window": w}}
+ res = post(f"/{IDX}/_search", body, timeout=180)
+ rr = res.get("_rerank") or {}
+ probs = {h["_id"]: h["_score"] for h in res.get("hits", {}).get("hits", [])
+ if h.get("_score") is not None}
+ return ids(res), rr, probs, res.get("_err")
+
+pilot = list(qrels.items())[:N]
+print(f"pilot: {len(pilot)} queries, model {MODEL}", flush=True)
+rows = []
+t_start = time.time()
+for n, (qid, rel) in enumerate(pilot):
+ q = queries[qid]
+ t0 = time.time()
+ base = bm25(q) # arm A first
+ j_ids, rr, probs, err = jev(q) # arm B immediately after
+ wall = time.time() - t0
+ row = {"qid": qid, "bm25": ndcg10(base, rel), "jev": ndcg10(j_ids, rel) if j_ids else None,
+ "applied": rr.get("applied"), "judged": rr.get("judged"), "unjudged": rr.get("unjudged"),
+ "partial_failures": rr.get("partial_failures"), "reason": rr.get("reason"),
+ "in_tok": (rr.get("usage") or {}).get("input_tokens"),
+ "out_tok": (rr.get("usage") or {}).get("output_tokens"),
+ "took_ms": rr.get("took_ms"), "err": err, "probs": probs}
+ rows.append(row); LOG.write(json.dumps({k: v for k, v in row.items() if k != "probs"}) + "\n"); LOG.flush()
+ if n % 10 == 0: print(f" {n+1}/{len(pilot)} ({time.time()-t_start:.0f}s)", flush=True)
+
+ok = [r for r in rows if r["jev"] is not None]
+deg = [r for r in rows if r["applied"] is False or r["err"]]
+lat = sorted(r["took_ms"] for r in ok if r["took_ms"] is not None)
+mb = sum(r["bm25"] for r in rows) / len(rows)
+mj = sum(r["jev"] for r in ok) / len(ok)
+wins = sum(1 for r in ok if r["jev"] > r["bm25"]); losses = sum(1 for r in ok if r["jev"] < r["bm25"])
+ti = sum(r["in_tok"] or 0 for r in ok); to = sum(r["out_tok"] or 0 for r in ok)
+judged = sum(r["judged"] or 0 for r in ok); unjudged = sum(r["unjudged"] or 0 for r in ok)
+print(f"\nqueries scored: {len(ok)} degraded/errored: {len(deg)} judged: {judged} unjudged: {unjudged}")
+print(f"bm25 nDCG@10 = {mb:.4f}")
+print(f"bm25+jev nDCG@10 = {mj:.4f} (paired: wins {wins} / losses {losses} / ties {len(ok)-wins-losses})")
+if lat: print(f"rerank took_ms p50={lat[len(lat)//2]} p95={lat[int(len(lat)*.95)]}")
+print(f"usage: input {ti} tok, output {to} tok -> ${ti*0.042/1e6:.4f} at $0.042/Mtok input-only")
+
+# determinism probe: re-ask 3 pilot queries, compare probabilities
+print("\ndeterminism probe (3 queries re-asked):")
+for r in rows[:3]:
+ q = queries[r["qid"]]
+ _, rr2, probs2, _ = jev(q)
+ common = set(r["probs"]) & set(probs2)
+ if not common: print(f" {r['qid']}: no overlap ({len(probs2)} answers)"); continue
+ dmax = max(abs(r["probs"][k] - probs2[k]) for k in common)
+ same = sum(1 for k in common if r["probs"][k] == probs2[k])
+ print(f" {r['qid']}: {same}/{len(common)} probs identical, max |delta| = {dmax}")
diff --git a/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/raw_bench.py b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/raw_bench.py
new file mode 100644
index 000000000..32de56b9b
--- /dev/null
+++ b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/raw_bench.py
@@ -0,0 +1,103 @@
+"""Raw-API bench of the FIXED question shape (V2): document inside each noul's
+instructions, state = the query string. Same 40 pilot queries, same BM25 top-30
+shortlists, same scoring as eval_rerank.py. Pin jev-1.13.0. One request per
+query (30 nouls), like the stage's batching.
+
+This measures what xerj-rerank would score after build_jev_request is fixed to
+the shape the TypeSafe docs themselves prescribe (per-question data fields,
+referred to by name in backticks).
+"""
+import json, math, sys, time, urllib.request, collections, os
+
+U = os.environ.get("XERJ_URL", "http://localhost:9410")
+KEY = open("/root/.config/xerj-bench/typesafe-api-key").read().strip()
+API = "https://api.typesafe.ai/v1/systemone"
+N = int(sys.argv[1]) if len(sys.argv) > 1 else 40
+MODEL = "jev-1.13.0"
+
+def node_search(body):
+ r = urllib.request.Request(U + "/scifact/_search", data=json.dumps(body).encode(),
+ method="POST", headers={"content-type": "application/json"})
+ return json.loads(urllib.request.urlopen(r, timeout=120).read())
+
+def jev_fixed(query, docs):
+ """docs: list of (ordinal, title, text). One request, one noul per doc, doc inline."""
+ questions = {}
+ for i, (t, x) in docs.items():
+ questions[f"d{i}"] = {"type": "noul", "instructions": {
+ "question": "Does `document` contain information that answers the query in the state? Judge only whether `document` is relevant to the query, not whether it is well written.",
+ "document": (t + ". " + x)[:1400]}}
+ body = {"state": query, "model": MODEL, "questions": questions}
+ r = urllib.request.Request(API, data=json.dumps(body).encode(), method="POST",
+ headers={"content-type": "application/json", "authorization": f"Bearer {KEY}"})
+ try:
+ return json.loads(urllib.request.urlopen(r, timeout=180).read())
+ except urllib.error.HTTPError as e:
+ return {"_err": e.code, "body": e.read().decode()[:300]}
+
+queries = {}
+for l in open("scifact/queries.jsonl"):
+ d = json.loads(l); queries[d["_id"]] = d["text"]
+qrels = collections.defaultdict(dict)
+for i, l in enumerate(open("scifact/qrels/test.tsv")):
+ if i == 0: continue
+ q, d, s = l.split("\t"); qrels[q][d] = int(s)
+
+def ndcg10(ranked, rel):
+ dcg = sum(rel.get(d, 0) / math.log2(i + 2) for i, d in enumerate(ranked[:10]))
+ ideal = sorted(rel.values(), reverse=True)[:10]
+ idcg = sum(g / math.log2(i + 2) for i, g in enumerate(ideal))
+ return dcg / idcg if idcg else 0.0
+
+# corpus text for the shortlists
+need = set(); shorts = {}
+pilot = list(qrels.items())[:N]
+for qid, _ in pilot:
+ res = node_search({"query": {"multi_match": {"query": queries[qid], "fields": ["title", "text"]}},
+ "size": 30, "_source": ["title", "text"]})
+ shorts[qid] = [(h["_id"], h["_source"].get("title", ""), h["_source"].get("text", "")) for h in res["hits"]["hits"]]
+ need.update(d[0] for d in shorts[qid])
+print(f"shortlists ready: {len(pilot)} queries, {len(need)} distinct docs")
+
+LOG = open("scifact-raw-fixed.jsonl", "w")
+rows = []
+t00 = time.time()
+for n, (qid, rel) in enumerate(pilot):
+ docs = {i: (t, x) for i, (idd, t, x) in enumerate(shorts[qid])}
+ t0 = time.time()
+ r = jev_fixed(queries[qid], docs)
+ wall = time.time() - t0
+ if "_err" in r or "answers" not in r:
+ print(f" qid {qid} FAILED: {str(r)[:200]}"); rows.append({"qid": qid, "err": str(r)[:200]}); continue
+ probs = {int(k[1:]): v.get("noul") for k, v in r["answers"].items() if isinstance(v, dict)}
+ order = sorted(probs.keys(), key=lambda i: (-probs[i], i)) # tie -> engine order
+ ranked = [shorts[qid][i][0] for i in order] + [shorts[qid][i][0] for i in range(len(shorts[qid])) if i not in probs]
+ row = {"qid": qid, "ndcg": ndcg10(ranked, rel), "answered": len(probs), "wall_s": round(wall, 2),
+ "in_tok": (r.get("usage") or {}).get("input_tokens"), "out_tok": (r.get("usage") or {}).get("output_tokens"),
+ "model": r.get("model"), "probs": {shorts[qid][i][0]: probs[i] for i in probs}}
+ rows.append(row); LOG.write(json.dumps({k: v for k, v in row.items() if k != "probs"}) + "\n"); LOG.flush()
+ if n % 10 == 0: print(f" {n+1}/{len(pilot)} ({time.time()-t00:.0f}s)", flush=True)
+
+ok = [r for r in rows if "ndcg" in r]
+bm = {json.loads(l)["qid"]: json.loads(l)["bm25"] for l in open("scifact-rerank-pilot.jsonl")}
+mb = sum(bm[r["qid"]] for r in ok) / len(ok)
+mj = sum(r["ndcg"] for r in ok) / len(ok)
+wins = sum(1 for r in ok if r["ndcg"] > bm[r["qid"]]); losses = sum(1 for r in ok if r["ndcg"] < bm[r["qid"]])
+ti = sum(r["in_tok"] or 0 for r in ok); to = sum(r["out_tok"] or 0 for r in ok)
+print(f"\nqueries: {len(ok)} ok / {len(rows)-len(ok)} failed")
+print(f"bm25 (pilot log) nDCG@10 = {mb:.4f}")
+print(f"bm25+jev FIXED nDCG@10 = {mj:.4f} (wins {wins} / losses {losses} / ties {len(ok)-wins-losses})")
+print(f"usage: input {ti} tok -> ${ti*0.042/1e6:.4f} at $0.042/Mtok input-only")
+print(f"models seen: {set(r.get('model') for r in ok)}")
+
+# determinism probe on the fixed shape
+print("\ndeterminism probe (3 queries re-asked, fixed shape):")
+for qid, rel in pilot[:3]:
+ docs = {i: (t, x) for i, (idd, t, x) in enumerate(shorts[qid])}
+ r2 = jev_fixed(queries[qid], docs)
+ p1 = next(x for x in rows if x["qid"] == qid)["probs"]
+ p2 = {shorts[qid][int(k[1:])][0]: v.get("noul") for k, v in r2.get("answers", {}).items() if isinstance(v, dict)}
+ common = set(p1) & set(p2)
+ dmax = max(abs(p1[k] - p2[k]) for k in common) if common else None
+ same = sum(1 for k in common if p1[k] == p2[k])
+ print(f" {qid}: {same}/{len(common)} identical, max |delta| = {dmax}")
diff --git a/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/scifact-rerank-pilot.jsonl b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/scifact-rerank-pilot.jsonl
new file mode 100644
index 000000000..799c382bf
--- /dev/null
+++ b/benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/scifact-rerank-pilot.jsonl
@@ -0,0 +1,40 @@
+{"qid": "1", "bm25": 0.0, "jev": 0.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 7031, "out_tok": 534, "took_ms": 1204, "err": null}
+{"qid": "3", "bm25": 0.5, "jev": 0.3562071871080222, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9812, "out_tok": 534, "took_ms": 382, "err": null}
+{"qid": "5", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10215, "out_tok": 534, "took_ms": 393, "err": null}
+{"qid": "13", "bm25": 0.0, "jev": 0.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 11013, "out_tok": 534, "took_ms": 386, "err": null}
+{"qid": "36", "bm25": 0.20438239758848611, "jev": 0.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 11047, "out_tok": 534, "took_ms": 411, "err": null}
+{"qid": "42", "bm25": 0.6309297535714575, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10903, "out_tok": 534, "took_ms": 389, "err": null}
+{"qid": "48", "bm25": 0.6309297535714575, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10528, "out_tok": 534, "took_ms": 397, "err": null}
+{"qid": "49", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10100, "out_tok": 534, "took_ms": 384, "err": null}
+{"qid": "50", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9603, "out_tok": 534, "took_ms": 378, "err": null}
+{"qid": "51", "bm25": 0.6309297535714575, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10619, "out_tok": 534, "took_ms": 384, "err": null}
+{"qid": "53", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10327, "out_tok": 534, "took_ms": 399, "err": null}
+{"qid": "54", "bm25": 0.5, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10437, "out_tok": 534, "took_ms": 360, "err": null}
+{"qid": "56", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10229, "out_tok": 534, "took_ms": 376, "err": null}
+{"qid": "57", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10224, "out_tok": 534, "took_ms": 405, "err": null}
+{"qid": "70", "bm25": 1.0, "jev": 0.9197207891481876, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10285, "out_tok": 534, "took_ms": 412, "err": null}
+{"qid": "72", "bm25": 0.3562071871080222, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10156, "out_tok": 534, "took_ms": 478, "err": null}
+{"qid": "75", "bm25": 0.5, "jev": 0.43067655807339306, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10630, "out_tok": 534, "took_ms": 385, "err": null}
+{"qid": "94", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 8418, "out_tok": 534, "took_ms": 383, "err": null}
+{"qid": "99", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9964, "out_tok": 534, "took_ms": 514, "err": null}
+{"qid": "100", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9191, "out_tok": 534, "took_ms": 371, "err": null}
+{"qid": "113", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 11195, "out_tok": 534, "took_ms": 354, "err": null}
+{"qid": "115", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9401, "out_tok": 534, "took_ms": 375, "err": null}
+{"qid": "118", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9595, "out_tok": 534, "took_ms": 360, "err": null}
+{"qid": "124", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9738, "out_tok": 534, "took_ms": 402, "err": null}
+{"qid": "127", "bm25": 0.6309297535714575, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10067, "out_tok": 534, "took_ms": 410, "err": null}
+{"qid": "128", "bm25": 0.5, "jev": 0.38685280723454163, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10261, "out_tok": 534, "took_ms": 390, "err": null}
+{"qid": "129", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9897, "out_tok": 534, "took_ms": 374, "err": null}
+{"qid": "130", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9386, "out_tok": 534, "took_ms": 410, "err": null}
+{"qid": "132", "bm25": 0.0, "jev": 0.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9887, "out_tok": 534, "took_ms": 420, "err": null}
+{"qid": "133", "bm25": 0.3391602052736161, "jev": 0.7198329954996533, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 11107, "out_tok": 534, "took_ms": 408, "err": null}
+{"qid": "137", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10115, "out_tok": 534, "took_ms": 435, "err": null}
+{"qid": "141", "bm25": 0.8772153153380493, "jev": 0.9197207891481876, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9233, "out_tok": 534, "took_ms": 406, "err": null}
+{"qid": "142", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10431, "out_tok": 534, "took_ms": 415, "err": null}
+{"qid": "143", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10285, "out_tok": 534, "took_ms": 399, "err": null}
+{"qid": "146", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10814, "out_tok": 534, "took_ms": 379, "err": null}
+{"qid": "148", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9583, "out_tok": 534, "took_ms": 422, "err": null}
+{"qid": "163", "bm25": 1.0, "jev": 0.6309297535714575, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9576, "out_tok": 534, "took_ms": 515, "err": null}
+{"qid": "171", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10372, "out_tok": 534, "took_ms": 404, "err": null}
+{"qid": "179", "bm25": 0.6984152163370878, "jev": 0.8318724637288826, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 10841, "out_tok": 534, "took_ms": 394, "err": null}
+{"qid": "180", "bm25": 1.0, "jev": 1.0, "applied": true, "judged": 30, "unjudged": 0, "partial_failures": 0, "reason": null, "in_tok": 9649, "out_tok": 534, "took_ms": 416, "err": null}
diff --git a/docs/RERANK.md b/docs/RERANK.md
index 30ce905c9..0412049aa 100644
--- a/docs/RERANK.md
+++ b/docs/RERANK.md
@@ -28,14 +28,18 @@ but has no absolute meaning: a `_score` of 7.2 is not comparable across queries
and cannot be used as a cut-off. A calibrated probability can, so
`rerank.min_score` is a threshold that means the same thing on every query.
-> **Not verified: ranking quality with the real model.** No provider key was
-> available to this project, so XERJ has **not** measured how well the Jev model
-> ranks. Every test of this stage runs against a test double that scores by a
-> table the test supplies. What is verified is the mechanism — the wire format,
-> the reorder, paging, pruning, the failure policy, and what is and is not sent.
-> The quality figures published by the provider's ecosystem are quoted in
-> [Quality numbers](#quality-numbers-and-what-we-did-not-measure) with that
-> caveat attached.
+> **Measured with the real model (2026-09-20, pilot).** Over the first 40
+> judged SciFact test queries, BM25 shortlists (default lexical embedder, no
+> vectors), model pinned `jev-1.13.0`: this stage scored **0.8299 nDCG@10
+> against BM25's 0.7750**. Until this change the stage put every candidate in
+> shared `state` and asked an untargeted "does this document answer the query"
+> per key — the model read that as one question about the pile and answered
+> near-uniformly, which scored **0.3822**; each candidate now rides inside its
+> own question's `instructions`, the shape the provider's own docs prescribe.
+> The test suite still runs against test doubles, and verifies the mechanism:
+> the wire format, the reorder, paging, pruning, the failure policy, and what
+> is and is not sent. All numbers and method:
+> [Quality numbers](#quality-numbers-and-what-we-did-not-measure).
The stage is in `engine/crates/xerj-api/src/rerank_stage.rs`; the provider
client, the request parser and the failure policy are the leaf crate
@@ -488,11 +492,12 @@ the same fields. `true` or `{}` means defaults.
## Quality numbers, and what we did not measure
-XERJ has not run the Jev model. The figures below for Jev, Voyage and Cohere are
-**published in the [`hev/jev-rerank`](https://github.com/hev/jev-rerank) README,
-were not run by this project, and rerank that project's own first-stage
+XERJ has run the Jev model through this stage — once, as a pilot (2026-09-20;
+rows below). The figures for Jev, Voyage and Cohere in the first table are
+still **published in the [`hev/jev-rerank`](https://github.com/hev/jev-rerank)
+README, were not run by this project, and rerank that project's own first-stage
shortlist, which is not XERJ's.** Same datasets, same metric, not a controlled
-comparison.
+comparison — the second table is.
nDCG@10 on the BEIR `test` split:
@@ -506,6 +511,23 @@ nDCG@10 on the BEIR `test` split:
| XERJ BM25 top-30, reordered by MiniLM | 0.6855 | 0.3323 | us, `--embed-mode neural` |
| XERJ hybrid RRF | 0.6993 | 0.3448 | us, `--embed-mode neural` |
+Pilot run by this project (2026-09-20): first 40 judged SciFact test queries,
+BM25 top-30 shortlists from the default lexical embedder (no vectors), model
+pinned `jev-1.13.0`, scoring byte-identical to `benchmarks/beir-hybrid`:
+
+| | SciFact (40 queries) | Run by |
+|---|---:|---|
+| XERJ BM25 shortlist | 0.7750 | us, controlled |
+| XERJ BM25 → this stage, `jev-1.13.0` | **0.8299** | us, controlled |
+| the same, request shape as shipped before this change | 0.3822 | us, controlled |
+| raw provider API, fixed shape, same shortlists | 0.8389 | us, controlled |
+
+Paired vs BM25: 9 wins / 6 losses / 25 ties; 1,200/1,200 documents judged, no
+degrades; stage latency p50 399 ms. The provider is not deterministic
+(re-asking the same window moves probabilities by up to ±0.05), which is why
+the stage and raw-API rows need not agree to the digit. Per-query logs and the
+runner: `benchmarks/beir-hybrid/results/2026-09-20-rerank-pilot/`.
+
**Our rows were measured with `--embed-mode neural` and the
`all-MiniLM-L6-v2` model, not with the default embedder.** XERJ's default
embedder is lexical feature hashing, which has no model in it; do not read the
@@ -526,7 +548,7 @@ Two things those runs do say:
## Credits
- The one-`noul`-per-document request shape:
- [`hev/jev-rerank`](https://github.com/hev/jev-rerank), Apache-2.0.
+ [`hev/jev-rerank`](https://github.com/hev/jev-rerank), MIT.
- The degrade-on-deadline, surface-on-contract failure policy, and the retry
back-off shape (exponential in the attempt, a flat extra pause after a 429):
Meilisearch's personalization module (MIT, not part of its Enterprise
diff --git a/engine/crates/xerj-api/tests/rerank_stage_http.rs b/engine/crates/xerj-api/tests/rerank_stage_http.rs
index 3862333c7..d5e9cefa8 100644
--- a/engine/crates/xerj-api/tests/rerank_stage_http.rs
+++ b/engine/crates/xerj-api/tests/rerank_stage_http.rs
@@ -3,14 +3,15 @@
//!
//! ```text
//! POST /v1/systemone
-//! request { state, model, questions: { id: { type:"noul", instructions, criteria } } }
+//! request { state: <query string>, model, questions: { id: { type:"noul", instructions: { question, document } } } }
//! response { model, answers: { id: { type:"noul", noul: 0..1 } }, usage }
//! ```
//!
//! THE STUB IS A TEST DOUBLE, NOT A JUDGE. It scores by a table the test hands
//! it (or by word overlap), so ordering is checkable. Nothing here says
-//! anything about ranking quality with the real model — that has not been
-//! verified by this project; no provider key was available.
+//! anything about ranking quality with the real model — that is measured in
+//! docs/research/typesafe-skill-2026-09 (NOTES.md, pilot of 2026-09-20), not
+//! by these tests.
//!
//! # Why there is no `set_var` in this file
//!
@@ -199,7 +200,7 @@ async fn systemone(
_ => {}
}
- let query = words(parsed["state"]["query"].as_str().unwrap_or_default());
+ let query = words(parsed["state"].as_str().unwrap_or_default());
let verdicts = stub.verdicts.lock().unwrap().clone();
let mut answers = serde_json::Map::new();
if matches!(fault, Fault::WrongKeys) {
@@ -212,13 +213,18 @@ async fn systemone(
}))
.into_response();
}
- if let Some(docs) = parsed["state"]["documents"].as_object() {
- for (i, (key, doc)) in docs.iter().enumerate() {
+ if let Some(qs) = parsed["questions"].as_object() {
+ for (i, (key, q)) in qs.iter().enumerate() {
if matches!(fault, Fault::Partial) && i % 2 == 1 {
continue;
}
- let title = doc["title"].as_str().unwrap_or_default();
- let text = doc["text"].as_str().unwrap_or_default();
+ // The candidate rides inside its own question's instructions,
+ // joined "title. text" — the wire shape `build_jev_request` sends.
+ let doc = q["instructions"]["document"].as_str().unwrap_or_default();
+ let (title, text) = match doc.split_once(". ") {
+ Some((t, x)) => (t, x),
+ None => (doc, ""),
+ };
let p = match &verdicts {
Verdicts::Overlap => {
let have = words(&format!("{title} {text}"));
@@ -795,7 +801,7 @@ async fn a_bool_query_works_once_the_question_is_spelled_out() {
assert_eq!(r["_rerank"]["applied"], true);
assert_eq!(r["_rerank"]["query"], "bone density trial");
assert_eq!(
- stub.calls()[0].body["state"]["query"],
+ stub.calls()[0].body["state"],
"bone density trial",
"the judge is asked the caller's question, not a guess at the bool tree"
);
@@ -994,25 +1000,39 @@ async fn wire_shape_bearer_auth_model_and_one_noul_per_document_keyed_identicall
let call = &calls[0];
assert_eq!(call.authorization.as_deref(), Some("Bearer test-key"));
assert_eq!(call.body["model"], "jev-2026-09");
- assert_eq!(call.body["state"]["query"], Q);
+ // `state` is the question itself, a plain string — not an object. The
+ // candidate a question is about rides inside that question's
+ // `instructions`, pointed at by name; nothing is shared.
+ assert_eq!(call.body["state"], Q);
+ assert!(
+ call.body["state"].is_string(),
+ "the wire format the API documents: state is a string"
+ );
- let documents = call.body["state"]["documents"].as_object().unwrap();
let questions = call.body["questions"].as_object().unwrap();
- assert_eq!(documents.len(), 4);
- let mut dk: Vec<&String> = documents.keys().collect();
- let mut qk: Vec<&String> = questions.keys().collect();
- dk.sort();
- qk.sort();
- assert_eq!(dk, qk, "one question per document, keyed identically");
+ assert_eq!(questions.len(), 4);
+ assert_eq!(
+ questions.keys().cloned().collect::<Vec<_>>(),
+ ["d0", "d1", "d2", "d3"],
+ "keyed by the caller's ordinals"
+ );
for (key, question) in questions {
assert_eq!(question["type"], "noul", "{key}");
- assert!(question["instructions"].is_string(), "{key}");
- assert!(question["criteria"]["true"].is_string(), "{key}");
- assert!(question["criteria"]["false"].is_string(), "{key}");
- }
- for (key, doc) in documents {
- assert!(doc["text"].is_string(), "{key}");
- assert!(doc["title"].is_string(), "{key}");
+ let instructions = question["instructions"].as_object().unwrap();
+ assert_eq!(
+ instructions.len(),
+ 2,
+ "{key}: the question and its data, nothing else"
+ );
+ let text = instructions["question"].as_str().unwrap();
+ assert!(
+ text.contains("`document`"),
+ "{key}: the question must name the field it judges: {text}"
+ );
+ assert!(
+ instructions["document"].as_str().is_some_and(|d| !d.is_empty()),
+ "{key}: the candidate itself travels with its question"
+ );
}
// The key is a bearer token and nothing else: never in the body.
assert!(!call.body.to_string().contains("test-key"));
@@ -1031,7 +1051,15 @@ async fn custom_instructions_reach_every_question() {
.await;
assert_eq!(st, StatusCode::OK);
for (_, q) in stub.calls()[0].body["questions"].as_object().unwrap() {
- assert_eq!(q["instructions"], "Is this a clinical result?");
+ let text = q["instructions"]["question"].as_str().unwrap();
+ assert!(
+ text.starts_with("Is this a clinical result?"),
+ "the caller's instructions lead: {text}"
+ );
+ assert!(
+ text.contains("`document`"),
+ "and the question still names what it judges: {text}"
+ );
}
}
@@ -1068,9 +1096,9 @@ async fn a_window_larger_than_thirty_is_split_and_ordinals_survive() {
assert_eq!(calls.len(), 2, "35 documents = 30 + 5");
let mut keys: Vec<String> = Vec::new();
for c in &calls {
- let docs = c.body["state"]["documents"].as_object().unwrap();
- assert!(docs.len() <= 30, "the provider ceiling is 30 per call");
- keys.extend(docs.keys().cloned());
+ let qs = c.body["questions"].as_object().unwrap();
+ assert!(qs.len() <= 30, "the provider ceiling is 30 per call");
+ keys.extend(qs.keys().cloned());
}
keys.sort();
keys.dedup();
@@ -1340,10 +1368,14 @@ async fn a_named_field_the_response_does_not_return_is_never_judged_blind() {
assert_eq!(st, StatusCode::OK, "{r}");
assert_eq!(r["_rerank"]["applied"], true, "{r}");
for c in stub.calls() {
- for (key, doc) in c.body["state"]["documents"].as_object().unwrap() {
+ for (key, q) in c.body["questions"].as_object().unwrap() {
assert!(
- !doc["text"].as_str().unwrap_or_default().trim().is_empty(),
- "document {key} was judged on blank text: {doc}"
+ !q["instructions"]["document"]
+ .as_str()
+ .unwrap_or_default()
+ .trim()
+ .is_empty(),
+ "document {key} was judged on blank text: {q}"
);
}
}
@@ -1415,24 +1447,25 @@ async fn rerank_fields_is_exhaustive_about_what_is_sent() {
"vitamin rich vegetables for dinner",
"bone china has high density",
];
+ let titles = [
+ "Bone health basics",
+ "Trial results",
+ "Cooking",
+ "Density of materials",
+ ];
let calls = stub.calls();
- let documents = calls[0].body["state"]["documents"].as_object().unwrap();
- assert_eq!(documents.len(), 4);
- for (key, doc) in documents {
- let keys: Vec<&String> = doc.as_object().unwrap().keys().collect();
- assert_eq!(
- keys,
- ["text"],
- "{key}: `rerank.fields` named only `body`, so not even the title may be sent"
- );
- let text = doc["text"].as_str().unwrap();
+ let questions = calls[0].body["questions"].as_object().unwrap();
+ assert_eq!(questions.len(), 4);
+ for (key, q) in questions {
+ let doc = q["instructions"]["document"].as_str().unwrap();
assert!(
- bodies.contains(&text),
- "{key}: the text is exactly the body — no `cat`, no `n`, nothing appended: {text:?}"
+ bodies.contains(&doc),
+ "{key}: `rerank.fields` named only `body`, so the judged text is exactly the \
+ body — no `cat`, no `n`, not even the title: {doc:?}"
);
}
- // Naming the title brings it back, in its own slot.
+ // Naming the title puts it back, in front of the body it belongs to.
let (_, _) = node
.search(
"/kb/_search",
@@ -1440,11 +1473,16 @@ async fn rerank_fields_is_exhaustive_about_what_is_sent() {
)
.await;
let last = stub.calls().last().unwrap().body.clone();
- for (key, doc) in last["state"]["documents"].as_object().unwrap() {
- assert!(doc["title"].is_string(), "{key}: {doc}");
+ for (key, q) in last["questions"].as_object().unwrap() {
+ let doc = q["instructions"]["document"].as_str().unwrap();
+ let joined: Vec<String> = titles
+ .iter()
+ .zip(bodies.iter())
+ .map(|(t, b)| format!("{t}. {b}"))
+ .collect();
assert!(
- bodies.contains(&doc["text"].as_str().unwrap()),
- "{key}: {doc}"
+ joined.contains(&doc.to_string()),
+ "{key}: title and body, joined as one document: {doc:?}"
);
}
}
@@ -1469,10 +1507,7 @@ async fn hits_with_no_prose_are_skipped_not_sent_blank() {
assert_eq!(r["_rerank"]["judged"], 4);
assert_eq!(r["_rerank"]["skipped_no_text"], 1);
assert_eq!(
- stub.calls()[0].body["state"]["documents"]
- .as_object()
- .unwrap()
- .len(),
+ stub.calls()[0].body["questions"].as_object().unwrap().len(),
4
);
// Unjudged is not a verdict of irrelevant: it sorts after the judged hits.
@@ -2147,16 +2182,16 @@ async fn the_matching_passage_is_judgeable_and_goes_first() {
assert_eq!(r["_rerank"]["applied"], true, "{r}");
assert!(r["_rerank"].get("fields_without_text").is_none(), "{r}");
let sent = stub.calls().last().unwrap().body.clone();
- for (key, doc) in sent["state"]["documents"].as_object().unwrap() {
+ for (key, q) in sent["questions"].as_object().unwrap() {
+ let doc = q["instructions"]["document"].as_str().unwrap();
+ assert!(!doc.trim().is_empty(), "{key}: {q}");
assert!(
- doc.get("title").is_none(),
- "{key}: the title was not named: {doc}"
+ bodies.iter().any(|b| b.contains(doc.trim())),
+ "{key}: a passage is a slice of the body it came from: {doc:?}"
);
- let text = doc["text"].as_str().unwrap();
- assert!(!text.trim().is_empty(), "{key}: {doc}");
assert!(
- bodies.iter().any(|b| b.contains(text.trim())),
- "{key}: a passage is a slice of the body it came from: {text:?}"
+ !doc.contains(". ") || bodies.iter().any(|b| b.starts_with(doc.trim())),
+ "{key}: the title was not returned, so it was not prefixed: {doc:?}"
);
}
@@ -2174,12 +2209,21 @@ async fn the_matching_passage_is_judgeable_and_goes_first() {
.await;
assert_eq!(st, StatusCode::OK, "{r}");
let sent = stub.calls().last().unwrap().body.clone();
- for (key, doc) in sent["state"]["documents"].as_object().unwrap() {
- let text = doc["text"].as_str().unwrap();
- let first_line = text.lines().next().unwrap_or_default();
+ for (key, q) in sent["questions"].as_object().unwrap() {
+ let text = q["instructions"]["document"].as_str().unwrap();
+ // The document is one string now: `<title>. <passage>` then the other
+ // returned `_source` strings — so a clip at `max_doc_chars` cuts the
+ // tail and never the part that matched.
+ let (title, rest) = text.split_once(". ").unwrap_or(("", text));
+ assert!(
+ ["Bone health basics", "Trial results", "Cooking", "Density of materials"]
+ .contains(&title),
+ "{key}: the returned title joins its document: {text:?}"
+ );
+ let first_line = rest.lines().next().unwrap_or_default();
assert!(
bodies.iter().any(|b| b.contains(first_line.trim())) && !first_line.trim().is_empty(),
- "{key}: the passage comes first: {text:?}"
+ "{key}: the passage comes first after it: {text:?}"
);
assert!(
["health", "trial", "food", "materials"].contains(&text.lines().last().unwrap()),
@@ -2926,8 +2970,8 @@ async fn text_returned_only_through_fields_is_judged_without_naming_it() {
.await;
assert_eq!(st, StatusCode::OK, "{r}");
for call in stub3.calls() {
- for (_, doc) in call.body["state"]["documents"].as_object().unwrap() {
- let text = doc["text"].as_str().unwrap_or_default();
+ for (_, q) in call.body["questions"].as_object().unwrap() {
+ let text = q["instructions"]["document"].as_str().unwrap_or_default();
assert!(
text.matches("vitamin rich vegetables").count() <= 1
&& text.matches("bone china").count() <= 1,
diff --git a/engine/crates/xerj-rerank/src/lib.rs b/engine/crates/xerj-rerank/src/lib.rs
index 1701f08d2..5f89745f2 100644
--- a/engine/crates/xerj-rerank/src/lib.rs
+++ b/engine/crates/xerj-rerank/src/lib.rs
@@ -111,7 +111,8 @@ pub const DEFAULT_MAX_DOC_CHARS: usize = 1200;
/// Ceiling on `rerank.instructions`, in characters.
///
/// The instructions are not sent once: the System One wire format carries them
-/// inside EVERY per-document question, so whatever the caller writes here is
+/// inside EVERY per-document question, alongside that question's candidate
+/// text (clipped by `max_doc_chars`), so whatever the caller writes here is
/// multiplied by the window. Uncapped, a 1 MB string at `window: 40` and
/// `max_doc_chars: 1` put 40 MB on the wire to the provider (measured in
/// review), and an 8 MB one grew the node's peak RSS by 700 MB for a single
@@ -651,40 +652,50 @@ fn error_body(text: &str, api_key: &str) -> String {
/// construction rather than by every caller remembering to validate. For a
/// request that came through the parser the cuts change nothing.
pub fn build_jev_request(query: &str, batch: &[Candidate], cfg: &RerankConfig) -> Value {
- let mut documents = Map::new();
let mut questions = Map::new();
- let instructions = clip(cfg.question(), MAX_INSTRUCTIONS_CHARS);
let max_doc_chars = cfg.max_doc_chars.min(MAX_DOC_CHARS);
+ // Each question carries its own candidate inside `instructions`, and the
+ // question names that field in backticks — the reference pattern the
+ // System One API documents for pointing a question at structured data.
+ // The previous shape kept every candidate in shared `state` and asked an
+ // untargeted "does this document answer the query" per key, which the
+ // model read as one question about the pile: measured 0.3822 nDCG@10 over
+ // 40 SciFact shortlists against 0.8389 for this shape, root cause isolated
+ // at the raw API (docs/research/typesafe-skill-2026-09/NOTES.md, 2026-09-20).
+ // hev/jev-reranker formats its listwise nouls the same way.
+ let question = match cfg.instructions.as_deref() {
+ Some(custom) => format!(
+ "{} Judge `document` against the query in the state.",
+ clip(custom, MAX_INSTRUCTIONS_CHARS)
+ ),
+ None => "Does `document` contain information that answers the query in the state? \
+ Judge only whether `document` is relevant to the query, not whether it is \
+ well written."
+ .to_string(),
+ };
+
for cand in batch {
// Keyed by the caller's ordinal, so a reordered or partial response
// still maps back to the right hit.
let key = format!("d{}", cand.ordinal);
- let mut doc = Map::new();
+ let mut doc = String::new();
if let Some(t) = &cand.title {
- doc.insert("title".into(), json!(clip(t, max_doc_chars)));
+ doc.push_str(clip(t, max_doc_chars));
+ doc.push_str(". ");
}
- doc.insert("text".into(), json!(clip(&cand.text, max_doc_chars)));
- documents.insert(key.clone(), Value::Object(doc));
-
+ doc.push_str(clip(&cand.text, max_doc_chars));
questions.insert(
key,
json!({
"type": "noul",
- "instructions": instructions,
- "criteria": {
- "true": "The document is relevant to the query.",
- "false": "The document is not relevant to the query."
- }
+ "instructions": { "question": question.as_str(), "document": doc }
}),
);
}
json!({
- "state": {
- "query": clip(query, MAX_QUERY_CHARS),
- "documents": Value::Object(documents),
- },
+ "state": clip(query, MAX_QUERY_CHARS),
"model": clip(&cfg.model, MAX_MODEL_CHARS),
"questions": Value::Object(questions),
})
@@ -1492,12 +1503,19 @@ mod tests {
let body = build_jev_request("vitamin d bone density", &cands(2), &cfg);
assert_eq!(body["model"], "jev-latest");
- assert_eq!(body["state"]["query"], "vitamin d bone density");
- assert_eq!(body["state"]["documents"]["d0"]["text"], "body 0");
- // One noul question per document, keyed identically to the document.
+ assert_eq!(body["state"], "vitamin d bone density");
+ // One noul question per candidate, and the candidate rides inside the
+ // question it is judged by — the question names `document` in backticks,
+ // so the model cannot judge the batch as one pile.
assert_eq!(body["questions"]["d0"]["type"], "noul");
assert_eq!(body["questions"]["d1"]["type"], "noul");
- assert!(body["questions"]["d0"]["criteria"]["true"].is_string());
+ let instr = &body["questions"]["d0"]["instructions"];
+ assert!(
+ instr["question"].as_str().unwrap().contains("`document`"),
+ "the question must target `document`: {}",
+ instr["question"]
+ );
+ assert_eq!(instr["document"], "title 0. body 0");
assert_eq!(body["questions"].as_object().unwrap().len(), 2);
}