Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(reranker): fail closed on malformed Jev choice probabilities

evanmaranzano5c893a60e807
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b2cda636..e298c0ee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+- Fixed the Jev choice reranker adapter silently fabricating scores from a
+ malformed judge response: `docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py`
+ defaulted missing or misnamed `probabilities` entries to `0.0`, so an HTTP
+ 200 with a broken payload produced a well-formed but meaningless ranking
+ the server could not tell apart from a real judgement. The shim now
+ validates the response (key set matches the candidate list, no duplicate
+ JSON keys, numeric non-boolean values in `[0, 1]`, NaN/Inf rejected,
+ distribution sum near 1) and answers HTTP 500 on any violation, which is
+ the failure mode ai-memory already degrades safely on. Includes stdlib
+ `unittest` coverage in `test_jev_rerank_shim_choice.py`. (#1198)
+
## [2.6.3] - 2026-10-09
### Fixed
diff --git a/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py b/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py
index 1079f684..932285d4 100644
--- a/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py
+++ b/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py
@@ -33,6 +33,7 @@ Stdlib only. No secrets are stored: Authorization headers are forwarded
verbatim to the upstream.
"""
import json
+import math
import os
import sys
import time
@@ -63,6 +64,61 @@ def log(msg):
sys.stderr.flush()
+def _reject_dupes(pairs):
+ """object_pairs_hook refusing duplicate keys in the Jev response JSON."""
+ d = {}
+ for k, v in pairs:
+ if k in d:
+ raise ValueError(f"duplicate key {k!r} in Jev response")
+ d[k] = v
+ return d
+
+
+def extract_choice_probs(out, cand_ids):
+ """Return one validated choice probability per candidate, in candidate order.
+
+ Raises ValueError on any malformed shape — missing/renamed keys, extra
+ candidates, non-numeric or boolean values, NaN/Inf, out-of-range numbers,
+ or a distribution whose mass is nowhere near 1 — so the caller answers
+ HTTP 500 and ai-memory keeps its original order, instead of silently
+ ranking on fabricated 0.0 scores.
+ """
+ if not cand_ids:
+ raise ValueError("no candidates")
+ if len(set(cand_ids)) != len(cand_ids):
+ raise ValueError("duplicate candidate indices in the rerank request")
+ answers = out.get("answers")
+ if not isinstance(answers, dict):
+ raise ValueError("Jev response has no answers object")
+ best = answers.get("best")
+ if not isinstance(best, dict):
+ raise ValueError("Jev response has no answers.best object")
+ probs = best.get("probabilities")
+ if not isinstance(probs, dict):
+ raise ValueError("Jev response has no answers.best.probabilities object")
+ expected = {f"c{n}" for n in cand_ids}
+ missing = sorted(expected - probs.keys())
+ extra = sorted(probs.keys() - expected)
+ if missing or extra:
+ raise ValueError(
+ f"probability keys mismatch: missing={missing} extra={extra}")
+ vals = []
+ for n in cand_ids:
+ p = probs[f"c{n}"]
+ if isinstance(p, bool) or not isinstance(p, (int, float)):
+ raise ValueError(f"probability c{n} is not a number: {p!r}")
+ p = float(p)
+ if not math.isfinite(p) or p < 0.0 or p > 1.0:
+ raise ValueError(f"probability c{n} outside [0, 1]: {p!r}")
+ vals.append(p)
+ total = math.fsum(vals)
+ # The choice distribution is normalised over the candidate set; allow
+ # generous rounding drift but reject degenerate shapes (all ~0 / all ~1).
+ if not 0.5 <= total <= 1.5:
+ raise ValueError(f"choice probabilities sum to {total:.4f}, not ~1.0")
+ return vals
+
+
def jev_rerank(payload):
"""Translate a reranker chat request into ONE batched Jev choice call.
@@ -79,6 +135,7 @@ def jev_rerank(payload):
if m.get("role") == "user"), "")
data = json.loads(user)
query, cands = data["query"], data["candidates"]
+ ids = [c["candidate"] for c in cands]
lines = [f"User query: {query}", "", "Candidate documents:"]
criteria = {}
for c in cands:
@@ -100,15 +157,10 @@ def jev_rerank(payload):
req = urllib.request.Request(JEV_URL, data=json.dumps(body).encode(),
headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=25) as resp:
- out = json.loads(resp.read())
+ out = json.loads(resp.read(), object_pairs_hook=_reject_dupes)
dt = time.perf_counter() - t0
- probs = ((out.get("answers") or {}).get("best") or {}).get("probabilities") or {}
- scores = []
- for c in cands:
- n = c["candidate"]
- p = probs.get(f"c{n}", 0.0)
- p = float(p) if isinstance(p, (int, float)) else 0.0
- scores.append({"candidate": n, "relevance": round(p, 4)})
+ vals = extract_choice_probs(out, ids)
+ scores = [{"candidate": n, "relevance": round(p, 4)} for n, p in zip(ids, vals)]
log(f"jev-choice {len(cands)} candidates in {dt:.3f}s query={query[:60]!r}")
return {"scores": scores}
diff --git a/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py b/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py
new file mode 100644
index 00000000..e19220a1
--- /dev/null
+++ b/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""Regression tests for jev_rerank_shim_choice probability validation.
+
+Run from this directory: python3 -m unittest test_jev_rerank_shim_choice -v
+Stdlib only, no Jev backend required (HTTP is monkeypatched).
+"""
+import io
+import json
+import unittest
+import unittest.mock
+
+import jev_rerank_shim_choice as shim
+
+
+def jev_response(probs):
+ return {"answers": {"best": {"probabilities": probs}}}
+
+
+def rerank_payload(ids=(1, 2, 3)):
+ cands = [{"candidate": n, "title": f"page {n}", "text": f"text {n}"}
+ for n in ids]
+ return {"messages": [
+ {"role": "system", "content": shim.RERANK_SYSTEM_PREFIX + " ..."},
+ {"role": "user",
+ "content": json.dumps({"query": "q", "candidates": cands})},
+ ]}
+
+
+class FakeResp(io.BytesIO):
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ return False
+
+
+class ExtractChoiceProbsTest(unittest.TestCase):
+ IDS = [1, 2, 3]
+
+ def test_valid(self):
+ vals = shim.extract_choice_probs(
+ jev_response({"c1": 0.7, "c2": 0.2, "c3": 0.1}), self.IDS)
+ self.assertEqual(vals, [0.7, 0.2, 0.1])
+
+ def test_rounding_drift_tolerated(self):
+ vals = shim.extract_choice_probs(
+ jev_response({"c1": 0.3333, "c2": 0.3333, "c3": 0.3333}), self.IDS)
+ self.assertAlmostEqual(sum(vals), 0.9999)
+
+ def test_missing_probabilities_object(self):
+ with self.assertRaises(ValueError):
+ shim.extract_choice_probs({"answers": {"best": {}}}, self.IDS)
+
+ def test_missing_answers(self):
+ with self.assertRaises(ValueError):
+ shim.extract_choice_probs({}, self.IDS)
+
+ def test_missing_key(self):
+ with self.assertRaisesRegex(ValueError, "missing=.*c3"):
+ shim.extract_choice_probs(
+ jev_response({"c1": 0.6, "c2": 0.4}), self.IDS)
+
+ def test_extra_key(self):
+ with self.assertRaisesRegex(ValueError, "extra=.*c9"):
+ shim.extract_choice_probs(
+ jev_response({"c1": 0.5, "c2": 0.3, "c3": 0.2, "c9": 0.0}),
+ self.IDS)
+
+ def test_bool_rejected(self):
+ with self.assertRaises(ValueError):
+ shim.extract_choice_probs(
+ jev_response({"c1": True, "c2": 0.0, "c3": 1.0}), self.IDS)
+
+ def test_string_rejected(self):
+ with self.assertRaises(ValueError):
+ shim.extract_choice_probs(
+ jev_response({"c1": "0.7", "c2": 0.2, "c3": 0.1}), self.IDS)
+
+ def test_nan_rejected(self):
+ with self.assertRaises(ValueError):
+ shim.extract_choice_probs(
+ jev_response({"c1": float("nan"), "c2": 0.5, "c3": 0.5}),
+ self.IDS)
+
+ def test_inf_rejected(self):
+ with self.assertRaises(ValueError):
+ shim.extract_choice_probs(
+ jev_response({"c1": float("inf"), "c2": 0.0, "c3": 0.0}),
+ self.IDS)
+
+ def test_out_of_range_rejected(self):
+ for bad in (1.7, -0.2):
+ with self.assertRaises(ValueError):
+ shim.extract_choice_probs(
+ jev_response({"c1": bad, "c2": 0.0, "c3": 0.0}),
+ self.IDS)
+
+ def test_degenerate_sum_rejected(self):
+ with self.assertRaisesRegex(ValueError, "sum"):
+ shim.extract_choice_probs(
+ jev_response({"c1": 0.9, "c2": 0.9, "c3": 0.9}), self.IDS)
+ with self.assertRaisesRegex(ValueError, "sum"):
+ shim.extract_choice_probs(
+ jev_response({"c1": 0.0, "c2": 0.0, "c3": 0.0}), self.IDS)
+
+ def test_duplicate_candidate_ids_rejected(self):
+ with self.assertRaisesRegex(ValueError, "duplicate candidate"):
+ shim.extract_choice_probs(
+ jev_response({"c1": 0.6, "c2": 0.4}), [1, 1])
+
+ def test_empty_candidates_rejected(self):
+ with self.assertRaises(ValueError):
+ shim.extract_choice_probs(jev_response({}), [])
+
+
+class DuplicateJsonKeyTest(unittest.TestCase):
+ def test_duplicate_keys_rejected(self):
+ with self.assertRaises(ValueError):
+ json.loads('{"c1": 0.6, "c1": 0.4}',
+ object_pairs_hook=shim._reject_dupes)
+
+ def test_normal_json_accepted(self):
+ d = json.loads('{"c1": 0.6, "c2": 0.4}',
+ object_pairs_hook=shim._reject_dupes)
+ self.assertEqual(d, {"c1": 0.6, "c2": 0.4})
+
+
+class JevRerankEndToEndTest(unittest.TestCase):
+ def run_shim(self, response_json):
+ fake = FakeResp(json.dumps(response_json).encode())
+ with unittest.mock.patch.object(shim.urllib.request, "urlopen",
+ return_value=fake):
+ return shim.jev_rerank(rerank_payload())
+
+ def test_valid_response_maps_scores(self):
+ out = self.run_shim(jev_response({"c1": 0.1, "c2": 0.7, "c3": 0.2}))
+ self.assertEqual(out, {"scores": [
+ {"candidate": 1, "relevance": 0.1},
+ {"candidate": 2, "relevance": 0.7},
+ {"candidate": 3, "relevance": 0.2},
+ ]})
+
+ def test_malformed_response_raises(self):
+ # Old behaviour silently fabricated 0.0 for missing keys; now the
+ # exception propagates so the handler answers 500 and ai-memory
+ # keeps its original order.
+ with self.assertRaises(ValueError):
+ self.run_shim(jev_response({"c1": 0.9}))
+
+ def test_200_with_wrong_shape_raises(self):
+ with self.assertRaises(ValueError):
+ self.run_shim({"answers": {"best": {"choice": "c1"}}})
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docs/jev-reranker-adapter.md b/docs/jev-reranker-adapter.md
index b173f241..beb1b29f 100644
--- a/docs/jev-reranker-adapter.md
+++ b/docs/jev-reranker-adapter.md
@@ -47,6 +47,15 @@ adapter answers HTTP 500 and ai-memory keeps its own candidate order, the
same as any provider outage. A judge endpoint that is down degrades to
"no reranking", never to "no search".
+The choice variant also fails closed on a *malformed* answer: an HTTP 200
+whose `probabilities` object is missing, renamed, short of candidates,
+padded with extras, or holds booleans, NaN/Inf, out-of-range numbers, or a
+distribution nowhere near normalised is rejected with 500 as well. The
+older shim defaulted those to `0.0`, which produced a well-formed but
+meaningless score set the server could not distinguish from a real
+judgement. `test_jev_rerank_shim_choice.py` (stdlib `unittest`, no backend
+needed) pins each rejection path.
+
One cosmetic note: the server logs still show the provider's configured
model name for reranking (it comes from provider config, not from the
adapter's reply), so the reranker leg will be attributed to your hosted