Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(reranker): fail closed on malformed Jev choice probabilities
evanmaranzano5c893a60e807
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b2cda636..e298c0ee 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed+- Fixed the Jev choice reranker adapter silently fabricating scores from a+ malformed judge response: `docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py`+ defaulted missing or misnamed `probabilities` entries to `0.0`, so an HTTP+ 200 with a broken payload produced a well-formed but meaningless ranking+ the server could not tell apart from a real judgement. The shim now+ validates the response (key set matches the candidate list, no duplicate+ JSON keys, numeric non-boolean values in `[0, 1]`, NaN/Inf rejected,+ distribution sum near 1) and answers HTTP 500 on any violation, which is+ the failure mode ai-memory already degrades safely on. Includes stdlib+ `unittest` coverage in `test_jev_rerank_shim_choice.py`. (#1198)+## [2.6.3] - 2026-10-09### Fixeddiff --git a/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py b/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.pyindex 1079f684..932285d4 100644--- a/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py+++ b/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py@@ -33,6 +33,7 @@ Stdlib only. No secrets are stored: Authorization headers are forwardedverbatim to the upstream."""import json+import mathimport osimport sysimport time@@ -63,6 +64,61 @@ def log(msg):sys.stderr.flush()+def _reject_dupes(pairs):+ """object_pairs_hook refusing duplicate keys in the Jev response JSON."""+ d = {}+ for k, v in pairs:+ if k in d:+ raise ValueError(f"duplicate key {k!r} in Jev response")+ d[k] = v+ return d+++def extract_choice_probs(out, cand_ids):+ """Return one validated choice probability per candidate, in candidate order.++ Raises ValueError on any malformed shape — missing/renamed keys, extra+ candidates, non-numeric or boolean values, NaN/Inf, out-of-range numbers,+ or a distribution whose mass is nowhere near 1 — so the caller answers+ HTTP 500 and ai-memory keeps its original order, instead of silently+ ranking on fabricated 0.0 scores.+ """+ if not cand_ids:+ raise ValueError("no candidates")+ if len(set(cand_ids)) != len(cand_ids):+ raise ValueError("duplicate candidate indices in the rerank request")+ answers = out.get("answers")+ if not isinstance(answers, dict):+ raise ValueError("Jev response has no answers object")+ best = answers.get("best")+ if not isinstance(best, dict):+ raise ValueError("Jev response has no answers.best object")+ probs = best.get("probabilities")+ if not isinstance(probs, dict):+ raise ValueError("Jev response has no answers.best.probabilities object")+ expected = {f"c{n}" for n in cand_ids}+ missing = sorted(expected - probs.keys())+ extra = sorted(probs.keys() - expected)+ if missing or extra:+ raise ValueError(+ f"probability keys mismatch: missing={missing} extra={extra}")+ vals = []+ for n in cand_ids:+ p = probs[f"c{n}"]+ if isinstance(p, bool) or not isinstance(p, (int, float)):+ raise ValueError(f"probability c{n} is not a number: {p!r}")+ p = float(p)+ if not math.isfinite(p) or p < 0.0 or p > 1.0:+ raise ValueError(f"probability c{n} outside [0, 1]: {p!r}")+ vals.append(p)+ total = math.fsum(vals)+ # The choice distribution is normalised over the candidate set; allow+ # generous rounding drift but reject degenerate shapes (all ~0 / all ~1).+ if not 0.5 <= total <= 1.5:+ raise ValueError(f"choice probabilities sum to {total:.4f}, not ~1.0")+ return vals++def jev_rerank(payload):"""Translate a reranker chat request into ONE batched Jev choice call.@@ -79,6 +135,7 @@ def jev_rerank(payload):if m.get("role") == "user"), "")data = json.loads(user)query, cands = data["query"], data["candidates"]+ ids = [c["candidate"] for c in cands]lines = [f"User query: {query}", "", "Candidate documents:"]criteria = {}for c in cands:@@ -100,15 +157,10 @@ def jev_rerank(payload):req = urllib.request.Request(JEV_URL, data=json.dumps(body).encode(),headers={"Content-Type": "application/json"})with urllib.request.urlopen(req, timeout=25) as resp:- out = json.loads(resp.read())+ out = json.loads(resp.read(), object_pairs_hook=_reject_dupes)dt = time.perf_counter() - t0- probs = ((out.get("answers") or {}).get("best") or {}).get("probabilities") or {}- scores = []- for c in cands:- n = c["candidate"]- p = probs.get(f"c{n}", 0.0)- p = float(p) if isinstance(p, (int, float)) else 0.0- scores.append({"candidate": n, "relevance": round(p, 4)})+ vals = extract_choice_probs(out, ids)+ scores = [{"candidate": n, "relevance": round(p, 4)} for n, p in zip(ids, vals)]log(f"jev-choice {len(cands)} candidates in {dt:.3f}s query={query[:60]!r}")return {"scores": scores}diff --git a/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py b/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.pynew file mode 100644index 00000000..e19220a1--- /dev/null+++ b/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py@@ -0,0 +1,156 @@+#!/usr/bin/env python3+"""Regression tests for jev_rerank_shim_choice probability validation.++Run from this directory: python3 -m unittest test_jev_rerank_shim_choice -v+Stdlib only, no Jev backend required (HTTP is monkeypatched).+"""+import io+import json+import unittest+import unittest.mock++import jev_rerank_shim_choice as shim+++def jev_response(probs):+ return {"answers": {"best": {"probabilities": probs}}}+++def rerank_payload(ids=(1, 2, 3)):+ cands = [{"candidate": n, "title": f"page {n}", "text": f"text {n}"}+ for n in ids]+ return {"messages": [+ {"role": "system", "content": shim.RERANK_SYSTEM_PREFIX + " ..."},+ {"role": "user",+ "content": json.dumps({"query": "q", "candidates": cands})},+ ]}+++class FakeResp(io.BytesIO):+ def __enter__(self):+ return self++ def __exit__(self, *args):+ return False+++class ExtractChoiceProbsTest(unittest.TestCase):+ IDS = [1, 2, 3]++ def test_valid(self):+ vals = shim.extract_choice_probs(+ jev_response({"c1": 0.7, "c2": 0.2, "c3": 0.1}), self.IDS)+ self.assertEqual(vals, [0.7, 0.2, 0.1])++ def test_rounding_drift_tolerated(self):+ vals = shim.extract_choice_probs(+ jev_response({"c1": 0.3333, "c2": 0.3333, "c3": 0.3333}), self.IDS)+ self.assertAlmostEqual(sum(vals), 0.9999)++ def test_missing_probabilities_object(self):+ with self.assertRaises(ValueError):+ shim.extract_choice_probs({"answers": {"best": {}}}, self.IDS)++ def test_missing_answers(self):+ with self.assertRaises(ValueError):+ shim.extract_choice_probs({}, self.IDS)++ def test_missing_key(self):+ with self.assertRaisesRegex(ValueError, "missing=.*c3"):+ shim.extract_choice_probs(+ jev_response({"c1": 0.6, "c2": 0.4}), self.IDS)++ def test_extra_key(self):+ with self.assertRaisesRegex(ValueError, "extra=.*c9"):+ shim.extract_choice_probs(+ jev_response({"c1": 0.5, "c2": 0.3, "c3": 0.2, "c9": 0.0}),+ self.IDS)++ def test_bool_rejected(self):+ with self.assertRaises(ValueError):+ shim.extract_choice_probs(+ jev_response({"c1": True, "c2": 0.0, "c3": 1.0}), self.IDS)++ def test_string_rejected(self):+ with self.assertRaises(ValueError):+ shim.extract_choice_probs(+ jev_response({"c1": "0.7", "c2": 0.2, "c3": 0.1}), self.IDS)++ def test_nan_rejected(self):+ with self.assertRaises(ValueError):+ shim.extract_choice_probs(+ jev_response({"c1": float("nan"), "c2": 0.5, "c3": 0.5}),+ self.IDS)++ def test_inf_rejected(self):+ with self.assertRaises(ValueError):+ shim.extract_choice_probs(+ jev_response({"c1": float("inf"), "c2": 0.0, "c3": 0.0}),+ self.IDS)++ def test_out_of_range_rejected(self):+ for bad in (1.7, -0.2):+ with self.assertRaises(ValueError):+ shim.extract_choice_probs(+ jev_response({"c1": bad, "c2": 0.0, "c3": 0.0}),+ self.IDS)++ def test_degenerate_sum_rejected(self):+ with self.assertRaisesRegex(ValueError, "sum"):+ shim.extract_choice_probs(+ jev_response({"c1": 0.9, "c2": 0.9, "c3": 0.9}), self.IDS)+ with self.assertRaisesRegex(ValueError, "sum"):+ shim.extract_choice_probs(+ jev_response({"c1": 0.0, "c2": 0.0, "c3": 0.0}), self.IDS)++ def test_duplicate_candidate_ids_rejected(self):+ with self.assertRaisesRegex(ValueError, "duplicate candidate"):+ shim.extract_choice_probs(+ jev_response({"c1": 0.6, "c2": 0.4}), [1, 1])++ def test_empty_candidates_rejected(self):+ with self.assertRaises(ValueError):+ shim.extract_choice_probs(jev_response({}), [])+++class DuplicateJsonKeyTest(unittest.TestCase):+ def test_duplicate_keys_rejected(self):+ with self.assertRaises(ValueError):+ json.loads('{"c1": 0.6, "c1": 0.4}',+ object_pairs_hook=shim._reject_dupes)++ def test_normal_json_accepted(self):+ d = json.loads('{"c1": 0.6, "c2": 0.4}',+ object_pairs_hook=shim._reject_dupes)+ self.assertEqual(d, {"c1": 0.6, "c2": 0.4})+++class JevRerankEndToEndTest(unittest.TestCase):+ def run_shim(self, response_json):+ fake = FakeResp(json.dumps(response_json).encode())+ with unittest.mock.patch.object(shim.urllib.request, "urlopen",+ return_value=fake):+ return shim.jev_rerank(rerank_payload())++ def test_valid_response_maps_scores(self):+ out = self.run_shim(jev_response({"c1": 0.1, "c2": 0.7, "c3": 0.2}))+ self.assertEqual(out, {"scores": [+ {"candidate": 1, "relevance": 0.1},+ {"candidate": 2, "relevance": 0.7},+ {"candidate": 3, "relevance": 0.2},+ ]})++ def test_malformed_response_raises(self):+ # Old behaviour silently fabricated 0.0 for missing keys; now the+ # exception propagates so the handler answers 500 and ai-memory+ # keeps its original order.+ with self.assertRaises(ValueError):+ self.run_shim(jev_response({"c1": 0.9}))++ def test_200_with_wrong_shape_raises(self):+ with self.assertRaises(ValueError):+ self.run_shim({"answers": {"best": {"choice": "c1"}}})+++if __name__ == "__main__":+ unittest.main()diff --git a/docs/jev-reranker-adapter.md b/docs/jev-reranker-adapter.mdindex b173f241..beb1b29f 100644--- a/docs/jev-reranker-adapter.md+++ b/docs/jev-reranker-adapter.md@@ -47,6 +47,15 @@ adapter answers HTTP 500 and ai-memory keeps its own candidate order, thesame as any provider outage. A judge endpoint that is down degrades to"no reranking", never to "no search".+The choice variant also fails closed on a *malformed* answer: an HTTP 200+whose `probabilities` object is missing, renamed, short of candidates,+padded with extras, or holds booleans, NaN/Inf, out-of-range numbers, or a+distribution nowhere near normalised is rejected with 500 as well. The+older shim defaulted those to `0.0`, which produced a well-formed but+meaningless score set the server could not distinguish from a real+judgement. `test_jev_rerank_shim_choice.py` (stdlib `unittest`, no backend+needed) pins each rejection path.+One cosmetic note: the server logs still show the provider's configuredmodel name for reranking (it comes from provider config, not from theadapter's reply), so the reranker leg will be attributed to your hosted