Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

test(reranker): pin Jev choice shim fail-closed contract end to end

AkitaOnRails7d377498ba66
diff --git a/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py b/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py
index 932285d4..e99a9e88 100644
--- a/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py
+++ b/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py
@@ -87,6 +87,8 @@ def extract_choice_probs(out, cand_ids):
raise ValueError("no candidates")
if len(set(cand_ids)) != len(cand_ids):
raise ValueError("duplicate candidate indices in the rerank request")
+ if not isinstance(out, dict):
+ raise ValueError("Jev response is not a JSON object")
answers = out.get("answers")
if not isinstance(answers, dict):
raise ValueError("Jev response has no answers object")
diff --git a/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py b/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py
index e19220a1..4bd07ab0 100644
--- a/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py
+++ b/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py
@@ -4,10 +4,13 @@
Run from this directory: python3 -m unittest test_jev_rerank_shim_choice -v
Stdlib only, no Jev backend required (HTTP is monkeypatched).
"""
+import http.client
import io
import json
+import threading
import unittest
import unittest.mock
+from http.server import ThreadingHTTPServer
import jev_rerank_shim_choice as shim
@@ -51,6 +54,11 @@ class ExtractChoiceProbsTest(unittest.TestCase):
with self.assertRaises(ValueError):
shim.extract_choice_probs({"answers": {"best": {}}}, self.IDS)
+ def test_non_object_response_rejected(self):
+ for bad in ([], "c1", 0.5, None):
+ with self.assertRaisesRegex(ValueError, "not a JSON object"):
+ shim.extract_choice_probs(bad, self.IDS)
+
def test_missing_answers(self):
with self.assertRaises(ValueError):
shim.extract_choice_probs({}, self.IDS)
@@ -127,11 +135,21 @@ class DuplicateJsonKeyTest(unittest.TestCase):
class JevRerankEndToEndTest(unittest.TestCase):
def run_shim(self, response_json):
- fake = FakeResp(json.dumps(response_json).encode())
+ return self.run_shim_raw(json.dumps(response_json).encode())
+
+ def run_shim_raw(self, raw):
with unittest.mock.patch.object(shim.urllib.request, "urlopen",
- return_value=fake):
+ return_value=FakeResp(raw)):
return shim.jev_rerank(rerank_payload())
+ def test_duplicate_json_key_in_response_raises(self):
+ # Last-wins parsing would read c1=0.1 and accept a valid-looking
+ # distribution; the shim must refuse the ambiguous payload instead.
+ raw = (b'{"answers": {"best": {"probabilities": '
+ b'{"c1": 0.9, "c1": 0.1, "c2": 0.5, "c3": 0.4}}}}')
+ with self.assertRaisesRegex(ValueError, "duplicate key"):
+ self.run_shim_raw(raw)
+
def test_valid_response_maps_scores(self):
out = self.run_shim(jev_response({"c1": 0.1, "c2": 0.7, "c3": 0.2}))
self.assertEqual(out, {"scores": [
@@ -152,5 +170,41 @@ class JevRerankEndToEndTest(unittest.TestCase):
self.run_shim({"answers": {"best": {"choice": "c1"}}})
+class HandlerFailClosedTest(unittest.TestCase):
+ """The documented contract: a malformed judge answer becomes HTTP 500."""
+
+ def post_rerank(self, jev_json):
+ srv = ThreadingHTTPServer(("127.0.0.1", 0), shim.Handler)
+ thread = threading.Thread(
+ target=srv.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True)
+ thread.start()
+ self.addCleanup(thread.join)
+ self.addCleanup(srv.server_close)
+ self.addCleanup(srv.shutdown)
+ fake = FakeResp(json.dumps(jev_json).encode())
+ conn = http.client.HTTPConnection(*srv.server_address, timeout=5)
+ self.addCleanup(conn.close)
+ with unittest.mock.patch.object(shim.urllib.request, "urlopen",
+ return_value=fake), \
+ unittest.mock.patch.object(shim, "log"):
+ conn.request("POST", "/v1/chat/completions",
+ body=json.dumps(rerank_payload()),
+ headers={"Content-Type": "application/json"})
+ resp = conn.getresponse()
+ return resp.status, resp.read()
+
+ def test_malformed_probabilities_answer_500(self):
+ status, _ = self.post_rerank(jev_response({"c1": 0.9}))
+ self.assertEqual(status, 500)
+
+ def test_valid_probabilities_answer_200_with_scores(self):
+ status, body = self.post_rerank(
+ jev_response({"c1": 0.1, "c2": 0.7, "c3": 0.2}))
+ self.assertEqual(status, 200)
+ content = json.loads(body)["choices"][0]["message"]["content"]
+ self.assertEqual(json.loads(content)["scores"][1],
+ {"candidate": 2, "relevance": 0.7})
+
+
if __name__ == "__main__":
unittest.main()