Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
test(reranker): pin Jev choice shim fail-closed contract end to end
AkitaOnRails7d377498ba66
diff --git a/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py b/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.pyindex 932285d4..e99a9e88 100644--- a/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py+++ b/docs/examples/jev-reranker-adapter/jev_rerank_shim_choice.py@@ -87,6 +87,8 @@ def extract_choice_probs(out, cand_ids):raise ValueError("no candidates")if len(set(cand_ids)) != len(cand_ids):raise ValueError("duplicate candidate indices in the rerank request")+ if not isinstance(out, dict):+ raise ValueError("Jev response is not a JSON object")answers = out.get("answers")if not isinstance(answers, dict):raise ValueError("Jev response has no answers object")diff --git a/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py b/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.pyindex e19220a1..4bd07ab0 100644--- a/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py+++ b/docs/examples/jev-reranker-adapter/test_jev_rerank_shim_choice.py@@ -4,10 +4,13 @@Run from this directory: python3 -m unittest test_jev_rerank_shim_choice -vStdlib only, no Jev backend required (HTTP is monkeypatched)."""+import http.clientimport ioimport json+import threadingimport unittestimport unittest.mock+from http.server import ThreadingHTTPServerimport jev_rerank_shim_choice as shim@@ -51,6 +54,11 @@ class ExtractChoiceProbsTest(unittest.TestCase):with self.assertRaises(ValueError):shim.extract_choice_probs({"answers": {"best": {}}}, self.IDS)+ def test_non_object_response_rejected(self):+ for bad in ([], "c1", 0.5, None):+ with self.assertRaisesRegex(ValueError, "not a JSON object"):+ shim.extract_choice_probs(bad, self.IDS)+def test_missing_answers(self):with self.assertRaises(ValueError):shim.extract_choice_probs({}, self.IDS)@@ -127,11 +135,21 @@ class DuplicateJsonKeyTest(unittest.TestCase):class JevRerankEndToEndTest(unittest.TestCase):def run_shim(self, response_json):- fake = FakeResp(json.dumps(response_json).encode())+ return self.run_shim_raw(json.dumps(response_json).encode())++ def run_shim_raw(self, raw):with unittest.mock.patch.object(shim.urllib.request, "urlopen",- return_value=fake):+ return_value=FakeResp(raw)):return shim.jev_rerank(rerank_payload())+ def test_duplicate_json_key_in_response_raises(self):+ # Last-wins parsing would read c1=0.1 and accept a valid-looking+ # distribution; the shim must refuse the ambiguous payload instead.+ raw = (b'{"answers": {"best": {"probabilities": '+ b'{"c1": 0.9, "c1": 0.1, "c2": 0.5, "c3": 0.4}}}}')+ with self.assertRaisesRegex(ValueError, "duplicate key"):+ self.run_shim_raw(raw)+def test_valid_response_maps_scores(self):out = self.run_shim(jev_response({"c1": 0.1, "c2": 0.7, "c3": 0.2}))self.assertEqual(out, {"scores": [@@ -152,5 +170,41 @@ class JevRerankEndToEndTest(unittest.TestCase):self.run_shim({"answers": {"best": {"choice": "c1"}}})+class HandlerFailClosedTest(unittest.TestCase):+ """The documented contract: a malformed judge answer becomes HTTP 500."""++ def post_rerank(self, jev_json):+ srv = ThreadingHTTPServer(("127.0.0.1", 0), shim.Handler)+ thread = threading.Thread(+ target=srv.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True)+ thread.start()+ self.addCleanup(thread.join)+ self.addCleanup(srv.server_close)+ self.addCleanup(srv.shutdown)+ fake = FakeResp(json.dumps(jev_json).encode())+ conn = http.client.HTTPConnection(*srv.server_address, timeout=5)+ self.addCleanup(conn.close)+ with unittest.mock.patch.object(shim.urllib.request, "urlopen",+ return_value=fake), \+ unittest.mock.patch.object(shim, "log"):+ conn.request("POST", "/v1/chat/completions",+ body=json.dumps(rerank_payload()),+ headers={"Content-Type": "application/json"})+ resp = conn.getresponse()+ return resp.status, resp.read()++ def test_malformed_probabilities_answer_500(self):+ status, _ = self.post_rerank(jev_response({"c1": 0.9}))+ self.assertEqual(status, 500)++ def test_valid_probabilities_answer_200_with_scores(self):+ status, body = self.post_rerank(+ jev_response({"c1": 0.1, "c2": 0.7, "c3": 0.2}))+ self.assertEqual(status, 200)+ content = json.loads(body)["choices"][0]["message"]["content"]+ self.assertEqual(json.loads(content)["scores"][1],+ {"candidate": 2, "relevance": 0.7})++if __name__ == "__main__":unittest.main()