Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Preserve observed Jev costs when transport logs are missing
David Zhangf6e9337bba03
diff --git a/evals/implementation/swebench/installed.md b/evals/implementation/swebench/installed.mdindex fcf78af..98961a5 100644--- a/evals/implementation/swebench/installed.md+++ b/evals/implementation/swebench/installed.md@@ -54,6 +54,8 @@ fallbacks do not multiply the response cost. This is observed API metadata, notinvoice reconciliation. Complete totals require matching client starts, completedtransport receipts, request/response files and valid cost fields. Missing orinvalid evidence leaves the total unknown while preserving a known subtotal.+Retained response costs still contribute to that subtotal when the transport log+or a request-start entry is missing.These observations never enter the scored Sol task cost or change cost-win rules.`aggregate --plan ...` requires ten terminal attempts with official grading receipts,diff --git a/evals/implementation/swebench/installed.py b/evals/implementation/swebench/installed.pyindex f1af330..6174744 100644--- a/evals/implementation/swebench/installed.py+++ b/evals/implementation/swebench/installed.py@@ -427,7 +427,8 @@ def observed_jev(out, events, log_valid, traces_copied):costs, inputs, outputs, attempts = [], [], [], []def count(value):return type(value) is int and value >= 0- for identifier in sorted(safe_ids):+ # Retained response metadata remains known even if transport logs were lost.+ for identifier in sorted(responses):try:response = responses[identifier]body = json.loads(response.read_text())diff --git a/evals/implementation/swebench/test_installed.py b/evals/implementation/swebench/test_installed.pyindex 26630e1..8831e31 100644--- a/evals/implementation/swebench/test_installed.py+++ b/evals/implementation/swebench/test_installed.py@@ -198,6 +198,23 @@ class InstalledTests(unittest.TestCase):events[-1]['transportError']='Interrupted'self.assertIsNone(runner.observed_jev(root,events,True,True)['observed_cost_usd'])+ def test_retained_jev_cost_survives_missing_log_or_request_start(self):+ for missing_log in [True, False]:+ with self.subTest(missing_log=missing_log),tempfile.TemporaryDirectory() as temporary:+ path,plan=self.fixture(Path(temporary));out=self.receipt(path,plan)+ events=self.jev_fixture(out)+ if not missing_log:+ (out/'proxy.jsonl').write_text('\n'.join(json.dumps(event) for event in events[1:]))+ receipt=json.loads((out/'receipt.json').read_text());receipt['jev_traces_copied']=True;runner.write_json(out/'receipt.json',receipt)+ with patch.object(runner.urllib.request,'urlopen',side_effect=AssertionError('No network')),contextlib.redirect_stdout(io.StringIO()):+ runner.account(argparse.Namespace(plan=path,task=None))+ result=json.loads((out/'generation-accounting.json').read_text())['jev']+ self.assertEqual(result['known_cost_usd'],0.03)+ self.assertEqual(result['responses_with_cost'],2)+ self.assertEqual(result['known_input_tokens'],20)+ self.assertEqual(result['known_provider_attempts'],3)+ self.assertFalse(result['complete']);self.assertIsNone(result['observed_cost_usd'])+def test_observed_jev_cost_does_not_change_scored_sol_cost_win(self):with tempfile.TemporaryDirectory() as temporary:path,plan=self.fixture(Path(temporary));out=self.receipt(path,plan)