Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Preserve observed Jev costs when transport logs are missing

David Zhangf6e9337bba03
diff --git a/evals/implementation/swebench/installed.md b/evals/implementation/swebench/installed.md
index fcf78af..98961a5 100644
--- a/evals/implementation/swebench/installed.md
+++ b/evals/implementation/swebench/installed.md
@@ -54,6 +54,8 @@ fallbacks do not multiply the response cost. This is observed API metadata, not
invoice reconciliation. Complete totals require matching client starts, completed
transport receipts, request/response files and valid cost fields. Missing or
invalid evidence leaves the total unknown while preserving a known subtotal.
+Retained response costs still contribute to that subtotal when the transport log
+or a request-start entry is missing.
These observations never enter the scored Sol task cost or change cost-win rules.
`aggregate --plan ...` requires ten terminal attempts with official grading receipts,
diff --git a/evals/implementation/swebench/installed.py b/evals/implementation/swebench/installed.py
index f1af330..6174744 100644
--- a/evals/implementation/swebench/installed.py
+++ b/evals/implementation/swebench/installed.py
@@ -427,7 +427,8 @@ def observed_jev(out, events, log_valid, traces_copied):
costs, inputs, outputs, attempts = [], [], [], []
def count(value):
return type(value) is int and value >= 0
- for identifier in sorted(safe_ids):
+ # Retained response metadata remains known even if transport logs were lost.
+ for identifier in sorted(responses):
try:
response = responses[identifier]
body = json.loads(response.read_text())
diff --git a/evals/implementation/swebench/test_installed.py b/evals/implementation/swebench/test_installed.py
index 26630e1..8831e31 100644
--- a/evals/implementation/swebench/test_installed.py
+++ b/evals/implementation/swebench/test_installed.py
@@ -198,6 +198,23 @@ class InstalledTests(unittest.TestCase):
events[-1]['transportError']='Interrupted'
self.assertIsNone(runner.observed_jev(root,events,True,True)['observed_cost_usd'])
+ def test_retained_jev_cost_survives_missing_log_or_request_start(self):
+ for missing_log in [True, False]:
+ with self.subTest(missing_log=missing_log),tempfile.TemporaryDirectory() as temporary:
+ path,plan=self.fixture(Path(temporary));out=self.receipt(path,plan)
+ events=self.jev_fixture(out)
+ if not missing_log:
+ (out/'proxy.jsonl').write_text('\n'.join(json.dumps(event) for event in events[1:]))
+ receipt=json.loads((out/'receipt.json').read_text());receipt['jev_traces_copied']=True;runner.write_json(out/'receipt.json',receipt)
+ with patch.object(runner.urllib.request,'urlopen',side_effect=AssertionError('No network')),contextlib.redirect_stdout(io.StringIO()):
+ runner.account(argparse.Namespace(plan=path,task=None))
+ result=json.loads((out/'generation-accounting.json').read_text())['jev']
+ self.assertEqual(result['known_cost_usd'],0.03)
+ self.assertEqual(result['responses_with_cost'],2)
+ self.assertEqual(result['known_input_tokens'],20)
+ self.assertEqual(result['known_provider_attempts'],3)
+ self.assertFalse(result['complete']);self.assertIsNone(result['observed_cost_usd'])
+
def test_observed_jev_cost_does_not_change_scored_sol_cost_win(self):
with tempfile.TemporaryDirectory() as temporary:
path,plan=self.fixture(Path(temporary));out=self.receipt(path,plan)