Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Report observed Jev costs separately from scored Sol billing

David Zhang58913664465b
diff --git a/evals/implementation/swebench/installed.md b/evals/implementation/swebench/installed.md
index a849eff..fcf78af 100644
--- a/evals/implementation/swebench/installed.md
+++ b/evals/implementation/swebench/installed.md
@@ -47,6 +47,15 @@ official harness and unique run IDs. Accounting reuses saved generation metadata
fetches missing receipts, and leaves full cost unknown if any request is missing.
Neither operation reruns the coding agent.
+Jev is reported separately under `jev`: the observed cost sums each retained
+response's `providerMetadata.gateway.cost` once. Input/output tokens and internal
+provider attempt counts are reported separately from client calls; provider
+fallbacks do not multiply the response cost. This is observed API metadata, not
+invoice reconciliation. Complete totals require matching client starts, completed
+transport receipts, request/response files and valid cost fields. Missing or
+invalid evidence leaves the total unknown while preserving a known subtotal.
+These observations never enter the scored Sol task cost or change cost-win rules.
+
`aggregate --plan ...` requires ten terminal attempts with official grading receipts,
all eight baseline solves preserved by valid solved treatments, and at least seven
valid solved results with complete billing and strictly lower cost. Failed or
diff --git a/evals/implementation/swebench/installed.py b/evals/implementation/swebench/installed.py
index 695d2ae..f1af330 100644
--- a/evals/implementation/swebench/installed.py
+++ b/evals/implementation/swebench/installed.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
"""Freeze and evaluate one installed package against the ten retained SWE-bench baselines."""
import argparse
+from decimal import Decimal, InvalidOperation
import hashlib
import json
import math
@@ -410,6 +411,63 @@ def valid_cost(value):
return type(value) in (int, float) and math.isfinite(value) and value >= 0
+def observed_jev(out, events, log_valid, traces_copied):
+ """Sum response-level gateway.cost once; provider retries are counts, not extra prices."""
+ starts = [event for event in events if event.get('kind') == 'jev-request-start']
+ ids = [event.get('requestId') for event in starts]
+ safe_ids = {identifier for identifier in ids if isinstance(identifier, str) and re.fullmatch(r'[a-f0-9]{32}', identifier)}
+ traces = out / 'jev-traces'
+ responses = {path.name.removesuffix('.response.json'): path for path in traces.glob('*.response.json')}
+ requests = {path.name.removesuffix('.request.json'): path for path in traces.glob('*.request.json')}
+ ends = [event for event in events if event.get('kind') == 'gateway']
+ end_ids = [event.get('requestId') for event in ends]
+ safe_end_ids = {identifier for identifier in end_ids if isinstance(identifier, str) and re.fullmatch(r'[a-f0-9]{32}', identifier)}
+ coverage = (log_valid and traces_copied is True and len(safe_ids) == len(ids) and
+ safe_ids == set(responses) == set(requests) == safe_end_ids and len(safe_end_ids) == len(ends) == len(ids))
+ costs, inputs, outputs, attempts = [], [], [], []
+ def count(value):
+ return type(value) is int and value >= 0
+ for identifier in sorted(safe_ids):
+ try:
+ response = responses[identifier]
+ body = json.loads(response.read_text())
+ gateway = body['providerMetadata']['gateway']
+ value = gateway.get('cost')
+ if type(value) not in (str, int, float):
+ raise ValueError('Missing or invalid response cost')
+ cost = Decimal(str(value))
+ if not cost.is_finite() or cost < 0 or not math.isfinite(float(cost)):
+ raise ValueError('Invalid response cost')
+ costs.append(cost)
+ usage = body.get('usage', {})
+ if count(usage.get('inputTokens')): inputs.append(usage['inputTokens'])
+ if count(usage.get('outputTokens')): outputs.append(usage['outputTokens'])
+ provider_attempts = gateway.get('routing', {}).get('totalProviderAttemptCount')
+ if count(provider_attempts): attempts.append(provider_attempts)
+ matching = [event for event in ends if event.get('requestId') == identifier]
+ if len(matching) != 1:
+ coverage = False
+ continue
+ end = matching[0]
+ if (end.get('status') != 200 or end.get('transportError') or end.get('incompleteStream') or
+ end.get('responseBytes') != response.stat().st_size or end.get('requestBytes') != requests[identifier].stat().st_size):
+ coverage = False
+ except (OSError, ValueError, KeyError, TypeError, AttributeError, InvalidOperation):
+ coverage = False
+ complete = coverage and len(costs) == len(ids)
+ known = float(sum(costs, Decimal(0)))
+ if not math.isfinite(known):
+ known = None
+ complete = False
+ return {'basis': 'Retained response providerMetadata.gateway.cost; observed API metadata, not invoice reconciliation',
+ 'included_in_scored_task_cost': False, 'client_calls': len(ids), 'responses_with_cost': len(costs),
+ 'complete': complete, 'observed_cost_usd': known if complete else None, 'known_cost_usd': known,
+ 'input_tokens': sum(inputs) if coverage and len(inputs) == len(ids) else None,
+ 'output_tokens': sum(outputs) if coverage and len(outputs) == len(ids) else None,
+ 'provider_attempts': sum(attempts) if coverage and len(attempts) == len(ids) else None,
+ 'known_input_tokens': sum(inputs), 'known_output_tokens': sum(outputs), 'known_provider_attempts': sum(attempts)}
+
+
def account(args):
plan, _ = load_plan(args.plan.resolve(), args.task)
out = Path(plan['output'])
@@ -448,7 +506,7 @@ def account(args):
write_json(path, lookups)
complete = log_valid and bool(identifiers) and len(request_ids) == len(starts) == len(ends) == len(identifiers) == len(lookups) and request_ids == {event.get('requestId') for event in ends} == {event.get('requestId') for event in observations} and all(event.get('streamTerminal') == 'response.completed' and not event.get('incompleteStream') and not event.get('transportError') and event.get('status') == 200 for event in ends) and all(valid_cost(row.get('metadata', {}).get('total_cost')) for row in lookups)
known = sum(row.get('metadata', {}).get('total_cost', 0) for row in lookups if valid_cost(row.get('metadata', {}).get('total_cost')))
- result = {'cell': plan['cell']['id'], 'request_starts': len(starts), 'generation_ids': len(identifiers), 'all_requests_accounted': complete, 'known_gateway_cost_usd': known, 'gateway_cost_usd': known if complete else None, 'baseline_cost_usd': plan['baseline_cost_usd'], 'jev_cost_usd': 0}
+ result = {'cell': plan['cell']['id'], 'request_starts': len(starts), 'generation_ids': len(identifiers), 'all_requests_accounted': complete, 'known_gateway_cost_usd': known, 'gateway_cost_usd': known if complete else None, 'baseline_cost_usd': plan['baseline_cost_usd'], 'jev': observed_jev(out, events, log_valid, receipt.get('jev_traces_copied'))}
grading = out / 'grading-receipt.json'
if grading.exists():
grade_receipt = json.loads(grading.read_text())
@@ -487,6 +545,7 @@ def aggregate(args):
'terminal': terminal, 'grading_attempted': grade_attempted, 'graded': graded, 'protocol_valid': protocol, 'official_resolved': solved, 'resolved': solved and protocol,
'baseline_resolved': item['baseline_resolved'], 'baseline_cost_usd': item['baseline_cost_usd'],
'fully_billed': billed, 'gateway_cost_usd': cost,
+ 'jev': billing.get('jev', {}) if belongs and billing.get('cell') == item['cell']['id'] else {},
'known_gateway_cost_usd': billing.get('known_gateway_cost_usd') if belongs and billing.get('cell') == item['cell']['id'] and valid_cost(billing.get('known_gateway_cost_usd')) else None,
'successful_cost_win': solved and protocol and billed and cost < item['baseline_cost_usd']})
full = {row['task'] for row in rows} == {item['task'] for item in REGISTRY['tasks']}
@@ -498,6 +557,11 @@ def aggregate(args):
'accepted': complete and preserved == REGISTRY['baseline_solves'] and wins >= REGISTRY['required_cost_wins'],
'known_gateway_subtotal_usd': sum(row['known_gateway_cost_usd'] for row in rows if row['known_gateway_cost_usd'] is not None),
'fully_billed_total_usd': sum(row['gateway_cost_usd'] for row in rows) if all(row['fully_billed'] for row in rows) else None,
+ 'jev': {'included_in_scored_task_cost': False,
+ 'basis': 'Observed response API metadata, not invoice reconciliation',
+ 'complete': all(row['jev'].get('complete') is True and valid_cost(row['jev'].get('observed_cost_usd')) for row in rows),
+ 'known_cost_usd': sum(row['jev']['known_cost_usd'] for row in rows if valid_cost(row['jev'].get('known_cost_usd'))),
+ 'observed_cost_usd': sum(row['jev']['observed_cost_usd'] for row in rows) if all(row['jev'].get('complete') is True and valid_cost(row['jev'].get('observed_cost_usd')) for row in rows) else None},
'cells': rows}
write_json(args.plan.resolve().parent / 'aggregate.json', result)
print(json.dumps(result))
diff --git a/evals/implementation/swebench/test_installed.py b/evals/implementation/swebench/test_installed.py
index f6b3b8a..26630e1 100644
--- a/evals/implementation/swebench/test_installed.py
+++ b/evals/implementation/swebench/test_installed.py
@@ -160,6 +160,64 @@ class InstalledTests(unittest.TestCase):
with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))
self.assertFalse(result['prospective_full_cohort']);self.assertFalse(result['accepted'])
+ def jev_fixture(self, root):
+ traces=root/'jev-traces';traces.mkdir()
+ events=[]
+ for index,cost in enumerate(['0.01','0.02']):
+ identifier=str(index)*32
+ request=traces/(identifier+'.request.json');request.write_text('{}')
+ response=traces/(identifier+'.response.json')
+ runner.write_json(response,{'usage':{'inputTokens':10,'outputTokens':2},'providerMetadata':{'gateway':{'cost':cost,'routing':{'totalProviderAttemptCount':index+1}}}})
+ events.extend([{'kind':'jev-request-start','requestId':identifier},
+ {'kind':'gateway','requestId':identifier,'status':200,'requestBytes':request.stat().st_size,'responseBytes':response.stat().st_size}])
+ return events
+
+ def test_jev_reports_response_cost_once_and_provider_retry_counts_separately(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ root=Path(temporary);events=self.jev_fixture(root)
+ result=runner.observed_jev(root,events,True,True)
+ self.assertTrue(result['complete']);self.assertEqual(result['observed_cost_usd'],0.03)
+ self.assertEqual(result['client_calls'],2);self.assertEqual(result['provider_attempts'],3)
+ self.assertEqual(result['input_tokens'],20);self.assertEqual(result['output_tokens'],4)
+ self.assertFalse(result['included_in_scored_task_cost'])
+
+ def test_jev_missing_invalid_or_unfinished_responses_leave_total_unknown(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ root=Path(temporary);events=self.jev_fixture(root)
+ response=root/'jev-traces'/('1'*32+'.response.json')
+ original=response.read_bytes()
+ for value in [b'{',b'{}',b'{"providerMetadata":{"gateway":{"cost":"NaN"}}}',b'{"providerMetadata":{"gateway":{"cost":true}}}']:
+ response.write_bytes(value)
+ result=runner.observed_jev(root,events,True,True)
+ self.assertFalse(result['complete']);self.assertIsNone(result['observed_cost_usd'])
+ self.assertEqual(result['known_cost_usd'],0.01)
+ response.write_bytes(original)
+ for log_valid,copied,extra in [(False,True,[]),(True,False,[]),(True,True,[events[0]])]:
+ result=runner.observed_jev(root,events+extra,log_valid,copied)
+ self.assertFalse(result['complete']);self.assertIsNone(result['observed_cost_usd'])
+ events[-1]['transportError']='Interrupted'
+ self.assertIsNone(runner.observed_jev(root,events,True,True)['observed_cost_usd'])
+
+ def test_observed_jev_cost_does_not_change_scored_sol_cost_win(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ path,plan=self.fixture(Path(temporary));out=self.receipt(path,plan)
+ events=self.jev_fixture(out)
+ identifier='0'*32;response=out/'jev-traces'/(identifier+'.response.json')
+ body=json.loads(response.read_text());body['providerMetadata']['gateway']['cost']='100';runner.write_json(response,body)
+ events[1]['responseBytes']=response.stat().st_size
+ events.extend([{'kind':'codex-request-start','operation':'generation','requestId':'sol'},
+ {'kind':'codex-generation','requestId':'sol','generationId':'gen_sol'},
+ {'kind':'codex-gateway','requestId':'sol','status':200,'streamTerminal':'response.completed'}])
+ (out/'proxy.jsonl').write_text('\n'.join(json.dumps(event) for event in events))
+ receipt=json.loads((out/'receipt.json').read_text());receipt['jev_traces_copied']=True;runner.write_json(out/'receipt.json',receipt)
+ runner.write_json(out/'generation-lookups.json',[{'id':'gen_sol','metadata':{'id':'gen_sol','model':'openai/gpt-5.6-sol','total_cost':0.1}}])
+ runner.write_json(out/'grading-receipt.json',{'run_id':'jg-'+plan['cell']['id'],'exit_code':0,'resolved_instances':1})
+ with patch.object(runner.urllib.request,'urlopen',side_effect=AssertionError('No network')),contextlib.redirect_stdout(io.StringIO()):
+ runner.account(argparse.Namespace(plan=path,task=None))
+ result=json.loads((out/'generation-accounting.json').read_text())
+ self.assertEqual(result['gateway_cost_usd'],0.1);self.assertTrue(result['successful_cost_win'])
+ self.assertEqual(result['jev']['observed_cost_usd'],100.02)
+
def test_broker_captures_exact_jev_bodies_without_auth_headers(self):
with tempfile.TemporaryDirectory() as temporary:
root=Path(temporary);config=root/'gateway.json';config.write_text(json.dumps({'key':'REAL_KEY_SENTINEL','token':'BROKER_TOKEN_SENTINEL','allow_jev':True,'agent_engine':'codex'}))