Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Add frozen installed-package Requests checkpoint runner
David Zhangc41fbbe3be3f
diff --git a/evals/implementation/swebench/gateway_broker.py b/evals/implementation/swebench/gateway_broker.pynew file mode 100644index 0000000..fba46ca--- /dev/null+++ b/evals/implementation/swebench/gateway_broker.py@@ -0,0 +1,147 @@+"""Isolated gateway broker; opt-in Jev body receipts never include authentication headers."""+import socket,socketserver,select,json,threading,os,uuid,re+LOG_LOCK=threading.Lock()+ALLOW=set() # All model traffic uses the authenticated local broker.+class Handler(socketserver.StreamRequestHandler):+ def handle(self):+ line=self.rfile.readline(8192).decode('ascii','replace').strip(); parts=line.split()+ while True:+ h=self.rfile.readline(8192)+ if h in (b'\r\n',b'\n',b''):break+ if len(parts)!=3 or parts[0]!='CONNECT':self.wfile.write(b'HTTP/1.1 403 Forbidden\r\n\r\n');return+ host,_,port=parts[1].rpartition(':');allowed=host.lower() in ALLOW and port=='443'+ with LOG_LOCK: print(json.dumps({'host':host,'port':port,'allowed':allowed}),flush=True)+ if not allowed:self.wfile.write(b'HTTP/1.1 403 Forbidden\r\n\r\n');return+ try:remote=socket.create_connection((host,443),timeout=20)+ except OSError:self.wfile.write(b'HTTP/1.1 502 Bad Gateway\r\n\r\n');return+ self.wfile.write(b'HTTP/1.1 200 Connection Established\r\n\r\n');self.wfile.flush()+ with remote:+ peers=[self.connection,remote]+ while True:+ ready,_,_=select.select(peers,[],[],120)+ if not ready:return+ for src in ready:+ data=src.recv(65536)+ if not data:return+ (remote if src is self.connection else self.connection).sendall(data)+class Server(socketserver.ThreadingTCPServer):+ allow_reuse_address=True+ daemon_threads=True+# Only this separate, non-agent container ever receives the real gateway key.+import http.server,http.client,urllib.request,urllib.error,urllib.parse,pathlib,time,hmac+CONFIG_PATH=os.environ.get('JEVGREP_GATEWAY_CONFIG','/run/gateway.json')+GATEWAY_ORIGIN='https://ai-gateway.vercel.sh'+TRACE_DIR=os.environ.get('JEVGREP_TRACE_DIR')+def capture_jev(request_id,data):+ if not TRACE_DIR:return None+ directory=pathlib.Path(TRACE_DIR);directory.mkdir(mode=0o700,parents=True,exist_ok=True)+ request=directory/(request_id+'.request.json')+ with request.open('xb') as output:os.chmod(request,0o600);output.write(data)+ response=directory/(request_id+'.response.json')+ output=response.open('xb');os.chmod(response,0o600);return output+def generation_id(value):+ if not isinstance(value,dict):return None+ value=value.get('id')+ return value if isinstance(value,str) and len(value)<=128 and re.fullmatch(r'gen_[A-Za-z0-9]+',value) else None+class StreamCompletion:+ def __init__(self):self.pending=bytearray();self.data=[];self.terminal=None;self.after_cr=False;self.generation_id=None+ def feed(self,chunk):+ for byte in chunk:+ if self.after_cr and byte==10:self.after_cr=False;continue+ self.after_cr=byte==13+ if byte not in (10,13):self.pending.append(byte);continue+ line=bytes(self.pending);self.pending.clear()+ if line.startswith(b'data:'):self.data.append(line[5:].lstrip())+ elif not line:+ try:+ event=json.loads(b'\n'.join(self.data))+ if isinstance(event,dict):+ if event.get('type') in ('message_stop','error','response.completed','response.failed','response.incomplete'):self.terminal=event['type']+ if event.get('type')=='message_start':self.generation_id=generation_id(event.get('message'))+ if event.get('type') in ('response.created','response.completed'):self.generation_id=generation_id(event.get('response'))+ except ValueError:pass+ self.data=[]+class Gateway(http.server.BaseHTTPRequestHandler):+ def log_message(self,*args):pass+ def do_POST(self):+ started=time.monotonic();sent=False;request_id=uuid.uuid4().hex;response_trace=None+ try:+ config=json.loads(pathlib.Path(CONFIG_PATH).read_text())+ if not hmac.compare_digest(self.headers.get('Authorization',''),'Bearer '+config['token']):+ self.send_error(403);return+ route=urllib.parse.urlsplit(self.path)+ claude=not route.scheme and not route.netloc and route.path in ('/claude-code/v1/messages','/claude-code/v1/messages/count_tokens')+ codex=self.path=='/codex/v1/responses'+ engine='codex' if codex else 'claude' if claude else 'jev'+ if (codex or claude) and config.get('agent_engine')!=engine:self.send_error(403);return+ jev=self.path=='/evaluation-model' and self.headers.get('ai-model-id')=='typesafe-ai/jev'+ if jev and config.get('allow_jev') is not True:self.send_error(403);return+ if not (claude or codex or jev):self.send_error(403);return+ size=int(self.headers.get('Content-Length','0'))+ if size<1 or size>2000000:self.send_error(413);return+ data=self.rfile.read(size)+ if jev:response_trace=capture_jev(request_id,data)+ headers={'Authorization':'Bearer '+config['key'],'Content-Type':'application/json'}+ if codex:+ body=json.loads(data)+ if not isinstance(body,dict) or body.get('model')!='openai/gpt-5.6-sol':+ self.send_error(403);return+ target=GATEWAY_ORIGIN+'/v1/responses'+ elif claude:+ body=json.loads(data)+ if not isinstance(body,dict) or body.get('model') not in ('anthropic/claude-opus-5','claude-opus-5'):+ self.send_error(403);return+ for header in ('anthropic-version','anthropic-beta'):+ if self.headers.get(header):headers[header]=self.headers[header]+ target=GATEWAY_ORIGIN+self.path+ else:+ target=GATEWAY_ORIGIN+'/v4/ai/evaluation-model'+ headers.update({'ai-model-id':'typesafe-ai/jev','ai-evaluation-model-specification-version':'4','ai-gateway-protocol-version':'0.0.1','ai-gateway-auth-method':'api-key'})+ with LOG_LOCK:print(json.dumps({'kind':engine+'-request-start','requestId':request_id,'startedAt':time.time(),'operation':'count_tokens' if route.path.endswith('/count_tokens') else 'generation','streamed':bool(body.get('stream')) if claude or codex else False}),flush=True)+ request=urllib.request.Request(target,data=data,headers=headers)+ try:response=urllib.request.urlopen(request,timeout=90)+ except urllib.error.HTTPError as error:response=error+ with response:+ status=response.status;self.send_response(status)+ self.send_header('Content-Type',response.headers.get('Content-Type','application/json'))+ self.send_header('Connection','close');self.end_headers();sent=True;response_bytes=0+ streaming=(claude or codex) and body.get('stream') is True and 'text/event-stream' in response.headers.get('Content-Type','')+ completion=StreamCompletion();transport_error=None;observed_generation=None;metadata_body=bytearray();metadata_oversized=False+ try:+ while True:+ chunk=response.read1(16384)+ if not chunk:break+ if streaming:+ completion.feed(chunk)+ if completion.generation_id and completion.generation_id!=observed_generation:+ observed_generation=completion.generation_id+ with LOG_LOCK:print(json.dumps({'kind':engine+'-generation','requestId':request_id,'generationId':observed_generation,'receivedAt':time.time()}),flush=True)+ elif (codex or (claude and route.path=='/claude-code/v1/messages')) and not metadata_oversized:+ if len(metadata_body)+len(chunk)<=2000000:metadata_body.extend(chunk)+ else:metadata_body.clear();metadata_oversized=True+ if response_trace:response_trace.write(chunk)+ self.wfile.write(chunk);self.wfile.flush();response_bytes+=len(chunk)+ except (OSError,http.client.HTTPException) as error:transport_error=type(error).__name__+ if (claude or codex) and not streaming and metadata_body:+ try:observed_generation=generation_id(json.loads(metadata_body))+ except ValueError:pass+ if observed_generation:+ with LOG_LOCK:print(json.dumps({'kind':engine+'-generation','requestId':request_id,'generationId':observed_generation,'receivedAt':time.time()}),flush=True)+ incomplete=streaming and completion.terminal is None+ if incomplete and claude:+ # Close any partial SSE event before sending a client-visible failure.+ failure=b'\n\nevent: error\ndata: {"type":"error","error":{"type":"api_error","message":"Upstream response interrupted before completion"}}\n\n'+ try:self.wfile.write(failure);self.wfile.flush()+ except OSError:pass+ with LOG_LOCK:print(json.dumps({'kind':engine+'-gateway' if claude or codex else 'gateway','requestId':request_id,'generationId':observed_generation,'status':status,'requestBytes':len(data),'responseBytes':response_bytes,'streamTerminal':completion.terminal if streaming else None,'incompleteStream':incomplete,'transportError':transport_error,'elapsedSeconds':time.monotonic()-started}),flush=True)+ except (OSError,ValueError,KeyError,http.client.HTTPException) as error:+ with LOG_LOCK:print(json.dumps({'kind':'broker-error','requestId':request_id,'headersSent':sent,'error':type(error).__name__,'elapsedSeconds':time.monotonic()-started}),flush=True)+ if not sent:+ try:self.send_error(502)+ except OSError:pass+ finally:+ if response_trace:response_trace.close()+ self.close_connection=True+if __name__=='__main__':+ threading.Thread(target=http.server.ThreadingHTTPServer(('0.0.0.0',3129),Gateway).serve_forever,daemon=True).start()+ Server(('0.0.0.0',3128),Handler).serve_forever()diff --git a/evals/implementation/swebench/installed.md b/evals/implementation/swebench/installed.mdnew file mode 100644index 0000000..0f36a47--- /dev/null+++ b/evals/implementation/swebench/installed.md@@ -0,0 +1,46 @@+# Installed Requests checkpoint++[The runner](installed.py) compares one frozen installed `jg` package with the+retained Requests Sol baseline. It refuses changed model, source, runtime and+baseline-prompt identities. It never launches a baseline. A successful checkpoint+is integration evidence, not the ten-task quality gate.++Preparation installs the packed npm artifact and its dependencies in a disposable+copy of the retained Linux image. Registry access is confined to preparation;+the agent receives the archived installed prefix on an internal network. It runs+the real `jg` executable. The packaged skill is verified against the canonical+skill, and the benchmark explicitly requires initial retrieval even when the+production skill would consider it optional.++Use `python3 evals/implementation/swebench/installed.py --help` for commands.+The intended root script alias is `eval:swebench` pointing to that entry point.+A typical sequence is:++```sh+python3 evals/implementation/swebench/installed.py prepare \+ --package /absolute/path/to/package.tgz --output /absolute/path/to/new-study+python3 evals/implementation/swebench/installed.py run \+ --plan /absolute/path/to/new-study/plan.json --dry-run+```++Preparation and dry-run make no model calls. After reviewing the frozen plan,+`run` without `--dry-run` executes the paid treatment exactly once. Load the+Gateway credential through the authorized environment workflow; never place a+real key in the plan or command arguments. Each attempt uses fresh home/cache+state and the package's default retrieval policy.++Follow execution with `grade --plan ...` and `account --plan ...`. Grading uses+the existing pinned official environment and a unique run ID. Accounting reuses+saved generation metadata, fetches missing receipts, and leaves the full cost+unknown if any generation is unaccounted for. A missing required retrieval call+cannot count as a successful cost win.++The plan, installation archive, raw agent output, patch, official reports and+billing remain under ignored run storage. [The broker](gateway_broker.py) retains+raw Jev request/response bodies separately from transport metadata, only in the+non-agent container; authentication headers are never captured. These receipts+contain benchmark source and should stay local with the other study evidence.++Run [the focused validations](test_installed.py) inside an isolated Docker+container. They exercise frozen prompt identity, drift rejection, no-call dry-run,+partial billing and exact Jev body capture through a local synthetic provider.diff --git a/evals/implementation/swebench/installed.py b/evals/implementation/swebench/installed.pynew file mode 100644index 0000000..f5c57e7--- /dev/null+++ b/evals/implementation/swebench/installed.py@@ -0,0 +1,387 @@+#!/usr/bin/env python3+"""Prepare, execute and grade one frozen installed-package SWE-bench treatment."""+import argparse+import hashlib+import json+import math+import os+from pathlib import Path+import re+import shutil+import subprocess+import time+import urllib.parse+import urllib.request+import uuid++HERE = Path(__file__).resolve().parent+ROOT = HERE.parents[2]+RUNS = ROOT / 'evals/runs/swebench'+TOOLING = ROOT / 'evals/runs/tooling/swebench'+TASK = 'psf__requests-1142'+BASELINE_ID = 'lookahead-native-v88-psf__requests-1142-codex-baseline'+RUNTIME_IMAGE = 'sha256:2264606b0765897361bb392db8aa3226e171719ba97e82ce5205ba122a43eaff'+HARNESS_COMMIT = '02e7a74ffd0b707aab73d203fe87bdc7c76afc8e'+BASELINE = RUNS / 'lookahead-native-v88' / TASK / 'codex-baseline'+SAFE_FIELDS = {'instance_id', 'repo', 'base_commit', 'image', 'problem_statement'}+VERSIONS = {'node': 'v24.14.0', 'codex': 'codex-cli 0.153.4', 'claude': '2.1.278 (Claude Code)'}+VERIFICATION = (+ 'Save test stdout/stderr to a log and preserve the test exit status. Inspect that log '+ 'to understand failures; do not rerun unchanged tests merely to change an output filter. '+ 'Prefer foreground test commands with an adequate tool timeout within the task deadline. '+ 'In Claude, a Bash timeout may background a command rather than stop it. If backgrounded, '+ 'use its completion notification and output or exit-status file; do not invent process-name '+ 'polling with pgrep or add fixed sleeps. '+ 'Keep your implementation applied in /testbed during long-running verification. '+ 'If a pristine comparison is needed, use a separate temporary checkout and verify tests import '+ 'that checkout. Report verification that could not finish.'+)+++def baseline_prompt(row):+ common = ('Implement the following request in /testbed. Work only from local source and tests; '+ 'do not fetch external research, packages, solutions or hidden tests. Do not use network tools. '+ 'Preserve existing behavior outside the request. Run relevant available tests and wait for their results. '+ 'Report what changed, completed verification, and any unresolved limitations. '+ 'Do not change Git history or make commits.\n\n')+ return common + 'The entire task, including verification, has a 900-second deadline. ' + VERIFICATION + '\n\n' + row['problem_statement']+++def treatment_prompt(row):+ return ('$jevgrep\n\nFor this benchmark, you must run jg for initial repository research before '+ 'independently inspecting implementation files, even when the affected path seems known. '+ 'Read and follow /home/agent/.agents/skills/jevgrep/SKILL.md, then invoke the installed jg '+ 'command. This benchmark requirement overrides the skill\'s optional invocation guidance.\n\n'+ + baseline_prompt(row))+++def digest(path):+ return hashlib.sha256(Path(path).read_bytes()).hexdigest()+++def write_json(path, value):+ path.write_text(json.dumps(value, indent=2) + '\n')+ path.chmod(0o600)+++def command(args, **kwargs):+ return subprocess.run(args, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)+++def text(args):+ return command(args).stdout.decode().strip()+++def load_pair(baseline, inputs, task):+ receipt = json.loads((baseline / 'receipt.json').read_text())+ cell = receipt['cell']+ if task != TASK or cell['instance_id'] != task or cell['arm'] != 'baseline' or cell['id'] != BASELINE_ID or receipt['image'] != RUNTIME_IMAGE or cell['image'] != RUNTIME_IMAGE:+ raise ValueError('This checkpoint reuses only the registered Requests baseline')+ if (cell['model'], cell['effort'], cell['timeout_seconds']) != ('openai/gpt-5.6-sol', 'medium', 900):+ raise ValueError('Baseline model/effort/deadline differs from the registered harness')+ if receipt['versions'] != VERSIONS or receipt['provider_route'] != 'vercel-ai-gateway':+ raise ValueError('Baseline runtime or provider differs from the registered harness')+ rows = json.loads(inputs.read_text())+ if not isinstance(rows, list) or any(not isinstance(row, dict) or set(row) != SAFE_FIELDS for row in rows):+ raise ValueError('Only safe exported agent-input fields are accepted')+ matches = [row for row in rows if row['instance_id'] == task]+ if len(matches) != 1 or matches[0]['image'] != cell['source_image']:+ raise ValueError('Task input must match the fixed baseline source image')+ row = matches[0]+ prompt = baseline_prompt(row).encode()+ if hashlib.sha256(prompt).hexdigest() != receipt['prompt_sha256'] or (baseline / 'prompt.txt').read_bytes() != prompt:+ raise ValueError('Baseline prompt differs; do not rerun or overwrite it')+ grade = json.loads((baseline / 'grading-receipt.json').read_text())+ if grade.get('resolved_instances') != 1:+ raise ValueError('Expected the retained officially resolved Requests baseline')+ return receipt, row+++def check_image(image):+ if not re.fullmatch(r'sha256:[a-f0-9]{64}', image):+ raise ValueError('Use the immutable retained runtime image ID')+ return text(['docker', 'image', 'inspect', image, '--format', '{{.Id}}'])+++def prepare(args):+ baseline, row = load_pair(args.baseline.resolve(), args.inputs.resolve(), args.task)+ image = check_image(baseline['image'])+ out = args.output.resolve()+ out.mkdir(parents=True, exist_ok=False)+ out.chmod(0o700)+ package = out / 'package.tgz'+ shutil.copyfile(args.package, package)+ package.chmod(0o600)+ name = 'jg-prepare-' + uuid.uuid4().hex[:12]+ # Preparation may fetch pinned npm dependencies. The coding agent never receives that network.+ install = '''set -eu+cd /tmp+npm install --global --prefix /opt/jg-install /tmp/package.tgz --ignore-scripts --no-audit --no-fund+/opt/jg-install/bin/jg --version > /tmp/jg-version.txt+/opt/jg-install/bin/jg skill > /tmp/jg-skill.md+node -e 'const fs=require("fs"),cp=require("child_process");const p=require("/opt/jg-install/lib/node_modules/@dzhng/jevgrep/package.json");if(fs.realpathSync("/opt/jg-install/bin/jg")!=="/opt/jg-install/lib/node_modules/@dzhng/jevgrep/dist/bin/index.js")throw Error("Unexpected executable");const git=a=>cp.execFileSync("git",["-C","/testbed",...a],{encoding:"utf8"}).trim();fs.writeFileSync("/tmp/install.json",JSON.stringify({package:p.name,version:p.version,head:git(["rev-parse","HEAD"]),tree:git(["rev-parse","HEAD^{tree}"]),status:git(["status","--porcelain"])}));'+tar -C /opt -cf /tmp/installed-prefix.tar jg-install+'''+ try:+ command(['docker', 'create', '--platform', 'linux/amd64', '--name', name, image, 'sh', '-c', install])+ command(['docker', 'cp', str(package), name + ':/tmp/package.tgz'])+ with (out / 'prepare.log').open('wb') as log:+ subprocess.run(['docker', 'start', '-a', name], stdout=log, stderr=subprocess.STDOUT, check=True)+ for remote, local in [('installed-prefix.tar', 'installed-prefix.tar'), ('install.json', 'installation.json'), ('jg-skill.md', 'skill.md'), ('jg-version.txt', 'version.txt')]:+ command(['docker', 'cp', name + ':/tmp/' + remote, str(out / local)])+ (out / local).chmod(0o600)+ installed = json.loads((out / 'installation.json').read_text())+ cell = baseline['cell']+ if installed['package'] != '@dzhng/jevgrep' or installed['status'] or installed['head'] != cell['expected_image_head'] or installed['tree'] != cell['expected_source_tree']:+ raise ValueError('Installation changed source identity or installed the wrong package')+ if (out / 'skill.md').read_bytes() != args.skill.read_bytes():+ raise ValueError('Packaged skill differs from the canonical skill')+ write_json(out / 'agent-inputs.json', [row])+ cell = {**cell, 'id': 'installed-' + TASK + '-' + uuid.uuid4().hex[:12], 'arm': 'chunks', 'candidate': str(out / 'installed-prefix.tar')}+ artifacts = [Path(__file__).resolve(), HERE / 'gateway_broker.py', package, out / 'installed-prefix.tar', out / 'skill.md', out / 'agent-inputs.json']+ plan = {'status': 'frozen', 'purpose': 'Installed production Requests checkpoint; reuse the fixed baseline once, no baseline rerun.',+ 'cell': cell, 'baseline': str(args.baseline.resolve()), 'baseline_receipt_sha256': digest(args.baseline / 'receipt.json'),+ 'baseline_cost_usd': 0.2685004, 'baseline_resolved': True, 'inputs': str(out / 'agent-inputs.json'),+ 'skill': str(out / 'skill.md'), 'package': str(package), 'output': str(out / 'attempt'),+ 'artifacts': {str(path): digest(path) for path in artifacts}}+ write_json(out / 'plan.json', plan)+ print(json.dumps({'status': 'prepared', 'plan': str(out / 'plan.json'), 'cell': cell['id'], 'image': image}))+ finally:+ subprocess.run(['docker', 'rm', '-f', name], capture_output=True)+++def load_plan(path):+ plan = json.loads(path.read_text())+ if plan.get('status') != 'frozen':+ raise ValueError('Plan must be frozen before execution')+ for artifact, expected in plan['artifacts'].items():+ if digest(artifact) != expected:+ raise ValueError('Frozen artifact changed: ' + artifact)+ required = [Path(__file__).resolve(), HERE / 'gateway_broker.py', Path(plan['inputs']), Path(plan['skill']), Path(plan['package']), Path(plan['cell']['candidate'])]+ if any(str(path) not in plan['artifacts'] for path in required):+ raise ValueError('Plan omits a required artifact')+ base = Path(plan['baseline'])+ if digest(base / 'receipt.json') != plan['baseline_receipt_sha256']:+ raise ValueError('The retained baseline changed')+ baseline, row = load_pair(base, Path(plan['inputs']), plan['cell']['instance_id'])+ for key in ['instance_id', 'engine', 'invocation', 'source_image', 'source_pinned_ref', 'image', 'model', 'effort', 'timeout_seconds', 'gateway_model', 'expected_image_head', 'expected_source_tree']:+ if plan['cell'][key] != baseline['cell'][key]:+ raise ValueError('Treatment pairing changed: ' + key)+ if plan['cell']['arm'] != 'chunks' or plan['cell']['id'] == baseline['cell']['id'] or not re.fullmatch(r'[A-Za-z0-9_.-]+', plan['cell']['id']):+ raise ValueError('Treatment must use its own registered cell')+ return plan, row+++def run(args):+ plan, row = load_plan(args.plan.resolve())+ cell = plan['cell']+ image = check_image(cell['image'])+ if args.dry_run:+ print(json.dumps({'status': 'validated', 'paid_calls': 0, 'baseline_reused': plan['baseline'], 'image': image, 'cell': cell['id']}))+ return+ if not os.environ.get('AI_GATEWAY_API_KEY'):+ raise ValueError('AI_GATEWAY_API_KEY is required; load it through the authorized credential workflow')+ out = Path(plan['output'])+ out.mkdir(parents=True, exist_ok=False)+ out.chmod(0o700)+ tag = 'jg-native-' + uuid.uuid4().hex[:12]+ network, proxy = tag + '-net', tag + '-proxy'+ receipt = {'engine': 'codex', 'cell': cell, 'image': image, 'plan_sha256': digest(args.plan), 'started': time.time(), 'status': 'preparing',+ 'provider_route': 'vercel-ai-gateway', 'gateway_model': 'openai/gpt-5.6-sol', 'baseline': plan['baseline']}+ def dx(argv, user=None):+ return command(['docker', 'exec', *(['-u', user] if user else []), tag, *argv])+ def put(container, path, data, owner='agent:agent', mode='600'):+ command(['docker', 'exec', '-i', container, 'sh', '-c', 'umask 077; cat > "$1"; chown "$2" "$1"; chmod "$3" "$1"', 'sh', path, owner, mode], input=data)+ def capture(remote, destination):+ return subprocess.run(['docker', 'cp', remote, str(destination)], capture_output=True).returncode == 0+ try:+ command(['docker', 'network', 'create', '--internal', network])+ command(['docker', 'run', '-d', '--name', proxy, '-e', 'JEVGREP_TRACE_DIR=/run/jev-traces', image, 'python3', '-u', '-c', (HERE / 'gateway_broker.py').read_text()])+ command(['docker', 'network', 'connect', '--alias', 'model-egress', network, proxy])+ command(['docker', 'run', '-d', '--name', tag, '--network', network,+ '-e', 'HTTP_PROXY=http://model-egress:3128', '-e', 'HTTPS_PROXY=http://model-egress:3128',+ '-e', 'NO_PROXY=localhost,127.0.0.1,model-egress', image, 'sleep', '1800'])+ dx(['sh', '-c', 'useradd -m -u 1001 agent; mkdir -p /home/agent/.codex /home/agent/.claude /workspace; chown -R agent:agent /home/agent /workspace'])+ receipt['official_image_head'] = dx(['git', '-C', '/testbed', 'rev-parse', 'HEAD']).stdout.decode().strip()+ receipt['official_source_status'] = dx(['git', '-C', '/testbed', 'status', '--porcelain']).stdout.decode()+ receipt['official_image_tree'] = dx(['git', '-C', '/testbed', 'rev-parse', 'HEAD^{tree}']).stdout.decode().strip()+ if receipt['official_source_status'] or receipt['official_image_head'] != cell['expected_image_head'] or receipt['official_image_tree'] != cell['expected_source_tree']:+ raise ValueError('Native source differs from the retained baseline')+ dx(['sh', '-c', 'cd /testbed && tracked=$(mktemp) && git ls-files -z > "$tracked" && rm -rf .git && git init -q && git config user.email agent@localhost && git config user.name Agent && git --literal-pathspecs add -f --pathspec-from-file="$tracked" --pathspec-file-nul && rm "$tracked" && git commit -qm "Pristine official image source" && chown -R agent:agent /testbed'])+ base = dx(['git', '-C', '/testbed', 'rev-parse', 'HEAD'], 'agent').stdout.decode().strip()+ receipt['synthetic_base'] = base+ receipt['source_tree'] = dx(['git', '-C', '/testbed', 'rev-parse', 'HEAD^{tree}'], 'agent').stdout.decode().strip()+ if receipt['source_tree'] != cell['expected_source_tree']:+ raise ValueError('Normalized source tree changed')+ receipt['versions'] = {name: dx([name, '--version']).stdout.decode().strip() for name in VERSIONS}+ if receipt['versions'] != VERSIONS:+ raise ValueError('Agent runtime changed')+ command(['docker', 'cp', cell['candidate'], tag + ':/tmp/installed-prefix.tar'])+ dx(['tar', '-xf', '/tmp/installed-prefix.tar', '-C', '/opt'])+ skill_dir = '/home/agent/.agents/skills/jevgrep'+ dx(['mkdir', '-p', skill_dir])+ put(tag, skill_dir + '/SKILL.md', Path(plan['skill']).read_bytes())+ receipt['installed_skills'] = {skill_dir: dx(['sha256sum', skill_dir + '/SKILL.md']).stdout.decode().split()[0]}+ if receipt['installed_skills'][skill_dir] != digest(plan['skill']):+ raise ValueError('Installed skill differs from the frozen package')+ receipt['jg_version'] = dx(['/opt/jg-install/bin/jg', '--version'], 'agent').stdout.decode().strip()+ receipt['candidate_sha256'] = digest(cell['candidate'])+ receipt['package_sha256'] = digest(plan['package'])+ prompt = treatment_prompt(row)+ (out / 'prompt.txt').write_text(prompt)+ receipt['prompt_sha256'] = digest(out / 'prompt.txt')+ token = uuid.uuid4().hex+ put(proxy, '/run/gateway.json', json.dumps({'key': os.environ['AI_GATEWAY_API_KEY'], 'token': token, 'agent_engine': 'codex', 'allow_jev': True}).encode(), 'root:root')+ config = ('model = "openai/gpt-5.6-sol"\nmodel_provider = "vercel"\nmodel_reasoning_effort = "medium"\n'+ '[model_providers.vercel]\nname = "Vercel AI Gateway"\n'+ 'base_url = "http://model-egress:3129/codex/v1"\nenv_key = "JEVGREP_MODEL_TOKEN"\nwire_api = "responses"\n')+ put(tag, '/home/agent/.codex/config.toml', config.encode())+ native = ['codex', 'exec', '--model', 'openai/gpt-5.6-sol', '-c', 'model_reasoning_effort="medium"', '-c', 'web_search="disabled"', '--json', '--dangerously-bypass-approvals-and-sandbox', '--skip-git-repo-check', prompt]+ receipt['command'] = native+ path = '/opt/jg-install/bin:/opt/node-v24.14.0-linux-x64/bin:/opt/miniconda3/envs/testbed/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'+ invocation = ['docker', 'exec', '-u', 'agent', '-e', 'HOME=/home/agent', '-e', 'PATH=' + path,+ '-e', 'CONDA_PREFIX=/opt/miniconda3/envs/testbed', '-e', 'JEVGREP_MODEL_TOKEN=' + token,+ '-e', 'AI_GATEWAY_API_KEY=' + token, '-e', 'AI_GATEWAY_BASE_URL=http://model-egress:3129', '-w', '/testbed', tag, *native]+ receipt['status'] = 'running'+ receipt['agent_started_at'] = time.time()+ start = time.monotonic()+ with (out / 'events.jsonl').open('wb') as stdout, (out / 'stderr.txt').open('wb') as stderr:+ try:+ result = subprocess.run(invocation, stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr, timeout=900)+ receipt['exit_code'] = result.returncode+ except subprocess.TimeoutExpired:+ receipt['timed_out'] = True+ subprocess.run(['docker', 'exec', tag, 'pkill', '-KILL', '-u', '1001'], capture_output=True)+ receipt['agent_elapsed_seconds'] = time.monotonic() - start+ receipt['agent_finished_at'] = time.time()+ receipt['timing_valid'] = abs(receipt['agent_finished_at'] - receipt['agent_started_at'] - receipt['agent_elapsed_seconds']) <= 5+ events = []+ for line in (out / 'events.jsonl').read_text().splitlines():+ try: events.append(json.loads(line))+ except ValueError: pass+ receipt['native_completed'] = any(event.get('type') == 'turn.completed' for event in events)+ executions = [event['item'].get('command', '') for event in events if event.get('type') == 'item.completed' and event.get('item', {}).get('type') == 'command_execution']+ receipt['required_retrieval_observed'] = any(re.search(r'(?<![A-Za-z0-9_])jg\s+(?!doctor\b|skill\b|auth\b|cache\b|--help\b|--version\b)', execution) for execution in executions)+ receipt['usage_events'] = [event for event in events if event.get('type') == 'turn.completed']+ receipt['source_status'] = dx(['git', '-C', '/testbed', 'status', '--porcelain'], 'agent').stdout.decode()+ dx(['git', '-C', '/testbed', 'add', '-A'], 'agent')+ (out / 'agent.patch').write_bytes(dx(['git', '-C', '/testbed', 'diff', '--cached', base, '--binary'], 'agent').stdout)+ receipt['patch_sha256'] = digest(out / 'agent.patch')+ receipt['all_sessions_copied'] = capture(tag + ':/home/agent/.codex/sessions', out / 'codex-sessions')+ sessions = list((out / 'codex-sessions').rglob('*.jsonl'))+ if len(sessions) == 1:+ shutil.copyfile(sessions[0], out / 'raw-rollout.jsonl')+ receipt['status'] = 'completed' if receipt.get('exit_code') == 0 and receipt['native_completed'] and receipt['required_retrieval_observed'] else 'failed'+ except Exception as error:+ receipt['status'] = 'failed'+ # Tool output may contain environment arguments: retain error types, never raw commands or keys.+ receipt['error_type'] = type(error).__name__+ finally:+ logs = subprocess.run(['docker', 'logs', proxy], capture_output=True)+ (out / 'proxy.jsonl').write_bytes(logs.stdout)+ (out / 'proxy-stderr.txt').write_bytes(logs.stderr)+ receipt['jev_traces_copied'] = capture(proxy + ':/run/jev-traces', out / 'jev-traces')+ for container in [tag, proxy]: subprocess.run(['docker', 'rm', '-f', container], capture_output=True)+ subprocess.run(['docker', 'network', 'rm', network], capture_output=True)+ receipt['total_elapsed_seconds'] = time.time() - receipt['started']+ write_json(out / 'receipt.json', receipt)+ print(json.dumps({'status': receipt['status'], 'output': str(out), 'error_type': receipt.get('error_type')}))+ if receipt['status'] != 'completed': raise SystemExit(1)+++def grade(args):+ plan, _ = load_plan(args.plan.resolve())+ out, cell = Path(plan['output']), plan['cell']+ receipt = json.loads((out / 'receipt.json').read_text())+ if receipt['plan_sha256'] != digest(args.plan) or receipt['cell'] != cell:+ raise ValueError('Attempt does not belong to this frozen plan')+ run_id = 'jg-' + cell['id']+ report = args.tooling.resolve() / 'logs/evaluation' / run_id / 'results.json'+ if report.exists() or (out / 'grading-receipt.json').exists():+ raise ValueError('Refusing to reuse an official grading run ID')+ harness = args.tooling.resolve() / 'repo'+ if text(['git', '-C', str(harness), 'rev-parse', 'HEAD']) != HARNESS_COMMIT or text(['git', '-C', str(harness), 'status', '--porcelain']):+ raise ValueError('Official harness checkout changed')+ module = text([str(args.tooling.resolve() / 'venv/bin/python'), '-B', '-c', 'import pathlib,swebench; print(pathlib.Path(swebench.__file__).resolve())'])+ if Path(module).parent.parent != harness:+ raise ValueError('Python environment does not use the pinned official harness')+ prediction = out / 'predictions.jsonl'+ prediction.write_text(json.dumps({'instance_id': cell['instance_id'], 'model_name_or_path': cell['id'], 'model_patch': (out / 'agent.patch').read_text()}) + '\n')+ start = time.monotonic()+ with (out / 'grading-console.log').open('w') as log:+ result = subprocess.run([str(args.tooling.resolve() / 'venv/bin/python'), '-m', 'swebench.harness.run_evaluation', '--dataset_name', str(args.dataset.resolve()), '--predictions_path', str(prediction), '--instance_ids', cell['instance_id'], '--max_workers', '1', '--timeout', '1800', '--run_id', run_id], cwd=args.tooling, stdout=log, stderr=subprocess.STDOUT)+ grading = {'run_id': run_id, 'exit_code': result.returncode, 'grading_seconds': time.monotonic() - start, 'report': str(report)}+ if report.exists():+ summary = json.loads(report.read_text())+ grading.update({key: summary.get(key) for key in ['resolved_instances', 'unresolved_instances', 'infra_failure_instances', 'error_instances']})+ write_json(out / 'grading-receipt.json', grading)+ print(json.dumps(grading))+++def valid_cost(value):+ return type(value) in (int, float) and math.isfinite(value) and value >= 0+++def account(args):+ plan, _ = load_plan(args.plan.resolve())+ out = Path(plan['output'])+ events = [json.loads(line) for line in (out / 'proxy.jsonl').read_text().splitlines()]+ starts = [event for event in events if event.get('kind') == 'codex-request-start' and event.get('operation') == 'generation']+ request_ids = {event['requestId'] for event in starts}+ ends = [event for event in events if event.get('kind') == 'codex-gateway' and event.get('requestId') in request_ids]+ observations = [event for event in events if event.get('kind') == 'codex-generation']+ identifiers = list(dict.fromkeys(event['generationId'] for event in observations))+ path = out / 'generation-lookups.json'+ cached = {row['id']: row for row in json.loads(path.read_text())} if path.exists() else {}+ lookups = []+ for identifier in identifiers:+ if identifier in cached and 'metadata' in cached[identifier]:+ lookups.append(cached[identifier]); continue+ if not os.environ.get('AI_GATEWAY_API_KEY'): raise ValueError('Gateway credential required for uncached generation accounting')+ request = urllib.request.Request('https://ai-gateway.vercel.sh/v1/generation?' + urllib.parse.urlencode({'id': identifier}), headers={'Authorization': 'Bearer ' + os.environ['AI_GATEWAY_API_KEY']})+ try:+ with urllib.request.urlopen(request, timeout=20) as response: metadata = json.load(response)['data']+ if metadata['id'] != identifier or metadata['model'] != 'openai/gpt-5.6-sol': raise ValueError('Generation identity differs')+ lookups.append({'id': identifier, 'metadata': metadata})+ except (OSError, ValueError, KeyError) as error: lookups.append({'id': identifier, 'error_type': type(error).__name__})+ write_json(path, lookups)+ complete = bool(identifiers) and len(starts) == len(ends) == len(identifiers) == len(lookups) and request_ids == {event.get('requestId') for event in ends} == {event.get('requestId') for event in observations} and all(event.get('streamTerminal') == 'response.completed' and not event.get('incompleteStream') and not event.get('transportError') and event.get('status') == 200 for event in ends) and all(valid_cost(row.get('metadata', {}).get('total_cost')) for row in lookups)+ known = sum(row.get('metadata', {}).get('total_cost', 0) for row in lookups if valid_cost(row.get('metadata', {}).get('total_cost')))+ result = {'cell': plan['cell']['id'], 'request_starts': len(starts), 'generation_ids': len(identifiers), 'all_requests_accounted': complete, 'known_gateway_cost_usd': known, 'gateway_cost_usd': known if complete else None, 'baseline_cost_usd': plan['baseline_cost_usd'], 'jev_cost_usd': 0}+ grading = out / 'grading-receipt.json'+ if grading.exists():+ result['official_resolved'] = json.loads(grading.read_text()).get('resolved_instances') == 1+ result['protocol_valid'] = json.loads((out / 'receipt.json').read_text()).get('required_retrieval_observed') is True+ result['successful_cost_win'] = complete and result['official_resolved'] and result['protocol_valid'] and known < plan['baseline_cost_usd']+ write_json(out / 'generation-accounting.json', result)+ print(json.dumps(result))+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ sub = parser.add_subparsers(dest='operation', required=True)+ prep = sub.add_parser('prepare', help='Install a package in the retained image and freeze a new treatment; no model calls')+ prep.add_argument('--package', type=Path, required=True)+ prep.add_argument('--output', type=Path, required=True)+ prep.add_argument('--task', default=TASK)+ prep.add_argument('--baseline', type=Path, default=BASELINE)+ prep.add_argument('--inputs', type=Path, default=RUNS / 'auto-research-80/ten-agent-inputs.json')+ prep.add_argument('--skill', type=Path, default=ROOT / 'skills/jevgrep/SKILL.md')+ execute = sub.add_parser('run', help='Run the paid coding-agent treatment once, or validate without calls')+ execute.add_argument('--plan', type=Path, required=True)+ execute.add_argument('--dry-run', action='store_true')+ grading = sub.add_parser('grade', help='Run the official evaluator under a new run ID')+ grading.add_argument('--plan', type=Path, required=True)+ grading.add_argument('--dataset', type=Path, default=RUNS / 'auto-research-80/evaluator-only/ten-dataset.json')+ grading.add_argument('--tooling', type=Path, default=TOOLING)+ billing = sub.add_parser('account', help='Reconcile full Sol billing; never count unknown charges as zero')+ billing.add_argument('--plan', type=Path, required=True)+ args = parser.parse_args()+ {'prepare': prepare, 'run': run, 'grade': grade, 'account': account}[args.operation](args)+++if __name__ == '__main__':+ try: main()+ except (ValueError, OSError, KeyError, subprocess.CalledProcessError) as error:+ print(json.dumps({'status': 'failed', 'error_type': type(error).__name__, 'message': str(error) if isinstance(error, ValueError) else 'Inspect the retained attempt or preparation log.'}))+ raise SystemExit(1)diff --git a/evals/implementation/swebench/test_installed.py b/evals/implementation/swebench/test_installed.pynew file mode 100644index 0000000..0c1742d--- /dev/null+++ b/evals/implementation/swebench/test_installed.py@@ -0,0 +1,110 @@+"""Focused tests run inside Docker; every writable artifact stays in temporary storage."""+import argparse+import contextlib+import hashlib+import http.client+import http.server+import importlib.util+import io+import json+import os+from pathlib import Path+import tempfile+import threading+import unittest+from unittest.mock import patch++HERE = Path(__file__).resolve().parent++def load(name):+ spec = importlib.util.spec_from_file_location(name, HERE / (name + '.py'))+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module++runner = load('installed')+broker = load('gateway_broker')++class InstalledTests(unittest.TestCase):+ def test_frozen_prompt_and_required_product_call(self):+ row = {'problem_statement': 'Fix behavior.'}+ self.assertEqual(hashlib.sha256(runner.baseline_prompt(row).encode()).hexdigest(), '41c7e3cf7acbc19d0b24ab55e6ea29fe3cdbc2b777549d158488c6c55baaa643')+ prompt = runner.treatment_prompt(row)+ self.assertTrue(prompt.startswith('$jevgrep\n\n'))+ self.assertIn('must run jg for initial repository research', prompt)+ self.assertTrue(prompt.endswith(runner.baseline_prompt(row)))++ def fixture(self, root):+ baseline = root / 'baseline'; baseline.mkdir()+ row = {'instance_id':runner.TASK,'repo':'psf/requests','base_commit':'public-base','image':'official-image','problem_statement':'Fix behavior.'}+ inputs = root / 'agent-inputs.json'; runner.write_json(inputs,[row])+ cell = {'id':runner.BASELINE_ID,'instance_id':runner.TASK,'engine':'codex','arm':'baseline','invocation':'required','source_image':'official-image','source_pinned_ref':'official@sha256:retained','image':runner.RUNTIME_IMAGE,'model':'openai/gpt-5.6-sol','effort':'medium','timeout_seconds':900,'gateway_model':'openai/gpt-5.6-sol','expected_image_head':'a'*40,'expected_source_tree':'b'*40}+ (baseline/'prompt.txt').write_text(runner.baseline_prompt(row))+ runner.write_json(baseline/'receipt.json',{'image':runner.RUNTIME_IMAGE,'cell':cell,'versions':runner.VERSIONS,'provider_route':'vercel-ai-gateway','prompt_sha256':runner.digest(baseline/'prompt.txt')})+ runner.write_json(baseline/'grading-receipt.json',{'resolved_instances':1})+ skill = root/'skill.md'; skill.write_text('installed skill')+ package = root/'package.tgz'; package.write_bytes(b'package')+ prefix = root/'installed-prefix.tar'; prefix.write_bytes(b'installed prefix')+ paths = [Path(runner.__file__),HERE/'gateway_broker.py',inputs,skill,package,prefix]+ plan = {'status':'frozen','baseline':str(baseline),'baseline_receipt_sha256':runner.digest(baseline/'receipt.json'),'baseline_cost_usd':0.2685004,'cell':{**cell,'id':'installed-new','arm':'chunks','candidate':str(prefix)},'inputs':str(inputs),'skill':str(skill),'package':str(package),'output':str(root/'attempt'),'artifacts':{str(p):runner.digest(p) for p in paths}}+ plan_path=root/'plan.json'; runner.write_json(plan_path,plan)+ return plan_path,plan++ def test_dry_run_validates_pair_without_credentials_or_agent(self):+ with tempfile.TemporaryDirectory() as temporary:+ path,plan=self.fixture(Path(temporary))+ with patch.object(runner,'check_image',return_value=runner.RUNTIME_IMAGE),patch.object(runner,'command',side_effect=AssertionError('No agent or paid command allowed')),patch.dict(os.environ,{},clear=True),contextlib.redirect_stdout(io.StringIO()) as stdout:+ runner.run(argparse.Namespace(plan=path,dry_run=True))+ self.assertEqual(json.loads(stdout.getvalue())['paid_calls'],0)+ self.assertFalse(Path(plan['output']).exists())+ Path(plan['cell']['candidate']).write_bytes(b'changed')+ with self.assertRaisesRegex(ValueError,'Frozen artifact changed'):runner.load_plan(path)++ def test_changed_model_is_not_paired_to_the_old_baseline(self):+ with tempfile.TemporaryDirectory() as temporary:+ path,plan=self.fixture(Path(temporary))+ plan['cell']['model']='another-model';runner.write_json(path,plan)+ with self.assertRaisesRegex(ValueError,'pairing changed'):runner.load_plan(path)++ def test_grading_rejects_a_changed_official_harness_before_execution(self):+ with tempfile.TemporaryDirectory() as temporary:+ root=Path(temporary);path,plan=self.fixture(root);out=Path(plan['output']);out.mkdir()+ runner.write_json(out/'receipt.json',{'plan_sha256':runner.digest(path),'cell':plan['cell']})+ with patch.object(runner,'text',return_value='changed-harness'),patch.object(runner.subprocess,'run',side_effect=AssertionError('No grader launched')):+ with self.assertRaisesRegex(ValueError,'harness checkout changed'):+ runner.grade(argparse.Namespace(plan=path,tooling=root/'tooling',dataset=root/'dataset.json'))++ def test_incomplete_billing_never_counts_missing_charges_as_free(self):+ with tempfile.TemporaryDirectory() as temporary:+ path,plan=self.fixture(Path(temporary));out=Path(plan['output']);out.mkdir()+ events=[{'kind':'codex-request-start','operation':'generation','requestId':'one'},{'kind':'codex-generation','requestId':'one','generationId':'gen_one'},{'kind':'codex-gateway','requestId':'one','status':200,'streamTerminal':'response.completed'}, {'kind':'codex-request-start','operation':'generation','requestId':'missing'}]+ (out/'proxy.jsonl').write_text('\n'.join(json.dumps(e) for e in events))+ runner.write_json(out/'generation-lookups.json',[{'id':'gen_one','metadata':{'total_cost':0.1}}])+ runner.write_json(out/'grading-receipt.json',{'resolved_instances':1});runner.write_json(out/'receipt.json',{'required_retrieval_observed':True})+ with patch.object(runner.urllib.request,'urlopen',side_effect=AssertionError('No network')),contextlib.redirect_stdout(io.StringIO()):runner.account(argparse.Namespace(plan=path))+ result=json.loads((out/'generation-accounting.json').read_text())+ self.assertEqual(result['known_gateway_cost_usd'],0.1)+ self.assertIsNone(result['gateway_cost_usd']);self.assertFalse(result['successful_cost_win'])++ def test_broker_captures_exact_jev_bodies_without_auth_headers(self):+ with tempfile.TemporaryDirectory() as temporary:+ root=Path(temporary);config=root/'gateway.json';config.write_text(json.dumps({'key':'REAL_KEY_SENTINEL','token':'BROKER_TOKEN_SENTINEL','allow_jev':True,'agent_engine':'codex'}))+ request_body=b'{"state":{"source":"public fixture"},"questions":{}}';response_body=b'{"answers":{}}'+ class Response(io.BytesIO):+ status=200;headers={'Content-Type':'application/json'}+ def read1(self,size):return self.read(size)+ with patch.object(broker,'CONFIG_PATH',str(config)),patch.object(broker,'TRACE_DIR',str(root/'traces')),patch.object(broker.urllib.request,'urlopen',return_value=Response(response_body)),contextlib.redirect_stdout(io.StringIO()):+ server=http.server.ThreadingHTTPServer(('127.0.0.1',0),broker.Gateway)+ thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()+ try:+ client=http.client.HTTPConnection('127.0.0.1',server.server_port)+ client.request('POST','/evaluation-model',request_body,{'Authorization':'Bearer BROKER_TOKEN_SENTINEL','ai-model-id':'typesafe-ai/jev'})+ response=client.getresponse();self.assertEqual(response.status,200);self.assertEqual(response.read(),response_body);client.close()+ finally:server.shutdown();server.server_close();thread.join()+ requests=list((root/'traces').glob('*.request.json'));responses=list((root/'traces').glob('*.response.json'))+ self.assertEqual(requests[0].read_bytes(),request_body);self.assertEqual(responses[0].read_bytes(),response_body)+ for file in requests+responses:+ self.assertEqual(file.stat().st_mode&0o777,0o600)+ self.assertNotIn(b'REAL_KEY_SENTINEL',file.read_bytes());self.assertNotIn(b'BROKER_TOKEN_SENTINEL',file.read_bytes())++if __name__=='__main__':unittest.main()