Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Evaluate one frozen installed package across fixed SWE-bench cohort
David Zhang679bb9a92425
diff --git a/evals/implementation/swebench/fixed-baselines.json b/evals/implementation/swebench/fixed-baselines.jsonnew file mode 100644index 0000000..bc2d5af--- /dev/null+++ b/evals/implementation/swebench/fixed-baselines.json@@ -0,0 +1,300 @@+{+ "inputs": "evals/runs/swebench/auto-research-80/ten-agent-inputs.json",+ "dataset": "evals/runs/swebench/auto-research-80/evaluator-only/ten-dataset.json",+ "tooling": "evals/runs/tooling/swebench",+ "baseline_solves": 8,+ "required_cost_wins": 7,+ "tasks": [+ {+ "task": "scikit-learn__scikit-learn-13124",+ "baseline": "evals/runs/swebench/ranked-leads-additional/scikit-learn__scikit-learn-13124-baseline",+ "cell": {+ "id": "ranked-additional-scikit-learn__scikit-learn-13124-baseline",+ "instance_id": "scikit-learn__scikit-learn-13124",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-13124:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-13124@sha256:bdeb5e17bf55616082bf65f2b83de5cfc92556f3548fb38f4095ed1f802a7ec0",+ "image": "sha256:ad9300592673fe015a1cea83a5697685b2b1c8068cf5f44126aad1531fadb851",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "aec0d00a29e5283d5a134489e27dcdfc92f350e3",+ "expected_source_tree": "72e1da048a03e5f2d1728e17a93b7b687ae78b80"+ },+ "cost_usd": 0.29443600000000003,+ "resolved": true,+ "requests": 13,+ "files": {+ "receipt.json": "10f9381d2ab45233f935f318a0a6753c2634308e57d136b33f5cffadf2150ab7",+ "prompt.txt": "e1662a59921f366d1e98b0c0f4d317e65dda11d10268a066a28fb6ce08f0d0e8",+ "grading-receipt.json": "a323f3720d7acb68b91d46e3b39b0e47322fc5358256e2702be3a063ec8c33cf",+ "generation-lookups.json": "1233a1be3db32a2e32a18555132741a499b3236d195672734faae9b561a98836"+ }+ },+ {+ "task": "django__django-15629",+ "baseline": "evals/runs/swebench/lookahead-native-v88/django__django-15629/codex-baseline",+ "cell": {+ "id": "lookahead-native-v88-django__django-15629-codex-baseline",+ "instance_id": "django__django-15629",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.django_1776_django-15629:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.django_1776_django-15629@sha256:70d0f93a3891c6d334c7cc3028b2bf1de51b64a216c3d99712608cc9a0e501ae",+ "image": "sha256:514e444b11b59fe02f003eca0253c59c3a28a8b6377c65e64d021176d180dffe",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "9b70bb1938e97614e57443bbade13b71730f813e",+ "expected_source_tree": "ae7d451039d57611fc1269a6167d55fe1f4d49be"+ },+ "cost_usd": 1.5055472,+ "resolved": true,+ "requests": 40,+ "files": {+ "receipt.json": "7eb3d68ad77860a0d8978ffa754836e40f6830a19aa3cd7a92220a3da6b28af9",+ "prompt.txt": "4b22482d501f6fbd977344ba5309a928f57ac7809f59713cb191c99ab7c04640",+ "grading-receipt.json": "c5a4b7649c658b1f5f149c8e452b72a33b34b83457b968fc4175a93e684c6d51",+ "generation-lookups.json": "9ab4210b09194ac149d083668fe70743e7a17c42e855457b01f5ccb7841a8b1f"+ }+ },+ {+ "task": "astropy__astropy-13579",+ "baseline": "evals/runs/swebench/ranked-leads-additional/astropy__astropy-13579-baseline",+ "cell": {+ "id": "ranked-additional-astropy__astropy-13579-baseline",+ "instance_id": "astropy__astropy-13579",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.astropy_1776_astropy-13579:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.astropy_1776_astropy-13579@sha256:3a4b8059992d3eed81046bd4b3bd3450993a237a909c2298ce69b59bb5b9c863",+ "image": "sha256:2dd8342d0335721880149fa975922d788a13523f44273c6165e3057bc4da0322",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "62194260181d107f2f6630d4588887fbf46f5a0c",+ "expected_source_tree": "55a69b663232a3ee49ae06632d535dd3b1ced6c8"+ },+ "cost_usd": 0.428631,+ "resolved": true,+ "requests": 17,+ "files": {+ "receipt.json": "2dc42050943fc27a131a5f59c8c9375a75a76af29688a56be08a3ee0c95a7bed",+ "prompt.txt": "33c6e22d8f2072d941c0a0f3d513494646cdaaa58bf354404121ab565c062bf9",+ "grading-receipt.json": "1052eb2459ce64de6b4b451ecbb169d231246a69ffbb60dd8222e3497ccbb5dc",+ "generation-lookups.json": "b95664cf3aebb15f8a08788854707719b28a273dc75d6b5e5f2ba01eb5ffe9a9"+ }+ },+ {+ "task": "pydata__xarray-3305",+ "baseline": "evals/runs/swebench/auto-research-80/reserved-check/pydata__xarray-3305-baseline",+ "cell": {+ "id": "auto80-reserved-pydata__xarray-3305-baseline",+ "instance_id": "pydata__xarray-3305",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.pydata_1776_xarray-3305:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.pydata_1776_xarray-3305@sha256:0e234daddf9a42b85a77ba0436f3fcbad90759921c978b127dff90d55cba0f89",+ "image": "sha256:e0188f9a67225bf0fbb0bad90f5751a751e42002b690396695f25fa8f30741e3",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "cad44a667322dadb32d09f1db9ed62d9875ac43e",+ "expected_source_tree": "f0cc03a70ee44542b9f21046fbb5fe3305e36c77"+ },+ "cost_usd": 0.4937066,+ "resolved": true,+ "requests": 23,+ "files": {+ "receipt.json": "55ed7868a65323413496849d2fca6547fdb8ff5895da9036645a84738319dd12",+ "prompt.txt": "5e3fe2093c9948dca235a26fce9825a7f8e9c89fc9da953fe270c61682bdf3f6",+ "grading-receipt.json": "c75fdec46bcd1bbe2c5bdca098119d1137ae8201b009a8385b422ce0b8ec8f20",+ "generation-lookups.json": "fd0ee68b0ddf51cc8488db797ec48eda7ef02b701af2d9477d7bbc1335a5862d"+ }+ },+ {+ "task": "psf__requests-1142",+ "baseline": "evals/runs/swebench/lookahead-native-v88/psf__requests-1142/codex-baseline",+ "cell": {+ "id": "lookahead-native-v88-psf__requests-1142-codex-baseline",+ "instance_id": "psf__requests-1142",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.psf_1776_requests-1142:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.psf_1776_requests-1142@sha256:fa9c859412109d003a68c2bbe3c65541ca4714526afeb6906fe589c18af0b30d",+ "image": "sha256:2264606b0765897361bb392db8aa3226e171719ba97e82ce5205ba122a43eaff",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "546130c8c878adabd16ac5221a36e39e4dab3645",+ "expected_source_tree": "3b8ba30961592000c9a9a73b41d8452c871f47b4"+ },+ "cost_usd": 0.2685004,+ "resolved": true,+ "requests": 11,+ "files": {+ "receipt.json": "ee0d055679453e5c236a6a4d6a34e53b3deb9ca0436e2d37357df4d0c40ca88d",+ "prompt.txt": "12ba196760c861377a033958d9d1a3cd3cc25dd4a1fa6cf877425001a14bae5c",+ "grading-receipt.json": "5ec41b60522d4f4bd16d55f96991c4f4a53af4df7f9a13096e53597357c8d7f1",+ "generation-lookups.json": "d7e657c0ff71b2279cb1dabfea058de6a914c3f68b2eab473fdbfc6f6af0e757"+ }+ },+ {+ "task": "sympy__sympy-16792",+ "baseline": "evals/runs/swebench/lookahead-native-v88/sympy__sympy-16792/codex-baseline",+ "cell": {+ "id": "lookahead-native-v88-sympy__sympy-16792-codex-baseline",+ "instance_id": "sympy__sympy-16792",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.sympy_1776_sympy-16792:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.sympy_1776_sympy-16792@sha256:3064872f9fef7139d864d821938635b2798232942db1dbe1b95bc085b45bfc61",+ "image": "sha256:664a16898db69c3ad4d85d6db4dc0cf50cdbacc34a58ec5b6465ec9cc20eb771",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "4d0daaed7f939d3a8fbacfcb571561b7c3745e05",+ "expected_source_tree": "5b2ea26c6ef59f15dc46c0e9b5c90d8935101ce7"+ },+ "cost_usd": 0.547975,+ "resolved": true,+ "requests": 28,+ "files": {+ "receipt.json": "0daf2c17c6fc7a13258722b5fe5d7657d0e8a862ba2d289a6392367da5ca26d0",+ "prompt.txt": "bda6bd748dfb21a74c6ea7298a66937da2e530621918ac0bb2426f406b2d3d5c",+ "grading-receipt.json": "62e0af48d4353b5d4ecf600fe040350c4f06fe41a1ae941f4a6af72642173d3b",+ "generation-lookups.json": "431a34b987ed2f117c077dfb5fae22633fcccaabbc234df2fe49d49935052e95"+ }+ },+ {+ "task": "pytest-dev__pytest-6197",+ "baseline": "evals/runs/swebench/ranked-leads-additional/pytest-dev__pytest-6197-baseline",+ "cell": {+ "id": "ranked-additional-pytest-dev__pytest-6197-baseline",+ "instance_id": "pytest-dev__pytest-6197",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-6197:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-6197@sha256:15d39592171743355fdade433454d2dbd5cd7d7db21bdcf0e2f125edecabaf09",+ "image": "sha256:77ae3ccb908cc80d961c74e8351029382ab74fa171b2f5387eceb10da2c2b23f",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "4eecc51564648be4823fcc5f6f7ef208a4f84a8c",+ "expected_source_tree": "4024da6943c752a72e2ed45b8b57c03ccd047371"+ },+ "cost_usd": 2.344458,+ "resolved": true,+ "requests": 62,+ "files": {+ "receipt.json": "1997160d2bb3c3bb2da69b948ca36a836022309c4e6b81d8275d7325cfc59187",+ "prompt.txt": "04386d2e8c1845f42ab6d26f77b6daf1ea7a320dc6df5b1c4c005fd5044a9bc6",+ "grading-receipt.json": "e4d4ee179321ec72d3ede2a69e94eb2396734d1c3b9bf9e002c2938db9c47f87",+ "generation-lookups.json": "66de257f7f8e0ca6aaadc9a8d268faf12763a298843ae5393e99861238f002da"+ }+ },+ {+ "task": "sphinx-doc__sphinx-8638",+ "baseline": "evals/runs/swebench/auto-research-80/reserved-check/sphinx-doc__sphinx-8638-baseline",+ "cell": {+ "id": "auto80-reserved-sphinx-doc__sphinx-8638-baseline",+ "instance_id": "sphinx-doc__sphinx-8638",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-8638:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-8638@sha256:3f2d3335613ab578fc026ddf8cda2a835e8f0f24ba6c0605e53241ec518bf78a",+ "image": "sha256:98e6227437c5cdc843e663065b3118603afac2239de969097735a564e82705e9",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "27a63c4bbe23e14896382548356d2f9c880812d5",+ "expected_source_tree": "742277e5b9486c9781a497aad5da88b8b0d5bb96"+ },+ "cost_usd": 1.0594724,+ "resolved": true,+ "requests": 25,+ "files": {+ "receipt.json": "ced56c0bcc21d699cf7d1dd9616a47453293a7687f8d5ae43d0a5f273f368dcf",+ "prompt.txt": "78e276ae82c6252b2e1b0a1ac8b9833271bc0be79f9a0310d3d9b4b82da12be8",+ "grading-receipt.json": "2a86f9fe4dcf991a7e24af4c566d31b8be1148828379883e67193e326605a218",+ "generation-lookups.json": "30ebe7af2afb540a067e364b68056e3cca16011384ddbaee3d258c31a425f9f0"+ }+ },+ {+ "task": "matplotlib__matplotlib-26466",+ "baseline": "evals/runs/swebench/auto-research-80/reserved-check/matplotlib__matplotlib-26466-baseline",+ "cell": {+ "id": "auto80-reserved-matplotlib__matplotlib-26466-baseline",+ "instance_id": "matplotlib__matplotlib-26466",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.matplotlib_1776_matplotlib-26466:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.matplotlib_1776_matplotlib-26466@sha256:35be1078dc800d0bfcb32fef52650b1915480bca8778f2ed0a5528d4b6929784",+ "image": "sha256:969a5ccf3f34af256d0ee9bbf87422c9483eef96c32687281f472dd66729a239",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "20e9c8c37ec0e0bb36f0bc6b1761200ffaa846fd",+ "expected_source_tree": "92c6a499aede7b947b743b5c89ca08e9ac523c0d"+ },+ "cost_usd": 0.395716,+ "resolved": false,+ "requests": 13,+ "files": {+ "receipt.json": "dd8e284ee2a077bc42166b5b0765bdefde4336293dd44d4569a72a626d6fbc94",+ "prompt.txt": "e15b7bc7bdaa1c59ac44f6904b4b0892e48bfea57e37589dae302b3cf8737d2f",+ "grading-receipt.json": "8ae2c27b62aa3c7144fe3b932b023b182dde3389c72658dac0288651bf73a3d2",+ "generation-lookups.json": "ddbe18196d46f27bece8efbff069ba4c19ea80f50941b2e0390436582b7b2b9f"+ }+ },+ {+ "task": "pylint-dev__pylint-4604",+ "baseline": "evals/runs/swebench/auto-research-80/reserved-check/pylint-dev__pylint-4604-baseline",+ "cell": {+ "id": "auto80-reserved-pylint-dev__pylint-4604-baseline",+ "instance_id": "pylint-dev__pylint-4604",+ "engine": "codex",+ "arm": "baseline",+ "invocation": "required",+ "source_image": "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-4604:latest",+ "source_pinned_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-4604@sha256:e24d647f8421c551d60da005ce2272910c0aae8bb37f9de870b10c5e70b840ce",+ "image": "sha256:802e9a848f7268b436980bc4305ea815d89a30219e3dd4727f26a1f38ae7563b",+ "model": "openai/gpt-5.6-sol",+ "effort": "medium",+ "timeout_seconds": 900,+ "gateway_model": "openai/gpt-5.6-sol",+ "expected_image_head": "0f266873c24f8f7270d257c8c8a8676930be0c72",+ "expected_source_tree": "05a6b93fcb973fb72fe368b7d43eee00ed6635da"+ },+ "cost_usd": 0.2836264,+ "resolved": false,+ "requests": 13,+ "files": {+ "receipt.json": "54b00479be48fc888e7f2ce51082ebf41d0de1a3e8e9677dabcc6dd949cac611",+ "prompt.txt": "7b88082c7bdcd31b3ecde24fd7a56e1d8261d2ffb4129828931e87178a2e6928",+ "grading-receipt.json": "e380e835bebd324b81942208c526405d45e8b8c7b2e76167e4947da63b1cedb2",+ "generation-lookups.json": "73bd2ed7d04398ade9954f7d2768357f5e91ee7127239c0ed24df1f33981acea"+ },+ "caveat": "Retained official nonpass: test collection fails on missing IS_PYPY export; not a demonstrated behavioral failure."+ }+ ]+}diff --git a/evals/implementation/swebench/installed.md b/evals/implementation/swebench/installed.mdindex 0f36a47..a849eff 100644--- a/evals/implementation/swebench/installed.md+++ b/evals/implementation/swebench/installed.md@@ -1,46 +1,64 @@-# Installed Requests checkpoint+# Installed package cohort-[The runner](installed.py) compares one frozen installed `jg` package with the-retained Requests Sol baseline. It refuses changed model, source, runtime and-baseline-prompt identities. It never launches a baseline. A successful checkpoint-is integration evidence, not the ten-task quality gate.+[The runner](installed.py) evaluates one frozen installed `jg` package against+[the ten fixed Sol baselines](fixed-baselines.json). Baseline receipts, prompts,+grades and billing are checked by hash; there is no baseline execution command.+The two recorded baseline nonpasses remain nonpasses, including the retained+Pylint test-collection limitation.-Preparation installs the packed npm artifact and its dependencies in a disposable-copy of the retained Linux image. Registry access is confined to preparation;-the agent receives the archived installed prefix on an internal network. It runs-the real `jg` executable. The packaged skill is verified against the canonical-skill, and the benchmark explicitly requires initial retrieval even when the-production skill would consider it optional.+Preparation installs the packed npm artifact and dependencies once, then checks+that same installation offline in every selected runtime image. Each check+verifies the original source, agent versions and canonical packaged skill.+The coding agent receives the archived prefix and runs the real `jg` executable+on an internal network. The unchanged benchmark prompt requires initial retrieval,+even when the production skill would consider it optional. Retrieval limits come+from the frozen package's default policy.-Use `python3 evals/implementation/swebench/installed.py --help` for commands.-The intended root script alias is `eval:swebench` pointing to that entry point.-A typical sequence is:+The plan binds one package, installed prefix, skill, safe task export, evaluator+dataset and archived runner/broker/registry sources before any treatment runs.+Existing studies remain unchanged; their archived source records the procedure used.+A fresh full-cohort plan prevents combining the cheapest outcomes from different+packages or experiments. A single-task plan is useful for diagnostics but cannot+satisfy the cohort gate.```shpython3 evals/implementation/swebench/installed.py prepare \--package /absolute/path/to/package.tgz --output /absolute/path/to/new-studypython3 evals/implementation/swebench/installed.py run \- --plan /absolute/path/to/new-study/plan.json --dry-run+ --plan /absolute/path/to/new-study/plan.json --all --dry-run```-Preparation and dry-run make no model calls. After reviewing the frozen plan,-`run` without `--dry-run` executes the paid treatment exactly once. Load the-Gateway credential through the authorized environment workflow; never place a-real key in the plan or command arguments. Each attempt uses fresh home/cache-state and the package's default retrieval policy.--Follow execution with `grade --plan ...` and `account --plan ...`. Grading uses-the existing pinned official environment and a unique run ID. Accounting reuses-saved generation metadata, fetches missing receipts, and leaves the full cost-unknown if any generation is unaccounted for. A missing required retrieval call-cannot count as a successful cost win.--The plan, installation archive, raw agent output, patch, official reports and-billing remain under ignored run storage. [The broker](gateway_broker.py) retains-raw Jev request/response bodies separately from transport metadata, only in the-non-agent container; authentication headers are never captured. These receipts-contain benchmark source and should stay local with the other study evidence.--Run [the focused validations](test_installed.py) inside an isolated Docker-container. They exercise frozen prompt identity, drift rejection, no-call dry-run,-partial billing and exact Jev body capture through a local synthetic provider.+Preparation and dry-run make no model calls. `prepare` defaults to all ten tasks;+`--task` selects one registered task. `--evidence-root` locates the retained+baseline and evaluator files when preparing from another checkout. The root+`eval:swebench` alias points to this entry point; `--help` lists its arguments.++After reviewing the frozen plan, `run --all` without `--dry-run` executes the paid+treatments sequentially. Load the Gateway credential through the authorized+environment workflow. Real keys stay out of plans and command arguments. Every+attempt has fresh home/cache state and a durable lifecycle receipt. Failed or+completed attempts are retained; they are never replaced. Active processes stop+continuation. An abandoned attempt whose runner and containers are gone becomes+an interrupted result, preserving its evidence.++Follow execution with `grade --plan ... --all` and `account --plan ... --all`.+Use `--task` instead of `--all` for an individual cell. Grading uses the pinned+official harness and unique run IDs. Accounting reuses saved generation metadata,+fetches missing receipts, and leaves full cost unknown if any request is missing.+Neither operation reruns the coding agent.++`aggregate --plan ...` requires ten terminal attempts with official grading receipts,+all eight baseline solves preserved by valid solved treatments, and at least seven+valid solved results with complete billing and strictly lower cost. Failed or+invalid treatments remain recorded as nonpasses and nonwins. Unknown costs cannot win and+prevent reporting a full cohort cost total; the known subtotal remains explicit. The aggregate reports individual+outcomes so failures remain visible.++Run storage holds the frozen artifacts, agent output, patches, official reports+and billing. [The broker](gateway_broker.py) retains exact Jev request/response+bodies in the non-agent container without authentication headers. These contain+benchmark source and remain local with the ignored study evidence.++[Focused tests](test_installed.py) run in an isolated Docker container. They cover+prompt identity, drift rejection, no-call validation, retained attempts, incomplete+billing, cohort acceptance and exact body capture through a synthetic provider.diff --git a/evals/implementation/swebench/installed.py b/evals/implementation/swebench/installed.pyindex f5c57e7..695d2ae 100644--- a/evals/implementation/swebench/installed.py+++ b/evals/implementation/swebench/installed.py@@ -1,5 +1,5 @@#!/usr/bin/env python3-"""Prepare, execute and grade one frozen installed-package SWE-bench treatment."""+"""Freeze and evaluate one installed package against the ten retained SWE-bench baselines."""import argparseimport hashlibimport json@@ -16,13 +16,8 @@ import uuidHERE = Path(__file__).resolve().parentROOT = HERE.parents[2]-RUNS = ROOT / 'evals/runs/swebench'-TOOLING = ROOT / 'evals/runs/tooling/swebench'-TASK = 'psf__requests-1142'-BASELINE_ID = 'lookahead-native-v88-psf__requests-1142-codex-baseline'-RUNTIME_IMAGE = 'sha256:2264606b0765897361bb392db8aa3226e171719ba97e82ce5205ba122a43eaff'+REGISTRY = json.loads((HERE / 'fixed-baselines.json').read_text())HARNESS_COMMIT = '02e7a74ffd0b707aab73d203fe87bdc7c76afc8e'-BASELINE = RUNS / 'lookahead-native-v88' / TASK / 'codex-baseline'SAFE_FIELDS = {'instance_id', 'repo', 'base_commit', 'image', 'problem_statement'}VERSIONS = {'node': 'v24.14.0', 'codex': 'codex-cli 0.153.4', 'claude': '2.1.278 (Claude Code)'}VERIFICATION = (@@ -75,8 +70,12 @@ def text(args):def load_pair(baseline, inputs, task):receipt = json.loads((baseline / 'receipt.json').read_text())cell = receipt['cell']- if task != TASK or cell['instance_id'] != task or cell['arm'] != 'baseline' or cell['id'] != BASELINE_ID or receipt['image'] != RUNTIME_IMAGE or cell['image'] != RUNTIME_IMAGE:- raise ValueError('This checkpoint reuses only the registered Requests baseline')+ registered = next(item for item in REGISTRY['tasks'] if item['task'] == task)+ if cell != registered['cell'] or receipt['image'] != cell['image']:+ raise ValueError('Baseline differs from the fixed registry')+ for name, expected in registered['files'].items():+ if digest(baseline / name) != expected:+ raise ValueError('Retained baseline evidence changed: ' + name)if (cell['model'], cell['effort'], cell['timeout_seconds']) != ('openai/gpt-5.6-sol', 'medium', 900):raise ValueError('Baseline model/effort/deadline differs from the registered harness')if receipt['versions'] != VERSIONS or receipt['provider_route'] != 'vercel-ai-gateway':@@ -92,19 +91,25 @@ def load_pair(baseline, inputs, task):if hashlib.sha256(prompt).hexdigest() != receipt['prompt_sha256'] or (baseline / 'prompt.txt').read_bytes() != prompt:raise ValueError('Baseline prompt differs; do not rerun or overwrite it')grade = json.loads((baseline / 'grading-receipt.json').read_text())- if grade.get('resolved_instances') != 1:- raise ValueError('Expected the retained officially resolved Requests baseline')+ if (grade.get('resolved_instances') == 1) != registered['resolved']:+ raise ValueError('Retained official outcome changed')return receipt, rowdef check_image(image):if not re.fullmatch(r'sha256:[a-f0-9]{64}', image):raise ValueError('Use the immutable retained runtime image ID')- return text(['docker', 'image', 'inspect', image, '--format', '{{.Id}}'])+ actual = text(['docker', 'image', 'inspect', image, '--format', '{{.Id}}'])+ if actual != image:+ raise ValueError('Runtime image identity changed')+ return actualdef prepare(args):- baseline, row = load_pair(args.baseline.resolve(), args.inputs.resolve(), args.task)+ root = args.evidence_root.resolve()+ selected = [item for item in REGISTRY['tasks'] if args.task in ('all', item['task'])]+ pairs = [(item, *load_pair(root / item['baseline'], root / REGISTRY['inputs'], item['task'])) for item in selected]+ baseline = pairs[0][1]image = check_image(baseline['image'])out = args.output.resolve()out.mkdir(parents=True, exist_ok=False)@@ -136,49 +141,123 @@ tar -C /opt -cf /tmp/installed-prefix.tar jg-installraise ValueError('Installation changed source identity or installed the wrong package')if (out / 'skill.md').read_bytes() != args.skill.read_bytes():raise ValueError('Packaged skill differs from the canonical skill')- write_json(out / 'agent-inputs.json', [row])- cell = {**cell, 'id': 'installed-' + TASK + '-' + uuid.uuid4().hex[:12], 'arm': 'chunks', 'candidate': str(out / 'installed-prefix.tar')}- artifacts = [Path(__file__).resolve(), HERE / 'gateway_broker.py', package, out / 'installed-prefix.tar', out / 'skill.md', out / 'agent-inputs.json']- plan = {'status': 'frozen', 'purpose': 'Installed production Requests checkpoint; reuse the fixed baseline once, no baseline rerun.',- 'cell': cell, 'baseline': str(args.baseline.resolve()), 'baseline_receipt_sha256': digest(args.baseline / 'receipt.json'),- 'baseline_cost_usd': 0.2685004, 'baseline_resolved': True, 'inputs': str(out / 'agent-inputs.json'),- 'skill': str(out / 'skill.md'), 'package': str(package), 'output': str(out / 'attempt'),+ # Every image gets an offline check using the same installed bytes.+ cells = []+ for registered, fixed, _ in pairs:+ runtime = check_image(fixed['image'])+ check = 'jg-preflight-' + uuid.uuid4().hex[:12]+ try:+ command(['docker', 'create', '--platform', 'linux/amd64', '--network', 'none', '--name', check, runtime, 'sh', '-ec',+ 'tar -xf /tmp/prefix.tar -C /opt; node --version; codex --version; claude --version; /opt/jg-install/bin/jg --version; /opt/jg-install/bin/jg skill; git -C /testbed rev-parse HEAD; git -C /testbed rev-parse HEAD^{tree}; git -C /testbed status --porcelain'])+ command(['docker', 'cp', str(out / 'installed-prefix.tar'), check + ':/tmp/prefix.tar'])+ output = command(['docker', 'start', '-a', check]).stdout+ expected = ((''.join(value + '\n' for value in VERSIONS.values())).encode() + (out / 'version.txt').read_bytes() + (out / 'skill.md').read_bytes() ++ (fixed['cell']['expected_image_head'] + '\n' + fixed['cell']['expected_source_tree'] + '\n').encode())+ if output != expected:+ raise ValueError('Offline runtime/source/skill preflight differs: ' + registered['task'])+ finally:+ subprocess.run(['docker', 'rm', '-f', check], capture_output=True)+ cell = {**fixed['cell'], 'id': 'installed-' + registered['task'] + '-' + uuid.uuid4().hex[:12],+ 'arm': 'chunks', 'candidate': str(out / 'installed-prefix.tar')}+ cells.append({'cell': cell, 'baseline': str(root / registered['baseline']),+ 'baseline_cost_usd': registered['cost_usd'], 'baseline_resolved': registered['resolved'],+ 'output': str(out / 'attempts' / registered['task'])})+ write_json(out / 'agent-inputs.json', [public for _, _, public in pairs])+ frozen = out / 'runner'+ frozen.mkdir(mode=0o700)+ for source in [Path(__file__).resolve(), HERE / 'gateway_broker.py', HERE / 'fixed-baselines.json']:+ shutil.copyfile(source, frozen / source.name)+ artifacts = [*frozen.iterdir(), package, out / 'installed-prefix.tar', out / 'skill.md', out / 'agent-inputs.json', root / REGISTRY['dataset']]+ plan = {'schema': 2, 'status': 'frozen', 'cells': cells,+ 'purpose': 'One prospective package cohort; reuse fixed baselines without execution.',+ 'inputs': str(out / 'agent-inputs.json'), 'skill': str(out / 'skill.md'), 'package': str(package),+ 'runner': str(frozen / 'installed.py'), 'broker': str(frozen / 'gateway_broker.py'),+ 'registry': str(frozen / 'fixed-baselines.json'), 'dataset': str(root / REGISTRY['dataset']),+ 'tooling': str(root / REGISTRY['tooling']),'artifacts': {str(path): digest(path) for path in artifacts}}write_json(out / 'plan.json', plan)- print(json.dumps({'status': 'prepared', 'plan': str(out / 'plan.json'), 'cell': cell['id'], 'image': image}))+ print(json.dumps({'status': 'prepared', 'plan': str(out / 'plan.json'), 'tasks': len(cells)}))finally:subprocess.run(['docker', 'rm', '-f', name], capture_output=True)-def load_plan(path):+def load_cohort(path):plan = json.loads(path.read_text())- if plan.get('status') != 'frozen':- raise ValueError('Plan must be frozen before execution')+ if plan.get('schema') != 2 or plan.get('status') != 'frozen':+ raise ValueError('Expected a frozen prospective cohort plan')for artifact, expected in plan['artifacts'].items():if digest(artifact) != expected:raise ValueError('Frozen artifact changed: ' + artifact)- required = [Path(__file__).resolve(), HERE / 'gateway_broker.py', Path(plan['inputs']), Path(plan['skill']), Path(plan['package']), Path(plan['cell']['candidate'])]- if any(str(path) not in plan['artifacts'] for path in required):- raise ValueError('Plan omits a required artifact')- base = Path(plan['baseline'])- if digest(base / 'receipt.json') != plan['baseline_receipt_sha256']:- raise ValueError('The retained baseline changed')- baseline, row = load_pair(base, Path(plan['inputs']), plan['cell']['instance_id'])- for key in ['instance_id', 'engine', 'invocation', 'source_image', 'source_pinned_ref', 'image', 'model', 'effort', 'timeout_seconds', 'gateway_model', 'expected_image_head', 'expected_source_tree']:- if plan['cell'][key] != baseline['cell'][key]:- raise ValueError('Treatment pairing changed: ' + key)- if plan['cell']['arm'] != 'chunks' or plan['cell']['id'] == baseline['cell']['id'] or not re.fullmatch(r'[A-Za-z0-9_.-]+', plan['cell']['id']):- raise ValueError('Treatment must use its own registered cell')+ for key in ['runner', 'broker', 'registry', 'inputs', 'skill', 'package', 'dataset']:+ if plan[key] not in plan['artifacts']:+ raise ValueError('Plan omits required artifact: ' + key)+ if digest(__file__) != digest(plan['runner']) or digest(HERE / 'gateway_broker.py') != digest(plan['broker']) or json.loads(Path(plan['registry']).read_text()) != REGISTRY:+ raise ValueError('Use the frozen runner and registry')+ prefixes = {item['cell']['candidate'] for item in plan['cells']}+ identities = {item['cell']['id'] for item in plan['cells']}+ outputs = {item['output'] for item in plan['cells']}+ if len(prefixes) != 1 or len(identities) != len(plan['cells']) or len(outputs) != len(plan['cells']):+ raise ValueError('Cohort requires one prefix and distinct attempt identities')+ tasks = [item['cell']['instance_id'] for item in plan['cells']]+ if not tasks or len(set(tasks)) != len(tasks):+ raise ValueError('Plan tasks must be unique')+ for item in plan['cells']:+ cell = item['cell']+ registered = next(row for row in REGISTRY['tasks'] if row['task'] == cell['instance_id'])+ baseline, _ = load_pair(Path(item['baseline']), Path(plan['inputs']), cell['instance_id'])+ expected = {**baseline['cell'], 'id': cell['id'], 'arm': 'chunks', 'candidate': cell['candidate']}+ if cell != expected or not re.fullmatch(r'installed-[A-Za-z0-9_.-]+', cell['id']):+ raise ValueError('Treatment pairing changed')+ if cell['candidate'] not in plan['artifacts'] or item['baseline_cost_usd'] != registered['cost_usd'] or item['baseline_resolved'] != registered['resolved']:+ raise ValueError('Treatment artifacts or baseline outcome changed')+ return plan+++def load_plan(path, task=None):+ cohort = load_cohort(path)+ matches = [item for item in cohort['cells'] if task is None or item['cell']['instance_id'] == task]+ if len(matches) != 1:+ raise ValueError('Select one frozen task or use --all')+ plan = {**cohort, **matches[0]}+ _, row = load_pair(Path(plan['baseline']), Path(plan['inputs']), plan['cell']['instance_id'])return plan, row+def existing_attempt(plan):+ out = Path(plan['output'])+ if not out.exists():+ return False+ path = out / 'receipt.json'+ if not path.exists():+ raise ValueError('Existing attempt lacks a lifecycle receipt; inspect it before continuing')+ receipt = json.loads(path.read_text())+ if receipt['cell'] != plan['cell']:+ raise ValueError('Existing attempt belongs to another cell')+ if receipt['status'] in ('completed', 'failed', 'interrupted'):+ return True+ try:+ os.kill(receipt['host_pid'], 0)+ alive = True+ except ProcessLookupError:+ alive = False+ active = text(['docker', 'ps', '-q', '--filter', 'label=jg.cell=' + plan['cell']['id']])+ if alive or active:+ raise ValueError('Existing attempt is active; wait for its actual process to finish')+ receipt.update(status='interrupted', interruption='Runner and labeled containers are no longer active')+ write_json(path, receipt)+ return True++def run(args):- plan, row = load_plan(args.plan.resolve())+ plan, row = load_plan(args.plan.resolve(), args.task)cell = plan['cell']image = check_image(cell['image'])if args.dry_run:print(json.dumps({'status': 'validated', 'paid_calls': 0, 'baseline_reused': plan['baseline'], 'image': image, 'cell': cell['id']}))return+ if existing_attempt(plan):+ print(json.dumps({'status': 'retained', 'cell': cell['id']}))+ returnif not os.environ.get('AI_GATEWAY_API_KEY'):raise ValueError('AI_GATEWAY_API_KEY is required; load it through the authorized credential workflow')out = Path(plan['output'])@@ -187,7 +266,8 @@ def run(args):tag = 'jg-native-' + uuid.uuid4().hex[:12]network, proxy = tag + '-net', tag + '-proxy'receipt = {'engine': 'codex', 'cell': cell, 'image': image, 'plan_sha256': digest(args.plan), 'started': time.time(), 'status': 'preparing',- 'provider_route': 'vercel-ai-gateway', 'gateway_model': 'openai/gpt-5.6-sol', 'baseline': plan['baseline']}+ 'provider_route': 'vercel-ai-gateway', 'gateway_model': 'openai/gpt-5.6-sol', 'baseline': plan['baseline'], 'host_pid': os.getpid()}+ write_json(out / 'receipt.json', receipt)def dx(argv, user=None):return command(['docker', 'exec', *(['-u', user] if user else []), tag, *argv])def put(container, path, data, owner='agent:agent', mode='600'):@@ -196,9 +276,9 @@ def run(args):return subprocess.run(['docker', 'cp', remote, str(destination)], capture_output=True).returncode == 0try:command(['docker', 'network', 'create', '--internal', network])- command(['docker', 'run', '-d', '--name', proxy, '-e', 'JEVGREP_TRACE_DIR=/run/jev-traces', image, 'python3', '-u', '-c', (HERE / 'gateway_broker.py').read_text()])+ command(['docker', 'run', '-d', '--name', proxy, '--label', 'jg.cell=' + cell['id'], '-e', 'JEVGREP_TRACE_DIR=/run/jev-traces', image, 'python3', '-u', '-c', (HERE / 'gateway_broker.py').read_text()])command(['docker', 'network', 'connect', '--alias', 'model-egress', network, proxy])- command(['docker', 'run', '-d', '--name', tag, '--network', network,+ command(['docker', 'run', '-d', '--name', tag, '--label', 'jg.cell=' + cell['id'], '--network', network,'-e', 'HTTP_PROXY=http://model-egress:3128', '-e', 'HTTPS_PROXY=http://model-egress:3128','-e', 'NO_PROXY=localhost,127.0.0.1,model-egress', image, 'sleep', '1800'])dx(['sh', '-c', 'useradd -m -u 1001 agent; mkdir -p /home/agent/.codex /home/agent/.claude /workspace; chown -R agent:agent /home/agent /workspace'])@@ -244,6 +324,7 @@ def run(args):'-e', 'AI_GATEWAY_API_KEY=' + token, '-e', 'AI_GATEWAY_BASE_URL=http://model-egress:3129', '-w', '/testbed', tag, *native]receipt['status'] = 'running'receipt['agent_started_at'] = time.time()+ write_json(out / 'receipt.json', receipt)start = time.monotonic()with (out / 'events.jsonl').open('wb') as stdout, (out / 'stderr.txt').open('wb') as stderr:try:@@ -283,6 +364,8 @@ def run(args):receipt['jev_traces_copied'] = capture(proxy + ':/run/jev-traces', out / 'jev-traces')for container in [tag, proxy]: subprocess.run(['docker', 'rm', '-f', container], capture_output=True)subprocess.run(['docker', 'network', 'rm', network], capture_output=True)+ if receipt['status'] in ('preparing', 'running'):+ receipt['status'] = 'interrupted'receipt['total_elapsed_seconds'] = time.time() - receipt['started']write_json(out / 'receipt.json', receipt)print(json.dumps({'status': receipt['status'], 'output': str(out), 'error_type': receipt.get('error_type')}))@@ -290,14 +373,19 @@ def run(args):def grade(args):- plan, _ = load_plan(args.plan.resolve())+ plan, _ = load_plan(args.plan.resolve(), args.task)+ args.tooling = Path(plan['tooling'])+ args.dataset = Path(plan['dataset'])out, cell = Path(plan['output']), plan['cell']receipt = json.loads((out / 'receipt.json').read_text())- if receipt['plan_sha256'] != digest(args.plan) or receipt['cell'] != cell:+ if receipt['plan_sha256'] != digest(args.plan) or receipt['cell'] != cell or receipt['status'] not in ('completed', 'failed', 'interrupted'):raise ValueError('Attempt does not belong to this frozen plan')run_id = 'jg-' + cell['id']report = args.tooling.resolve() / 'logs/evaluation' / run_id / 'results.json'- if report.exists() or (out / 'grading-receipt.json').exists():+ if (out / 'grading-receipt.json').exists():+ print(json.dumps({'status': 'retained', 'grading': str(out / 'grading-receipt.json')}))+ return+ if report.exists():raise ValueError('Refusing to reuse an official grading run ID')harness = args.tooling.resolve() / 'repo'if text(['git', '-C', str(harness), 'rev-parse', 'HEAD']) != HARNESS_COMMIT or text(['git', '-C', str(harness), 'status', '--porcelain']):@@ -306,7 +394,7 @@ def grade(args):if Path(module).parent.parent != harness:raise ValueError('Python environment does not use the pinned official harness')prediction = out / 'predictions.jsonl'- prediction.write_text(json.dumps({'instance_id': cell['instance_id'], 'model_name_or_path': cell['id'], 'model_patch': (out / 'agent.patch').read_text()}) + '\n')+ prediction.write_text(json.dumps({'instance_id': cell['instance_id'], 'model_name_or_path': cell['id'], 'model_patch': (out / 'agent.patch').read_text() if (out / 'agent.patch').exists() else ''}) + '\n')start = time.monotonic()with (out / 'grading-console.log').open('w') as log:result = subprocess.run([str(args.tooling.resolve() / 'venv/bin/python'), '-m', 'swebench.harness.run_evaluation', '--dataset_name', str(args.dataset.resolve()), '--predictions_path', str(prediction), '--instance_ids', cell['instance_id'], '--max_workers', '1', '--timeout', '1800', '--run_id', run_id], cwd=args.tooling, stdout=log, stderr=subprocess.STDOUT)@@ -323,9 +411,22 @@ def valid_cost(value):def account(args):- plan, _ = load_plan(args.plan.resolve())+ plan, _ = load_plan(args.plan.resolve(), args.task)out = Path(plan['output'])- events = [json.loads(line) for line in (out / 'proxy.jsonl').read_text().splitlines()]+ receipt = json.loads((out / 'receipt.json').read_text())+ if receipt['cell'] != plan['cell'] or receipt['plan_sha256'] != digest(args.plan) or receipt['status'] not in ('completed', 'failed', 'interrupted'):+ raise ValueError('Accounting requires this plan’s terminal attempt')+ events = []+ log = out / 'proxy.jsonl'+ log_valid = log.exists()+ for line in log.read_text().splitlines() if log.exists() else []:+ try:+ event = json.loads(line)+ if not isinstance(event, dict):+ raise ValueError('Invalid log event')+ events.append(event)+ except ValueError:+ log_valid = Falsestarts = [event for event in events if event.get('kind') == 'codex-request-start' and event.get('operation') == 'generation']request_ids = {event['requestId'] for event in starts}ends = [event for event in events if event.get('kind') == 'codex-gateway' and event.get('requestId') in request_ids]@@ -335,7 +436,7 @@ def account(args):cached = {row['id']: row for row in json.loads(path.read_text())} if path.exists() else {}lookups = []for identifier in identifiers:- if identifier in cached and 'metadata' in cached[identifier]:+ if identifier in cached and cached[identifier].get('metadata', {}).get('id') == identifier and cached[identifier].get('metadata', {}).get('model') == 'openai/gpt-5.6-sol':lookups.append(cached[identifier]); continueif not os.environ.get('AI_GATEWAY_API_KEY'): raise ValueError('Gateway credential required for uncached generation accounting')request = urllib.request.Request('https://ai-gateway.vercel.sh/v1/generation?' + urllib.parse.urlencode({'id': identifier}), headers={'Authorization': 'Bearer ' + os.environ['AI_GATEWAY_API_KEY']})@@ -345,39 +446,101 @@ def account(args):lookups.append({'id': identifier, 'metadata': metadata})except (OSError, ValueError, KeyError) as error: lookups.append({'id': identifier, 'error_type': type(error).__name__})write_json(path, lookups)- complete = bool(identifiers) and len(starts) == len(ends) == len(identifiers) == len(lookups) and request_ids == {event.get('requestId') for event in ends} == {event.get('requestId') for event in observations} and all(event.get('streamTerminal') == 'response.completed' and not event.get('incompleteStream') and not event.get('transportError') and event.get('status') == 200 for event in ends) and all(valid_cost(row.get('metadata', {}).get('total_cost')) for row in lookups)+ complete = log_valid and bool(identifiers) and len(request_ids) == len(starts) == len(ends) == len(identifiers) == len(lookups) and request_ids == {event.get('requestId') for event in ends} == {event.get('requestId') for event in observations} and all(event.get('streamTerminal') == 'response.completed' and not event.get('incompleteStream') and not event.get('transportError') and event.get('status') == 200 for event in ends) and all(valid_cost(row.get('metadata', {}).get('total_cost')) for row in lookups)known = sum(row.get('metadata', {}).get('total_cost', 0) for row in lookups if valid_cost(row.get('metadata', {}).get('total_cost')))result = {'cell': plan['cell']['id'], 'request_starts': len(starts), 'generation_ids': len(identifiers), 'all_requests_accounted': complete, 'known_gateway_cost_usd': known, 'gateway_cost_usd': known if complete else None, 'baseline_cost_usd': plan['baseline_cost_usd'], 'jev_cost_usd': 0}grading = out / 'grading-receipt.json'if grading.exists():- result['official_resolved'] = json.loads(grading.read_text()).get('resolved_instances') == 1- result['protocol_valid'] = json.loads((out / 'receipt.json').read_text()).get('required_retrieval_observed') is True+ grade_receipt = json.loads(grading.read_text())+ result['official_resolved'] = grade_receipt.get('run_id') == 'jg-' + plan['cell']['id'] and grade_receipt.get('exit_code') == 0 and grade_receipt.get('resolved_instances') == 1+ result['protocol_valid'] = protocol_valid(receipt)result['successful_cost_win'] = complete and result['official_resolved'] and result['protocol_valid'] and known < plan['baseline_cost_usd']write_json(out / 'generation-accounting.json', result)print(json.dumps(result))+def protocol_valid(receipt):+ return (receipt.get('status') == 'completed' and receipt.get('exit_code') == 0 and+ receipt.get('native_completed') is True and receipt.get('required_retrieval_observed') is True and+ receipt.get('timing_valid') is True)+++def aggregate(args):+ plan = load_cohort(args.plan.resolve())+ rows = []+ for item in plan['cells']:+ out = Path(item['output'])+ def read(name):+ path = out / name+ return json.loads(path.read_text()) if path.exists() else {}+ receipt, grading, billing = read('receipt.json'), read('grading-receipt.json'), read('generation-accounting.json')+ belongs = receipt.get('cell') == item['cell'] and receipt.get('plan_sha256') == digest(args.plan)+ terminal = belongs and receipt.get('status') in ('completed', 'failed', 'interrupted')+ grade_attempted = terminal and grading.get('run_id') == 'jg-' + item['cell']['id'] and type(grading.get('exit_code')) is int+ graded = grade_attempted and grading['exit_code'] == 0+ protocol = terminal and protocol_valid(receipt)+ solved = graded and grading.get('resolved_instances') == 1+ billed = (terminal and billing.get('cell') == item['cell']['id'] and+ billing.get('all_requests_accounted') is True and valid_cost(billing.get('gateway_cost_usd')))+ cost = billing.get('gateway_cost_usd') if billed else None+ rows.append({'task': item['cell']['instance_id'], 'status': receipt.get('status', 'not-started'),+ 'terminal': terminal, 'grading_attempted': grade_attempted, 'graded': graded, 'protocol_valid': protocol, 'official_resolved': solved, 'resolved': solved and protocol,+ 'baseline_resolved': item['baseline_resolved'], 'baseline_cost_usd': item['baseline_cost_usd'],+ 'fully_billed': billed, 'gateway_cost_usd': cost,+ 'known_gateway_cost_usd': billing.get('known_gateway_cost_usd') if belongs and billing.get('cell') == item['cell']['id'] and valid_cost(billing.get('known_gateway_cost_usd')) else None,+ 'successful_cost_win': solved and protocol and billed and cost < item['baseline_cost_usd']})+ full = {row['task'] for row in rows} == {item['task'] for item in REGISTRY['tasks']}+ preserved = sum(row['baseline_resolved'] and row['resolved'] and row['protocol_valid'] for row in rows)+ wins = sum(row['successful_cost_win'] for row in rows)+ complete = full and all(row['terminal'] and row['grading_attempted'] for row in rows)+ result = {'prospective_full_cohort': full, 'complete': complete, 'baseline_solves_preserved': preserved,+ 'fully_billed_solved_cost_wins': wins,+ 'accepted': complete and preserved == REGISTRY['baseline_solves'] and wins >= REGISTRY['required_cost_wins'],+ 'known_gateway_subtotal_usd': sum(row['known_gateway_cost_usd'] for row in rows if row['known_gateway_cost_usd'] is not None),+ 'fully_billed_total_usd': sum(row['gateway_cost_usd'] for row in rows) if all(row['fully_billed'] for row in rows) else None,+ 'cells': rows}+ write_json(args.plan.resolve().parent / 'aggregate.json', result)+ print(json.dumps(result))+ return result++def main():parser = argparse.ArgumentParser(description=__doc__)sub = parser.add_subparsers(dest='operation', required=True)prep = sub.add_parser('prepare', help='Install a package in the retained image and freeze a new treatment; no model calls')prep.add_argument('--package', type=Path, required=True)prep.add_argument('--output', type=Path, required=True)- prep.add_argument('--task', default=TASK)- prep.add_argument('--baseline', type=Path, default=BASELINE)- prep.add_argument('--inputs', type=Path, default=RUNS / 'auto-research-80/ten-agent-inputs.json')+ prep.add_argument('--task', default='all', choices=['all', *[item['task'] for item in REGISTRY['tasks']]])+ prep.add_argument('--evidence-root', type=Path, default=ROOT)prep.add_argument('--skill', type=Path, default=ROOT / 'skills/jevgrep/SKILL.md')execute = sub.add_parser('run', help='Run the paid coding-agent treatment once, or validate without calls')execute.add_argument('--plan', type=Path, required=True)execute.add_argument('--dry-run', action='store_true')grading = sub.add_parser('grade', help='Run the official evaluator under a new run ID')grading.add_argument('--plan', type=Path, required=True)- grading.add_argument('--dataset', type=Path, default=RUNS / 'auto-research-80/evaluator-only/ten-dataset.json')- grading.add_argument('--tooling', type=Path, default=TOOLING)billing = sub.add_parser('account', help='Reconcile full Sol billing; never count unknown charges as zero')billing.add_argument('--plan', type=Path, required=True)+ summary = sub.add_parser('aggregate', help='Evaluate the prospective ten-task acceptance rule without model calls')+ summary.add_argument('--plan', type=Path, required=True)+ for operation in [execute, grading, billing]:+ choice = operation.add_mutually_exclusive_group(required=True)+ choice.add_argument('--task', choices=[item['task'] for item in REGISTRY['tasks']])+ choice.add_argument('--all', action='store_true')args = parser.parse_args()- {'prepare': prepare, 'run': run, 'grade': grade, 'account': account}[args.operation](args)+ if args.operation in ('prepare', 'aggregate'):+ {'prepare': prepare, 'aggregate': aggregate}[args.operation](args)+ return+ cohort = load_cohort(args.plan.resolve())+ tasks = [item['cell']['instance_id'] for item in cohort['cells']] if args.all else [args.task]+ failed = False+ for task in tasks:+ args.task = task+ try:+ {'run': run, 'grade': grade, 'account': account}[args.operation](args)+ except SystemExit:+ failed = True+ if failed:+ raise SystemExit(1)if __name__ == '__main__':diff --git a/evals/implementation/swebench/test_installed.py b/evals/implementation/swebench/test_installed.pyindex 0c1742d..f6b3b8a 100644--- a/evals/implementation/swebench/test_installed.py+++ b/evals/implementation/swebench/test_installed.py@@ -34,27 +34,46 @@ class InstalledTests(unittest.TestCase):self.assertIn('must run jg for initial repository research', prompt)self.assertTrue(prompt.endswith(runner.baseline_prompt(row)))- def fixture(self, root):- baseline = root / 'baseline'; baseline.mkdir()- row = {'instance_id':runner.TASK,'repo':'psf/requests','base_commit':'public-base','image':'official-image','problem_statement':'Fix behavior.'}- inputs = root / 'agent-inputs.json'; runner.write_json(inputs,[row])- cell = {'id':runner.BASELINE_ID,'instance_id':runner.TASK,'engine':'codex','arm':'baseline','invocation':'required','source_image':'official-image','source_pinned_ref':'official@sha256:retained','image':runner.RUNTIME_IMAGE,'model':'openai/gpt-5.6-sol','effort':'medium','timeout_seconds':900,'gateway_model':'openai/gpt-5.6-sol','expected_image_head':'a'*40,'expected_source_tree':'b'*40}- (baseline/'prompt.txt').write_text(runner.baseline_prompt(row))- runner.write_json(baseline/'receipt.json',{'image':runner.RUNTIME_IMAGE,'cell':cell,'versions':runner.VERSIONS,'provider_route':'vercel-ai-gateway','prompt_sha256':runner.digest(baseline/'prompt.txt')})- runner.write_json(baseline/'grading-receipt.json',{'resolved_instances':1})+ def fixture(self, root, all_tasks=False):+ registry = json.loads(json.dumps(runner.REGISTRY))+ entries = registry['tasks'] if all_tasks else [registry['tasks'][4]]+ cells, rows = [], []skill = root/'skill.md'; skill.write_text('installed skill')package = root/'package.tgz'; package.write_bytes(b'package')prefix = root/'installed-prefix.tar'; prefix.write_bytes(b'installed prefix')- paths = [Path(runner.__file__),HERE/'gateway_broker.py',inputs,skill,package,prefix]- plan = {'status':'frozen','baseline':str(baseline),'baseline_receipt_sha256':runner.digest(baseline/'receipt.json'),'baseline_cost_usd':0.2685004,'cell':{**cell,'id':'installed-new','arm':'chunks','candidate':str(prefix)},'inputs':str(inputs),'skill':str(skill),'package':str(package),'output':str(root/'attempt'),'artifacts':{str(p):runner.digest(p) for p in paths}}- plan_path=root/'plan.json'; runner.write_json(plan_path,plan)- return plan_path,plan+ for index, entry in enumerate(entries):+ baseline = root / ('baseline-' + str(index)); baseline.mkdir()+ cell = entry['cell']+ row = {'instance_id':entry['task'],'repo':'fixture/repo','base_commit':'public-base','image':cell['source_image'],'problem_statement':'Fix behavior.'}+ rows.append(row)+ (baseline/'prompt.txt').write_text(runner.baseline_prompt(row))+ runner.write_json(baseline/'receipt.json',{'image':cell['image'],'cell':cell,'versions':runner.VERSIONS,'provider_route':'vercel-ai-gateway','prompt_sha256':runner.digest(baseline/'prompt.txt')})+ runner.write_json(baseline/'grading-receipt.json',{'resolved_instances':int(entry['resolved'])})+ entry['files']={name:runner.digest(baseline/name) for name in ['receipt.json','prompt.txt','grading-receipt.json']}+ cells.append({'baseline':str(baseline),'baseline_cost_usd':entry['cost_usd'],'baseline_resolved':entry['resolved'],+ 'cell':{**cell,'id':'installed-' + str(index),'arm':'chunks','candidate':str(prefix)},'output':str(root / ('attempt-' + str(index)))})+ registry_path = root/'fixed-baselines.json';runner.write_json(registry_path,registry)+ patcher=patch.object(runner,'REGISTRY',registry);patcher.start();self.addCleanup(patcher.stop)+ inputs = root / 'agent-inputs.json'; runner.write_json(inputs,rows)+ dataset=root/'dataset.json';dataset.write_text('fixture dataset')+ paths = [dataset,Path(runner.__file__),HERE/'gateway_broker.py',registry_path,inputs,skill,package,prefix]+ plan = {'schema':2,'status':'frozen','cells':cells,'runner':str(Path(runner.__file__)),'broker':str(HERE/'gateway_broker.py'),+ 'registry':str(registry_path),'tooling':str(root/'tooling'),'dataset':str(root/'dataset.json'),+ 'inputs':str(inputs),'skill':str(skill),'package':str(package),'artifacts':{str(p):runner.digest(p) for p in paths}}+ plan_path=root/'plan.json';runner.write_json(plan_path,plan)+ return plan_path,{**plan,**cells[0]}++ def receipt(self, path, plan, status='completed'):+ out=Path(plan['output']);out.mkdir(exist_ok=True)+ runner.write_json(out/'receipt.json',{'plan_sha256':runner.digest(path),'cell':plan['cell'],'status':status,+ 'required_retrieval_observed':True,'timing_valid':True,'native_completed':True,'exit_code':0,'host_pid':os.getpid()})+ return outdef test_dry_run_validates_pair_without_credentials_or_agent(self):with tempfile.TemporaryDirectory() as temporary:path,plan=self.fixture(Path(temporary))- with patch.object(runner,'check_image',return_value=runner.RUNTIME_IMAGE),patch.object(runner,'command',side_effect=AssertionError('No agent or paid command allowed')),patch.dict(os.environ,{},clear=True),contextlib.redirect_stdout(io.StringIO()) as stdout:- runner.run(argparse.Namespace(plan=path,dry_run=True))+ with patch.object(runner,'check_image',return_value=plan['cell']['image']),patch.object(runner,'command',side_effect=AssertionError('No agent or paid command allowed')),patch.dict(os.environ,{},clear=True),contextlib.redirect_stdout(io.StringIO()) as stdout:+ runner.run(argparse.Namespace(plan=path,dry_run=True,task=None))self.assertEqual(json.loads(stdout.getvalue())['paid_calls'],0)self.assertFalse(Path(plan['output']).exists())Path(plan['cell']['candidate']).write_bytes(b'changed')@@ -63,29 +82,84 @@ class InstalledTests(unittest.TestCase):def test_changed_model_is_not_paired_to_the_old_baseline(self):with tempfile.TemporaryDirectory() as temporary:path,plan=self.fixture(Path(temporary))- plan['cell']['model']='another-model';runner.write_json(path,plan)+ cohort=json.loads(path.read_text());cohort['cells'][0]['cell']['model']='another-model';runner.write_json(path,cohort)with self.assertRaisesRegex(ValueError,'pairing changed'):runner.load_plan(path)def test_grading_rejects_a_changed_official_harness_before_execution(self):with tempfile.TemporaryDirectory() as temporary:- root=Path(temporary);path,plan=self.fixture(root);out=Path(plan['output']);out.mkdir()- runner.write_json(out/'receipt.json',{'plan_sha256':runner.digest(path),'cell':plan['cell']})+ root=Path(temporary);path,plan=self.fixture(root);self.receipt(path,plan)with patch.object(runner,'text',return_value='changed-harness'),patch.object(runner.subprocess,'run',side_effect=AssertionError('No grader launched')):with self.assertRaisesRegex(ValueError,'harness checkout changed'):- runner.grade(argparse.Namespace(plan=path,tooling=root/'tooling',dataset=root/'dataset.json'))+ runner.grade(argparse.Namespace(plan=path,tooling=root/'tooling',dataset=root/'dataset.json',task=None))def test_incomplete_billing_never_counts_missing_charges_as_free(self):with tempfile.TemporaryDirectory() as temporary:- path,plan=self.fixture(Path(temporary));out=Path(plan['output']);out.mkdir()+ path,plan=self.fixture(Path(temporary));out=self.receipt(path,plan)events=[{'kind':'codex-request-start','operation':'generation','requestId':'one'},{'kind':'codex-generation','requestId':'one','generationId':'gen_one'},{'kind':'codex-gateway','requestId':'one','status':200,'streamTerminal':'response.completed'}, {'kind':'codex-request-start','operation':'generation','requestId':'missing'}](out/'proxy.jsonl').write_text('\n'.join(json.dumps(e) for e in events))- runner.write_json(out/'generation-lookups.json',[{'id':'gen_one','metadata':{'total_cost':0.1}}])- runner.write_json(out/'grading-receipt.json',{'resolved_instances':1});runner.write_json(out/'receipt.json',{'required_retrieval_observed':True})- with patch.object(runner.urllib.request,'urlopen',side_effect=AssertionError('No network')),contextlib.redirect_stdout(io.StringIO()):runner.account(argparse.Namespace(plan=path))+ runner.write_json(out/'generation-lookups.json',[{'id':'gen_one','metadata':{'id':'gen_one','model':'openai/gpt-5.6-sol','total_cost':0.1}}])+ runner.write_json(out/'grading-receipt.json',{'resolved_instances':1})+ with patch.object(runner.urllib.request,'urlopen',side_effect=AssertionError('No network')),contextlib.redirect_stdout(io.StringIO()):runner.account(argparse.Namespace(plan=path,task=None))result=json.loads((out/'generation-accounting.json').read_text())self.assertEqual(result['known_gateway_cost_usd'],0.1)self.assertIsNone(result['gateway_cost_usd']);self.assertFalse(result['successful_cost_win'])+ def test_terminal_failures_are_retained_and_active_attempts_stop(self):+ with tempfile.TemporaryDirectory() as temporary:+ path,plan=self.fixture(Path(temporary));out=self.receipt(path,plan,'failed')+ with patch.object(runner,'text',side_effect=AssertionError('Terminal attempts need no process probe')):+ self.assertTrue(runner.existing_attempt(plan))+ self.receipt(path,plan,'running')+ with patch.object(runner,'text',return_value='active-container'):+ with self.assertRaisesRegex(ValueError,'active'):runner.existing_attempt(plan)+ with patch.object(runner.os,'kill',side_effect=ProcessLookupError),patch.object(runner,'text',return_value=''):+ self.assertTrue(runner.existing_attempt(plan))+ self.assertEqual(json.loads((out/'receipt.json').read_text())['status'],'interrupted')++ def test_cohort_gate_requires_eight_preserved_and_seven_fully_billed_wins(self):+ with tempfile.TemporaryDirectory() as temporary:+ path,plan=self.fixture(Path(temporary),all_tasks=True)+ for index,item in enumerate(plan['cells']):+ out=self.receipt(path,item)+ runner.write_json(out/'grading-receipt.json',{'run_id':'jg-'+item['cell']['id'],'exit_code':0,'resolved_instances':int(item['baseline_resolved'])})+ runner.write_json(out/'generation-accounting.json',{'cell':item['cell']['id'],'all_requests_accounted':index<7,'gateway_cost_usd':0.01 if index<7 else None,'known_gateway_cost_usd':0.01})+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))+ self.assertTrue(result['accepted']);self.assertEqual(result['baseline_solves_preserved'],8)+ self.assertEqual(result['fully_billed_solved_cost_wins'],7);self.assertIsNone(result['fully_billed_total_usd'])+ for item in plan['cells'][8:]:+ self.receipt(path,item,'failed')+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))+ self.assertTrue(result['accepted'])+ self.assertFalse(result['cells'][-1]['protocol_valid'])+ self.assertAlmostEqual(result['known_gateway_subtotal_usd'],0.1)+ last=plan['cells'][-1]+ runner.write_json(Path(last['output'])/'grading-receipt.json',{'run_id':'jg-'+last['cell']['id'],'exit_code':1})+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))+ self.assertTrue(result['accepted'])+ out=Path(plan['cells'][0]['output']);runner.write_json(out/'generation-accounting.json',{'cell':plan['cells'][0]['cell']['id'],'all_requests_accounted':False,'gateway_cost_usd':None,'successful_cost_win':True})+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))+ self.assertFalse(result['accepted']);self.assertEqual(result['fully_billed_solved_cost_wins'],6)+ runner.write_json(out/'grading-receipt.json',{'run_id':'jg-'+plan['cells'][0]['cell']['id'],'exit_code':0,'resolved_instances':0})+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))+ self.assertEqual(result['baseline_solves_preserved'],7)++ def test_cohort_cannot_mix_installations_or_reuse_an_attempt_directory(self):+ with tempfile.TemporaryDirectory() as temporary:+ path,_=self.fixture(Path(temporary),all_tasks=True)+ plan=json.loads(path.read_text())+ original=plan['cells'][1]['cell']['candidate']+ plan['cells'][1]['cell']['candidate']='different-installation';runner.write_json(path,plan)+ with self.assertRaisesRegex(ValueError,'one prefix'):runner.load_cohort(path)+ plan['cells'][1]['cell']['candidate']=original+ plan['cells'][1]['output']=plan['cells'][0]['output'];runner.write_json(path,plan)+ with self.assertRaisesRegex(ValueError,'distinct attempt'):runner.load_cohort(path)++ def test_single_task_cannot_claim_full_cohort_acceptance(self):+ with tempfile.TemporaryDirectory() as temporary:+ path,_=self.fixture(Path(temporary))+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))+ self.assertFalse(result['prospective_full_cohort']);self.assertFalse(result['accepted'])+def test_broker_captures_exact_jev_bodies_without_auth_headers(self):with tempfile.TemporaryDirectory() as temporary:root=Path(temporary);config=root/'gateway.json';config.write_text(json.dumps({'key':'REAL_KEY_SENTINEL','token':'BROKER_TOKEN_SENTINEL','allow_jev':True,'agent_engine':'codex'}))