Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Evaluate one frozen installed package across fixed SWE-bench cohort

David Zhang679bb9a92425
diff --git a/evals/implementation/swebench/fixed-baselines.json b/evals/implementation/swebench/fixed-baselines.json
new file mode 100644
index 0000000..bc2d5af
--- /dev/null
+++ b/evals/implementation/swebench/fixed-baselines.json
@@ -0,0 +1,300 @@
+{
+ "inputs": "evals/runs/swebench/auto-research-80/ten-agent-inputs.json",
+ "dataset": "evals/runs/swebench/auto-research-80/evaluator-only/ten-dataset.json",
+ "tooling": "evals/runs/tooling/swebench",
+ "baseline_solves": 8,
+ "required_cost_wins": 7,
+ "tasks": [
+ {
+ "task": "scikit-learn__scikit-learn-13124",
+ "baseline": "evals/runs/swebench/ranked-leads-additional/scikit-learn__scikit-learn-13124-baseline",
+ "cell": {
+ "id": "ranked-additional-scikit-learn__scikit-learn-13124-baseline",
+ "instance_id": "scikit-learn__scikit-learn-13124",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-13124:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-13124@sha256:bdeb5e17bf55616082bf65f2b83de5cfc92556f3548fb38f4095ed1f802a7ec0",
+ "image": "sha256:ad9300592673fe015a1cea83a5697685b2b1c8068cf5f44126aad1531fadb851",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "aec0d00a29e5283d5a134489e27dcdfc92f350e3",
+ "expected_source_tree": "72e1da048a03e5f2d1728e17a93b7b687ae78b80"
+ },
+ "cost_usd": 0.29443600000000003,
+ "resolved": true,
+ "requests": 13,
+ "files": {
+ "receipt.json": "10f9381d2ab45233f935f318a0a6753c2634308e57d136b33f5cffadf2150ab7",
+ "prompt.txt": "e1662a59921f366d1e98b0c0f4d317e65dda11d10268a066a28fb6ce08f0d0e8",
+ "grading-receipt.json": "a323f3720d7acb68b91d46e3b39b0e47322fc5358256e2702be3a063ec8c33cf",
+ "generation-lookups.json": "1233a1be3db32a2e32a18555132741a499b3236d195672734faae9b561a98836"
+ }
+ },
+ {
+ "task": "django__django-15629",
+ "baseline": "evals/runs/swebench/lookahead-native-v88/django__django-15629/codex-baseline",
+ "cell": {
+ "id": "lookahead-native-v88-django__django-15629-codex-baseline",
+ "instance_id": "django__django-15629",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.django_1776_django-15629:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.django_1776_django-15629@sha256:70d0f93a3891c6d334c7cc3028b2bf1de51b64a216c3d99712608cc9a0e501ae",
+ "image": "sha256:514e444b11b59fe02f003eca0253c59c3a28a8b6377c65e64d021176d180dffe",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "9b70bb1938e97614e57443bbade13b71730f813e",
+ "expected_source_tree": "ae7d451039d57611fc1269a6167d55fe1f4d49be"
+ },
+ "cost_usd": 1.5055472,
+ "resolved": true,
+ "requests": 40,
+ "files": {
+ "receipt.json": "7eb3d68ad77860a0d8978ffa754836e40f6830a19aa3cd7a92220a3da6b28af9",
+ "prompt.txt": "4b22482d501f6fbd977344ba5309a928f57ac7809f59713cb191c99ab7c04640",
+ "grading-receipt.json": "c5a4b7649c658b1f5f149c8e452b72a33b34b83457b968fc4175a93e684c6d51",
+ "generation-lookups.json": "9ab4210b09194ac149d083668fe70743e7a17c42e855457b01f5ccb7841a8b1f"
+ }
+ },
+ {
+ "task": "astropy__astropy-13579",
+ "baseline": "evals/runs/swebench/ranked-leads-additional/astropy__astropy-13579-baseline",
+ "cell": {
+ "id": "ranked-additional-astropy__astropy-13579-baseline",
+ "instance_id": "astropy__astropy-13579",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.astropy_1776_astropy-13579:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.astropy_1776_astropy-13579@sha256:3a4b8059992d3eed81046bd4b3bd3450993a237a909c2298ce69b59bb5b9c863",
+ "image": "sha256:2dd8342d0335721880149fa975922d788a13523f44273c6165e3057bc4da0322",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "62194260181d107f2f6630d4588887fbf46f5a0c",
+ "expected_source_tree": "55a69b663232a3ee49ae06632d535dd3b1ced6c8"
+ },
+ "cost_usd": 0.428631,
+ "resolved": true,
+ "requests": 17,
+ "files": {
+ "receipt.json": "2dc42050943fc27a131a5f59c8c9375a75a76af29688a56be08a3ee0c95a7bed",
+ "prompt.txt": "33c6e22d8f2072d941c0a0f3d513494646cdaaa58bf354404121ab565c062bf9",
+ "grading-receipt.json": "1052eb2459ce64de6b4b451ecbb169d231246a69ffbb60dd8222e3497ccbb5dc",
+ "generation-lookups.json": "b95664cf3aebb15f8a08788854707719b28a273dc75d6b5e5f2ba01eb5ffe9a9"
+ }
+ },
+ {
+ "task": "pydata__xarray-3305",
+ "baseline": "evals/runs/swebench/auto-research-80/reserved-check/pydata__xarray-3305-baseline",
+ "cell": {
+ "id": "auto80-reserved-pydata__xarray-3305-baseline",
+ "instance_id": "pydata__xarray-3305",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.pydata_1776_xarray-3305:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.pydata_1776_xarray-3305@sha256:0e234daddf9a42b85a77ba0436f3fcbad90759921c978b127dff90d55cba0f89",
+ "image": "sha256:e0188f9a67225bf0fbb0bad90f5751a751e42002b690396695f25fa8f30741e3",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "cad44a667322dadb32d09f1db9ed62d9875ac43e",
+ "expected_source_tree": "f0cc03a70ee44542b9f21046fbb5fe3305e36c77"
+ },
+ "cost_usd": 0.4937066,
+ "resolved": true,
+ "requests": 23,
+ "files": {
+ "receipt.json": "55ed7868a65323413496849d2fca6547fdb8ff5895da9036645a84738319dd12",
+ "prompt.txt": "5e3fe2093c9948dca235a26fce9825a7f8e9c89fc9da953fe270c61682bdf3f6",
+ "grading-receipt.json": "c75fdec46bcd1bbe2c5bdca098119d1137ae8201b009a8385b422ce0b8ec8f20",
+ "generation-lookups.json": "fd0ee68b0ddf51cc8488db797ec48eda7ef02b701af2d9477d7bbc1335a5862d"
+ }
+ },
+ {
+ "task": "psf__requests-1142",
+ "baseline": "evals/runs/swebench/lookahead-native-v88/psf__requests-1142/codex-baseline",
+ "cell": {
+ "id": "lookahead-native-v88-psf__requests-1142-codex-baseline",
+ "instance_id": "psf__requests-1142",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.psf_1776_requests-1142:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.psf_1776_requests-1142@sha256:fa9c859412109d003a68c2bbe3c65541ca4714526afeb6906fe589c18af0b30d",
+ "image": "sha256:2264606b0765897361bb392db8aa3226e171719ba97e82ce5205ba122a43eaff",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "546130c8c878adabd16ac5221a36e39e4dab3645",
+ "expected_source_tree": "3b8ba30961592000c9a9a73b41d8452c871f47b4"
+ },
+ "cost_usd": 0.2685004,
+ "resolved": true,
+ "requests": 11,
+ "files": {
+ "receipt.json": "ee0d055679453e5c236a6a4d6a34e53b3deb9ca0436e2d37357df4d0c40ca88d",
+ "prompt.txt": "12ba196760c861377a033958d9d1a3cd3cc25dd4a1fa6cf877425001a14bae5c",
+ "grading-receipt.json": "5ec41b60522d4f4bd16d55f96991c4f4a53af4df7f9a13096e53597357c8d7f1",
+ "generation-lookups.json": "d7e657c0ff71b2279cb1dabfea058de6a914c3f68b2eab473fdbfc6f6af0e757"
+ }
+ },
+ {
+ "task": "sympy__sympy-16792",
+ "baseline": "evals/runs/swebench/lookahead-native-v88/sympy__sympy-16792/codex-baseline",
+ "cell": {
+ "id": "lookahead-native-v88-sympy__sympy-16792-codex-baseline",
+ "instance_id": "sympy__sympy-16792",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.sympy_1776_sympy-16792:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.sympy_1776_sympy-16792@sha256:3064872f9fef7139d864d821938635b2798232942db1dbe1b95bc085b45bfc61",
+ "image": "sha256:664a16898db69c3ad4d85d6db4dc0cf50cdbacc34a58ec5b6465ec9cc20eb771",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "4d0daaed7f939d3a8fbacfcb571561b7c3745e05",
+ "expected_source_tree": "5b2ea26c6ef59f15dc46c0e9b5c90d8935101ce7"
+ },
+ "cost_usd": 0.547975,
+ "resolved": true,
+ "requests": 28,
+ "files": {
+ "receipt.json": "0daf2c17c6fc7a13258722b5fe5d7657d0e8a862ba2d289a6392367da5ca26d0",
+ "prompt.txt": "bda6bd748dfb21a74c6ea7298a66937da2e530621918ac0bb2426f406b2d3d5c",
+ "grading-receipt.json": "62e0af48d4353b5d4ecf600fe040350c4f06fe41a1ae941f4a6af72642173d3b",
+ "generation-lookups.json": "431a34b987ed2f117c077dfb5fae22633fcccaabbc234df2fe49d49935052e95"
+ }
+ },
+ {
+ "task": "pytest-dev__pytest-6197",
+ "baseline": "evals/runs/swebench/ranked-leads-additional/pytest-dev__pytest-6197-baseline",
+ "cell": {
+ "id": "ranked-additional-pytest-dev__pytest-6197-baseline",
+ "instance_id": "pytest-dev__pytest-6197",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-6197:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-6197@sha256:15d39592171743355fdade433454d2dbd5cd7d7db21bdcf0e2f125edecabaf09",
+ "image": "sha256:77ae3ccb908cc80d961c74e8351029382ab74fa171b2f5387eceb10da2c2b23f",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "4eecc51564648be4823fcc5f6f7ef208a4f84a8c",
+ "expected_source_tree": "4024da6943c752a72e2ed45b8b57c03ccd047371"
+ },
+ "cost_usd": 2.344458,
+ "resolved": true,
+ "requests": 62,
+ "files": {
+ "receipt.json": "1997160d2bb3c3bb2da69b948ca36a836022309c4e6b81d8275d7325cfc59187",
+ "prompt.txt": "04386d2e8c1845f42ab6d26f77b6daf1ea7a320dc6df5b1c4c005fd5044a9bc6",
+ "grading-receipt.json": "e4d4ee179321ec72d3ede2a69e94eb2396734d1c3b9bf9e002c2938db9c47f87",
+ "generation-lookups.json": "66de257f7f8e0ca6aaadc9a8d268faf12763a298843ae5393e99861238f002da"
+ }
+ },
+ {
+ "task": "sphinx-doc__sphinx-8638",
+ "baseline": "evals/runs/swebench/auto-research-80/reserved-check/sphinx-doc__sphinx-8638-baseline",
+ "cell": {
+ "id": "auto80-reserved-sphinx-doc__sphinx-8638-baseline",
+ "instance_id": "sphinx-doc__sphinx-8638",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-8638:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-8638@sha256:3f2d3335613ab578fc026ddf8cda2a835e8f0f24ba6c0605e53241ec518bf78a",
+ "image": "sha256:98e6227437c5cdc843e663065b3118603afac2239de969097735a564e82705e9",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "27a63c4bbe23e14896382548356d2f9c880812d5",
+ "expected_source_tree": "742277e5b9486c9781a497aad5da88b8b0d5bb96"
+ },
+ "cost_usd": 1.0594724,
+ "resolved": true,
+ "requests": 25,
+ "files": {
+ "receipt.json": "ced56c0bcc21d699cf7d1dd9616a47453293a7687f8d5ae43d0a5f273f368dcf",
+ "prompt.txt": "78e276ae82c6252b2e1b0a1ac8b9833271bc0be79f9a0310d3d9b4b82da12be8",
+ "grading-receipt.json": "2a86f9fe4dcf991a7e24af4c566d31b8be1148828379883e67193e326605a218",
+ "generation-lookups.json": "30ebe7af2afb540a067e364b68056e3cca16011384ddbaee3d258c31a425f9f0"
+ }
+ },
+ {
+ "task": "matplotlib__matplotlib-26466",
+ "baseline": "evals/runs/swebench/auto-research-80/reserved-check/matplotlib__matplotlib-26466-baseline",
+ "cell": {
+ "id": "auto80-reserved-matplotlib__matplotlib-26466-baseline",
+ "instance_id": "matplotlib__matplotlib-26466",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.matplotlib_1776_matplotlib-26466:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.matplotlib_1776_matplotlib-26466@sha256:35be1078dc800d0bfcb32fef52650b1915480bca8778f2ed0a5528d4b6929784",
+ "image": "sha256:969a5ccf3f34af256d0ee9bbf87422c9483eef96c32687281f472dd66729a239",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "20e9c8c37ec0e0bb36f0bc6b1761200ffaa846fd",
+ "expected_source_tree": "92c6a499aede7b947b743b5c89ca08e9ac523c0d"
+ },
+ "cost_usd": 0.395716,
+ "resolved": false,
+ "requests": 13,
+ "files": {
+ "receipt.json": "dd8e284ee2a077bc42166b5b0765bdefde4336293dd44d4569a72a626d6fbc94",
+ "prompt.txt": "e15b7bc7bdaa1c59ac44f6904b4b0892e48bfea57e37589dae302b3cf8737d2f",
+ "grading-receipt.json": "8ae2c27b62aa3c7144fe3b932b023b182dde3389c72658dac0288651bf73a3d2",
+ "generation-lookups.json": "ddbe18196d46f27bece8efbff069ba4c19ea80f50941b2e0390436582b7b2b9f"
+ }
+ },
+ {
+ "task": "pylint-dev__pylint-4604",
+ "baseline": "evals/runs/swebench/auto-research-80/reserved-check/pylint-dev__pylint-4604-baseline",
+ "cell": {
+ "id": "auto80-reserved-pylint-dev__pylint-4604-baseline",
+ "instance_id": "pylint-dev__pylint-4604",
+ "engine": "codex",
+ "arm": "baseline",
+ "invocation": "required",
+ "source_image": "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-4604:latest",
+ "source_pinned_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-4604@sha256:e24d647f8421c551d60da005ce2272910c0aae8bb37f9de870b10c5e70b840ce",
+ "image": "sha256:802e9a848f7268b436980bc4305ea815d89a30219e3dd4727f26a1f38ae7563b",
+ "model": "openai/gpt-5.6-sol",
+ "effort": "medium",
+ "timeout_seconds": 900,
+ "gateway_model": "openai/gpt-5.6-sol",
+ "expected_image_head": "0f266873c24f8f7270d257c8c8a8676930be0c72",
+ "expected_source_tree": "05a6b93fcb973fb72fe368b7d43eee00ed6635da"
+ },
+ "cost_usd": 0.2836264,
+ "resolved": false,
+ "requests": 13,
+ "files": {
+ "receipt.json": "54b00479be48fc888e7f2ce51082ebf41d0de1a3e8e9677dabcc6dd949cac611",
+ "prompt.txt": "7b88082c7bdcd31b3ecde24fd7a56e1d8261d2ffb4129828931e87178a2e6928",
+ "grading-receipt.json": "e380e835bebd324b81942208c526405d45e8b8c7b2e76167e4947da63b1cedb2",
+ "generation-lookups.json": "73bd2ed7d04398ade9954f7d2768357f5e91ee7127239c0ed24df1f33981acea"
+ },
+ "caveat": "Retained official nonpass: test collection fails on missing IS_PYPY export; not a demonstrated behavioral failure."
+ }
+ ]
+}
diff --git a/evals/implementation/swebench/installed.md b/evals/implementation/swebench/installed.md
index 0f36a47..a849eff 100644
--- a/evals/implementation/swebench/installed.md
+++ b/evals/implementation/swebench/installed.md
@@ -1,46 +1,64 @@
-# Installed Requests checkpoint
+# Installed package cohort
-[The runner](installed.py) compares one frozen installed `jg` package with the
-retained Requests Sol baseline. It refuses changed model, source, runtime and
-baseline-prompt identities. It never launches a baseline. A successful checkpoint
-is integration evidence, not the ten-task quality gate.
+[The runner](installed.py) evaluates one frozen installed `jg` package against
+[the ten fixed Sol baselines](fixed-baselines.json). Baseline receipts, prompts,
+grades and billing are checked by hash; there is no baseline execution command.
+The two recorded baseline nonpasses remain nonpasses, including the retained
+Pylint test-collection limitation.
-Preparation installs the packed npm artifact and its dependencies in a disposable
-copy of the retained Linux image. Registry access is confined to preparation;
-the agent receives the archived installed prefix on an internal network. It runs
-the real `jg` executable. The packaged skill is verified against the canonical
-skill, and the benchmark explicitly requires initial retrieval even when the
-production skill would consider it optional.
+Preparation installs the packed npm artifact and dependencies once, then checks
+that same installation offline in every selected runtime image. Each check
+verifies the original source, agent versions and canonical packaged skill.
+The coding agent receives the archived prefix and runs the real `jg` executable
+on an internal network. The unchanged benchmark prompt requires initial retrieval,
+even when the production skill would consider it optional. Retrieval limits come
+from the frozen package's default policy.
-Use `python3 evals/implementation/swebench/installed.py --help` for commands.
-The intended root script alias is `eval:swebench` pointing to that entry point.
-A typical sequence is:
+The plan binds one package, installed prefix, skill, safe task export, evaluator
+dataset and archived runner/broker/registry sources before any treatment runs.
+Existing studies remain unchanged; their archived source records the procedure used.
+A fresh full-cohort plan prevents combining the cheapest outcomes from different
+packages or experiments. A single-task plan is useful for diagnostics but cannot
+satisfy the cohort gate.
```sh
python3 evals/implementation/swebench/installed.py prepare \
--package /absolute/path/to/package.tgz --output /absolute/path/to/new-study
python3 evals/implementation/swebench/installed.py run \
- --plan /absolute/path/to/new-study/plan.json --dry-run
+ --plan /absolute/path/to/new-study/plan.json --all --dry-run
```
-Preparation and dry-run make no model calls. After reviewing the frozen plan,
-`run` without `--dry-run` executes the paid treatment exactly once. Load the
-Gateway credential through the authorized environment workflow; never place a
-real key in the plan or command arguments. Each attempt uses fresh home/cache
-state and the package's default retrieval policy.
-
-Follow execution with `grade --plan ...` and `account --plan ...`. Grading uses
-the existing pinned official environment and a unique run ID. Accounting reuses
-saved generation metadata, fetches missing receipts, and leaves the full cost
-unknown if any generation is unaccounted for. A missing required retrieval call
-cannot count as a successful cost win.
-
-The plan, installation archive, raw agent output, patch, official reports and
-billing remain under ignored run storage. [The broker](gateway_broker.py) retains
-raw Jev request/response bodies separately from transport metadata, only in the
-non-agent container; authentication headers are never captured. These receipts
-contain benchmark source and should stay local with the other study evidence.
-
-Run [the focused validations](test_installed.py) inside an isolated Docker
-container. They exercise frozen prompt identity, drift rejection, no-call dry-run,
-partial billing and exact Jev body capture through a local synthetic provider.
+Preparation and dry-run make no model calls. `prepare` defaults to all ten tasks;
+`--task` selects one registered task. `--evidence-root` locates the retained
+baseline and evaluator files when preparing from another checkout. The root
+`eval:swebench` alias points to this entry point; `--help` lists its arguments.
+
+After reviewing the frozen plan, `run --all` without `--dry-run` executes the paid
+treatments sequentially. Load the Gateway credential through the authorized
+environment workflow. Real keys stay out of plans and command arguments. Every
+attempt has fresh home/cache state and a durable lifecycle receipt. Failed or
+completed attempts are retained; they are never replaced. Active processes stop
+continuation. An abandoned attempt whose runner and containers are gone becomes
+an interrupted result, preserving its evidence.
+
+Follow execution with `grade --plan ... --all` and `account --plan ... --all`.
+Use `--task` instead of `--all` for an individual cell. Grading uses the pinned
+official harness and unique run IDs. Accounting reuses saved generation metadata,
+fetches missing receipts, and leaves full cost unknown if any request is missing.
+Neither operation reruns the coding agent.
+
+`aggregate --plan ...` requires ten terminal attempts with official grading receipts,
+all eight baseline solves preserved by valid solved treatments, and at least seven
+valid solved results with complete billing and strictly lower cost. Failed or
+invalid treatments remain recorded as nonpasses and nonwins. Unknown costs cannot win and
+prevent reporting a full cohort cost total; the known subtotal remains explicit. The aggregate reports individual
+outcomes so failures remain visible.
+
+Run storage holds the frozen artifacts, agent output, patches, official reports
+and billing. [The broker](gateway_broker.py) retains exact Jev request/response
+bodies in the non-agent container without authentication headers. These contain
+benchmark source and remain local with the ignored study evidence.
+
+[Focused tests](test_installed.py) run in an isolated Docker container. They cover
+prompt identity, drift rejection, no-call validation, retained attempts, incomplete
+billing, cohort acceptance and exact body capture through a synthetic provider.
diff --git a/evals/implementation/swebench/installed.py b/evals/implementation/swebench/installed.py
index f5c57e7..695d2ae 100644
--- a/evals/implementation/swebench/installed.py
+++ b/evals/implementation/swebench/installed.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Prepare, execute and grade one frozen installed-package SWE-bench treatment."""
+"""Freeze and evaluate one installed package against the ten retained SWE-bench baselines."""
import argparse
import hashlib
import json
@@ -16,13 +16,8 @@ import uuid
HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[2]
-RUNS = ROOT / 'evals/runs/swebench'
-TOOLING = ROOT / 'evals/runs/tooling/swebench'
-TASK = 'psf__requests-1142'
-BASELINE_ID = 'lookahead-native-v88-psf__requests-1142-codex-baseline'
-RUNTIME_IMAGE = 'sha256:2264606b0765897361bb392db8aa3226e171719ba97e82ce5205ba122a43eaff'
+REGISTRY = json.loads((HERE / 'fixed-baselines.json').read_text())
HARNESS_COMMIT = '02e7a74ffd0b707aab73d203fe87bdc7c76afc8e'
-BASELINE = RUNS / 'lookahead-native-v88' / TASK / 'codex-baseline'
SAFE_FIELDS = {'instance_id', 'repo', 'base_commit', 'image', 'problem_statement'}
VERSIONS = {'node': 'v24.14.0', 'codex': 'codex-cli 0.153.4', 'claude': '2.1.278 (Claude Code)'}
VERIFICATION = (
@@ -75,8 +70,12 @@ def text(args):
def load_pair(baseline, inputs, task):
receipt = json.loads((baseline / 'receipt.json').read_text())
cell = receipt['cell']
- if task != TASK or cell['instance_id'] != task or cell['arm'] != 'baseline' or cell['id'] != BASELINE_ID or receipt['image'] != RUNTIME_IMAGE or cell['image'] != RUNTIME_IMAGE:
- raise ValueError('This checkpoint reuses only the registered Requests baseline')
+ registered = next(item for item in REGISTRY['tasks'] if item['task'] == task)
+ if cell != registered['cell'] or receipt['image'] != cell['image']:
+ raise ValueError('Baseline differs from the fixed registry')
+ for name, expected in registered['files'].items():
+ if digest(baseline / name) != expected:
+ raise ValueError('Retained baseline evidence changed: ' + name)
if (cell['model'], cell['effort'], cell['timeout_seconds']) != ('openai/gpt-5.6-sol', 'medium', 900):
raise ValueError('Baseline model/effort/deadline differs from the registered harness')
if receipt['versions'] != VERSIONS or receipt['provider_route'] != 'vercel-ai-gateway':
@@ -92,19 +91,25 @@ def load_pair(baseline, inputs, task):
if hashlib.sha256(prompt).hexdigest() != receipt['prompt_sha256'] or (baseline / 'prompt.txt').read_bytes() != prompt:
raise ValueError('Baseline prompt differs; do not rerun or overwrite it')
grade = json.loads((baseline / 'grading-receipt.json').read_text())
- if grade.get('resolved_instances') != 1:
- raise ValueError('Expected the retained officially resolved Requests baseline')
+ if (grade.get('resolved_instances') == 1) != registered['resolved']:
+ raise ValueError('Retained official outcome changed')
return receipt, row
def check_image(image):
if not re.fullmatch(r'sha256:[a-f0-9]{64}', image):
raise ValueError('Use the immutable retained runtime image ID')
- return text(['docker', 'image', 'inspect', image, '--format', '{{.Id}}'])
+ actual = text(['docker', 'image', 'inspect', image, '--format', '{{.Id}}'])
+ if actual != image:
+ raise ValueError('Runtime image identity changed')
+ return actual
def prepare(args):
- baseline, row = load_pair(args.baseline.resolve(), args.inputs.resolve(), args.task)
+ root = args.evidence_root.resolve()
+ selected = [item for item in REGISTRY['tasks'] if args.task in ('all', item['task'])]
+ pairs = [(item, *load_pair(root / item['baseline'], root / REGISTRY['inputs'], item['task'])) for item in selected]
+ baseline = pairs[0][1]
image = check_image(baseline['image'])
out = args.output.resolve()
out.mkdir(parents=True, exist_ok=False)
@@ -136,49 +141,123 @@ tar -C /opt -cf /tmp/installed-prefix.tar jg-install
raise ValueError('Installation changed source identity or installed the wrong package')
if (out / 'skill.md').read_bytes() != args.skill.read_bytes():
raise ValueError('Packaged skill differs from the canonical skill')
- write_json(out / 'agent-inputs.json', [row])
- cell = {**cell, 'id': 'installed-' + TASK + '-' + uuid.uuid4().hex[:12], 'arm': 'chunks', 'candidate': str(out / 'installed-prefix.tar')}
- artifacts = [Path(__file__).resolve(), HERE / 'gateway_broker.py', package, out / 'installed-prefix.tar', out / 'skill.md', out / 'agent-inputs.json']
- plan = {'status': 'frozen', 'purpose': 'Installed production Requests checkpoint; reuse the fixed baseline once, no baseline rerun.',
- 'cell': cell, 'baseline': str(args.baseline.resolve()), 'baseline_receipt_sha256': digest(args.baseline / 'receipt.json'),
- 'baseline_cost_usd': 0.2685004, 'baseline_resolved': True, 'inputs': str(out / 'agent-inputs.json'),
- 'skill': str(out / 'skill.md'), 'package': str(package), 'output': str(out / 'attempt'),
+ # Every image gets an offline check using the same installed bytes.
+ cells = []
+ for registered, fixed, _ in pairs:
+ runtime = check_image(fixed['image'])
+ check = 'jg-preflight-' + uuid.uuid4().hex[:12]
+ try:
+ command(['docker', 'create', '--platform', 'linux/amd64', '--network', 'none', '--name', check, runtime, 'sh', '-ec',
+ 'tar -xf /tmp/prefix.tar -C /opt; node --version; codex --version; claude --version; /opt/jg-install/bin/jg --version; /opt/jg-install/bin/jg skill; git -C /testbed rev-parse HEAD; git -C /testbed rev-parse HEAD^{tree}; git -C /testbed status --porcelain'])
+ command(['docker', 'cp', str(out / 'installed-prefix.tar'), check + ':/tmp/prefix.tar'])
+ output = command(['docker', 'start', '-a', check]).stdout
+ expected = ((''.join(value + '\n' for value in VERSIONS.values())).encode() + (out / 'version.txt').read_bytes() + (out / 'skill.md').read_bytes() +
+ (fixed['cell']['expected_image_head'] + '\n' + fixed['cell']['expected_source_tree'] + '\n').encode())
+ if output != expected:
+ raise ValueError('Offline runtime/source/skill preflight differs: ' + registered['task'])
+ finally:
+ subprocess.run(['docker', 'rm', '-f', check], capture_output=True)
+ cell = {**fixed['cell'], 'id': 'installed-' + registered['task'] + '-' + uuid.uuid4().hex[:12],
+ 'arm': 'chunks', 'candidate': str(out / 'installed-prefix.tar')}
+ cells.append({'cell': cell, 'baseline': str(root / registered['baseline']),
+ 'baseline_cost_usd': registered['cost_usd'], 'baseline_resolved': registered['resolved'],
+ 'output': str(out / 'attempts' / registered['task'])})
+ write_json(out / 'agent-inputs.json', [public for _, _, public in pairs])
+ frozen = out / 'runner'
+ frozen.mkdir(mode=0o700)
+ for source in [Path(__file__).resolve(), HERE / 'gateway_broker.py', HERE / 'fixed-baselines.json']:
+ shutil.copyfile(source, frozen / source.name)
+ artifacts = [*frozen.iterdir(), package, out / 'installed-prefix.tar', out / 'skill.md', out / 'agent-inputs.json', root / REGISTRY['dataset']]
+ plan = {'schema': 2, 'status': 'frozen', 'cells': cells,
+ 'purpose': 'One prospective package cohort; reuse fixed baselines without execution.',
+ 'inputs': str(out / 'agent-inputs.json'), 'skill': str(out / 'skill.md'), 'package': str(package),
+ 'runner': str(frozen / 'installed.py'), 'broker': str(frozen / 'gateway_broker.py'),
+ 'registry': str(frozen / 'fixed-baselines.json'), 'dataset': str(root / REGISTRY['dataset']),
+ 'tooling': str(root / REGISTRY['tooling']),
'artifacts': {str(path): digest(path) for path in artifacts}}
write_json(out / 'plan.json', plan)
- print(json.dumps({'status': 'prepared', 'plan': str(out / 'plan.json'), 'cell': cell['id'], 'image': image}))
+ print(json.dumps({'status': 'prepared', 'plan': str(out / 'plan.json'), 'tasks': len(cells)}))
finally:
subprocess.run(['docker', 'rm', '-f', name], capture_output=True)
-def load_plan(path):
+def load_cohort(path):
plan = json.loads(path.read_text())
- if plan.get('status') != 'frozen':
- raise ValueError('Plan must be frozen before execution')
+ if plan.get('schema') != 2 or plan.get('status') != 'frozen':
+ raise ValueError('Expected a frozen prospective cohort plan')
for artifact, expected in plan['artifacts'].items():
if digest(artifact) != expected:
raise ValueError('Frozen artifact changed: ' + artifact)
- required = [Path(__file__).resolve(), HERE / 'gateway_broker.py', Path(plan['inputs']), Path(plan['skill']), Path(plan['package']), Path(plan['cell']['candidate'])]
- if any(str(path) not in plan['artifacts'] for path in required):
- raise ValueError('Plan omits a required artifact')
- base = Path(plan['baseline'])
- if digest(base / 'receipt.json') != plan['baseline_receipt_sha256']:
- raise ValueError('The retained baseline changed')
- baseline, row = load_pair(base, Path(plan['inputs']), plan['cell']['instance_id'])
- for key in ['instance_id', 'engine', 'invocation', 'source_image', 'source_pinned_ref', 'image', 'model', 'effort', 'timeout_seconds', 'gateway_model', 'expected_image_head', 'expected_source_tree']:
- if plan['cell'][key] != baseline['cell'][key]:
- raise ValueError('Treatment pairing changed: ' + key)
- if plan['cell']['arm'] != 'chunks' or plan['cell']['id'] == baseline['cell']['id'] or not re.fullmatch(r'[A-Za-z0-9_.-]+', plan['cell']['id']):
- raise ValueError('Treatment must use its own registered cell')
+ for key in ['runner', 'broker', 'registry', 'inputs', 'skill', 'package', 'dataset']:
+ if plan[key] not in plan['artifacts']:
+ raise ValueError('Plan omits required artifact: ' + key)
+ if digest(__file__) != digest(plan['runner']) or digest(HERE / 'gateway_broker.py') != digest(plan['broker']) or json.loads(Path(plan['registry']).read_text()) != REGISTRY:
+ raise ValueError('Use the frozen runner and registry')
+ prefixes = {item['cell']['candidate'] for item in plan['cells']}
+ identities = {item['cell']['id'] for item in plan['cells']}
+ outputs = {item['output'] for item in plan['cells']}
+ if len(prefixes) != 1 or len(identities) != len(plan['cells']) or len(outputs) != len(plan['cells']):
+ raise ValueError('Cohort requires one prefix and distinct attempt identities')
+ tasks = [item['cell']['instance_id'] for item in plan['cells']]
+ if not tasks or len(set(tasks)) != len(tasks):
+ raise ValueError('Plan tasks must be unique')
+ for item in plan['cells']:
+ cell = item['cell']
+ registered = next(row for row in REGISTRY['tasks'] if row['task'] == cell['instance_id'])
+ baseline, _ = load_pair(Path(item['baseline']), Path(plan['inputs']), cell['instance_id'])
+ expected = {**baseline['cell'], 'id': cell['id'], 'arm': 'chunks', 'candidate': cell['candidate']}
+ if cell != expected or not re.fullmatch(r'installed-[A-Za-z0-9_.-]+', cell['id']):
+ raise ValueError('Treatment pairing changed')
+ if cell['candidate'] not in plan['artifacts'] or item['baseline_cost_usd'] != registered['cost_usd'] or item['baseline_resolved'] != registered['resolved']:
+ raise ValueError('Treatment artifacts or baseline outcome changed')
+ return plan
+
+
+def load_plan(path, task=None):
+ cohort = load_cohort(path)
+ matches = [item for item in cohort['cells'] if task is None or item['cell']['instance_id'] == task]
+ if len(matches) != 1:
+ raise ValueError('Select one frozen task or use --all')
+ plan = {**cohort, **matches[0]}
+ _, row = load_pair(Path(plan['baseline']), Path(plan['inputs']), plan['cell']['instance_id'])
return plan, row
+def existing_attempt(plan):
+ out = Path(plan['output'])
+ if not out.exists():
+ return False
+ path = out / 'receipt.json'
+ if not path.exists():
+ raise ValueError('Existing attempt lacks a lifecycle receipt; inspect it before continuing')
+ receipt = json.loads(path.read_text())
+ if receipt['cell'] != plan['cell']:
+ raise ValueError('Existing attempt belongs to another cell')
+ if receipt['status'] in ('completed', 'failed', 'interrupted'):
+ return True
+ try:
+ os.kill(receipt['host_pid'], 0)
+ alive = True
+ except ProcessLookupError:
+ alive = False
+ active = text(['docker', 'ps', '-q', '--filter', 'label=jg.cell=' + plan['cell']['id']])
+ if alive or active:
+ raise ValueError('Existing attempt is active; wait for its actual process to finish')
+ receipt.update(status='interrupted', interruption='Runner and labeled containers are no longer active')
+ write_json(path, receipt)
+ return True
+
+
def run(args):
- plan, row = load_plan(args.plan.resolve())
+ plan, row = load_plan(args.plan.resolve(), args.task)
cell = plan['cell']
image = check_image(cell['image'])
if args.dry_run:
print(json.dumps({'status': 'validated', 'paid_calls': 0, 'baseline_reused': plan['baseline'], 'image': image, 'cell': cell['id']}))
return
+ if existing_attempt(plan):
+ print(json.dumps({'status': 'retained', 'cell': cell['id']}))
+ return
if not os.environ.get('AI_GATEWAY_API_KEY'):
raise ValueError('AI_GATEWAY_API_KEY is required; load it through the authorized credential workflow')
out = Path(plan['output'])
@@ -187,7 +266,8 @@ def run(args):
tag = 'jg-native-' + uuid.uuid4().hex[:12]
network, proxy = tag + '-net', tag + '-proxy'
receipt = {'engine': 'codex', 'cell': cell, 'image': image, 'plan_sha256': digest(args.plan), 'started': time.time(), 'status': 'preparing',
- 'provider_route': 'vercel-ai-gateway', 'gateway_model': 'openai/gpt-5.6-sol', 'baseline': plan['baseline']}
+ 'provider_route': 'vercel-ai-gateway', 'gateway_model': 'openai/gpt-5.6-sol', 'baseline': plan['baseline'], 'host_pid': os.getpid()}
+ write_json(out / 'receipt.json', receipt)
def dx(argv, user=None):
return command(['docker', 'exec', *(['-u', user] if user else []), tag, *argv])
def put(container, path, data, owner='agent:agent', mode='600'):
@@ -196,9 +276,9 @@ def run(args):
return subprocess.run(['docker', 'cp', remote, str(destination)], capture_output=True).returncode == 0
try:
command(['docker', 'network', 'create', '--internal', network])
- command(['docker', 'run', '-d', '--name', proxy, '-e', 'JEVGREP_TRACE_DIR=/run/jev-traces', image, 'python3', '-u', '-c', (HERE / 'gateway_broker.py').read_text()])
+ command(['docker', 'run', '-d', '--name', proxy, '--label', 'jg.cell=' + cell['id'], '-e', 'JEVGREP_TRACE_DIR=/run/jev-traces', image, 'python3', '-u', '-c', (HERE / 'gateway_broker.py').read_text()])
command(['docker', 'network', 'connect', '--alias', 'model-egress', network, proxy])
- command(['docker', 'run', '-d', '--name', tag, '--network', network,
+ command(['docker', 'run', '-d', '--name', tag, '--label', 'jg.cell=' + cell['id'], '--network', network,
'-e', 'HTTP_PROXY=http://model-egress:3128', '-e', 'HTTPS_PROXY=http://model-egress:3128',
'-e', 'NO_PROXY=localhost,127.0.0.1,model-egress', image, 'sleep', '1800'])
dx(['sh', '-c', 'useradd -m -u 1001 agent; mkdir -p /home/agent/.codex /home/agent/.claude /workspace; chown -R agent:agent /home/agent /workspace'])
@@ -244,6 +324,7 @@ def run(args):
'-e', 'AI_GATEWAY_API_KEY=' + token, '-e', 'AI_GATEWAY_BASE_URL=http://model-egress:3129', '-w', '/testbed', tag, *native]
receipt['status'] = 'running'
receipt['agent_started_at'] = time.time()
+ write_json(out / 'receipt.json', receipt)
start = time.monotonic()
with (out / 'events.jsonl').open('wb') as stdout, (out / 'stderr.txt').open('wb') as stderr:
try:
@@ -283,6 +364,8 @@ def run(args):
receipt['jev_traces_copied'] = capture(proxy + ':/run/jev-traces', out / 'jev-traces')
for container in [tag, proxy]: subprocess.run(['docker', 'rm', '-f', container], capture_output=True)
subprocess.run(['docker', 'network', 'rm', network], capture_output=True)
+ if receipt['status'] in ('preparing', 'running'):
+ receipt['status'] = 'interrupted'
receipt['total_elapsed_seconds'] = time.time() - receipt['started']
write_json(out / 'receipt.json', receipt)
print(json.dumps({'status': receipt['status'], 'output': str(out), 'error_type': receipt.get('error_type')}))
@@ -290,14 +373,19 @@ def run(args):
def grade(args):
- plan, _ = load_plan(args.plan.resolve())
+ plan, _ = load_plan(args.plan.resolve(), args.task)
+ args.tooling = Path(plan['tooling'])
+ args.dataset = Path(plan['dataset'])
out, cell = Path(plan['output']), plan['cell']
receipt = json.loads((out / 'receipt.json').read_text())
- if receipt['plan_sha256'] != digest(args.plan) or receipt['cell'] != cell:
+ if receipt['plan_sha256'] != digest(args.plan) or receipt['cell'] != cell or receipt['status'] not in ('completed', 'failed', 'interrupted'):
raise ValueError('Attempt does not belong to this frozen plan')
run_id = 'jg-' + cell['id']
report = args.tooling.resolve() / 'logs/evaluation' / run_id / 'results.json'
- if report.exists() or (out / 'grading-receipt.json').exists():
+ if (out / 'grading-receipt.json').exists():
+ print(json.dumps({'status': 'retained', 'grading': str(out / 'grading-receipt.json')}))
+ return
+ if report.exists():
raise ValueError('Refusing to reuse an official grading run ID')
harness = args.tooling.resolve() / 'repo'
if text(['git', '-C', str(harness), 'rev-parse', 'HEAD']) != HARNESS_COMMIT or text(['git', '-C', str(harness), 'status', '--porcelain']):
@@ -306,7 +394,7 @@ def grade(args):
if Path(module).parent.parent != harness:
raise ValueError('Python environment does not use the pinned official harness')
prediction = out / 'predictions.jsonl'
- prediction.write_text(json.dumps({'instance_id': cell['instance_id'], 'model_name_or_path': cell['id'], 'model_patch': (out / 'agent.patch').read_text()}) + '\n')
+ prediction.write_text(json.dumps({'instance_id': cell['instance_id'], 'model_name_or_path': cell['id'], 'model_patch': (out / 'agent.patch').read_text() if (out / 'agent.patch').exists() else ''}) + '\n')
start = time.monotonic()
with (out / 'grading-console.log').open('w') as log:
result = subprocess.run([str(args.tooling.resolve() / 'venv/bin/python'), '-m', 'swebench.harness.run_evaluation', '--dataset_name', str(args.dataset.resolve()), '--predictions_path', str(prediction), '--instance_ids', cell['instance_id'], '--max_workers', '1', '--timeout', '1800', '--run_id', run_id], cwd=args.tooling, stdout=log, stderr=subprocess.STDOUT)
@@ -323,9 +411,22 @@ def valid_cost(value):
def account(args):
- plan, _ = load_plan(args.plan.resolve())
+ plan, _ = load_plan(args.plan.resolve(), args.task)
out = Path(plan['output'])
- events = [json.loads(line) for line in (out / 'proxy.jsonl').read_text().splitlines()]
+ receipt = json.loads((out / 'receipt.json').read_text())
+ if receipt['cell'] != plan['cell'] or receipt['plan_sha256'] != digest(args.plan) or receipt['status'] not in ('completed', 'failed', 'interrupted'):
+ raise ValueError('Accounting requires this plan’s terminal attempt')
+ events = []
+ log = out / 'proxy.jsonl'
+ log_valid = log.exists()
+ for line in log.read_text().splitlines() if log.exists() else []:
+ try:
+ event = json.loads(line)
+ if not isinstance(event, dict):
+ raise ValueError('Invalid log event')
+ events.append(event)
+ except ValueError:
+ log_valid = False
starts = [event for event in events if event.get('kind') == 'codex-request-start' and event.get('operation') == 'generation']
request_ids = {event['requestId'] for event in starts}
ends = [event for event in events if event.get('kind') == 'codex-gateway' and event.get('requestId') in request_ids]
@@ -335,7 +436,7 @@ def account(args):
cached = {row['id']: row for row in json.loads(path.read_text())} if path.exists() else {}
lookups = []
for identifier in identifiers:
- if identifier in cached and 'metadata' in cached[identifier]:
+ if identifier in cached and cached[identifier].get('metadata', {}).get('id') == identifier and cached[identifier].get('metadata', {}).get('model') == 'openai/gpt-5.6-sol':
lookups.append(cached[identifier]); continue
if not os.environ.get('AI_GATEWAY_API_KEY'): raise ValueError('Gateway credential required for uncached generation accounting')
request = urllib.request.Request('https://ai-gateway.vercel.sh/v1/generation?' + urllib.parse.urlencode({'id': identifier}), headers={'Authorization': 'Bearer ' + os.environ['AI_GATEWAY_API_KEY']})
@@ -345,39 +446,101 @@ def account(args):
lookups.append({'id': identifier, 'metadata': metadata})
except (OSError, ValueError, KeyError) as error: lookups.append({'id': identifier, 'error_type': type(error).__name__})
write_json(path, lookups)
- complete = bool(identifiers) and len(starts) == len(ends) == len(identifiers) == len(lookups) and request_ids == {event.get('requestId') for event in ends} == {event.get('requestId') for event in observations} and all(event.get('streamTerminal') == 'response.completed' and not event.get('incompleteStream') and not event.get('transportError') and event.get('status') == 200 for event in ends) and all(valid_cost(row.get('metadata', {}).get('total_cost')) for row in lookups)
+ complete = log_valid and bool(identifiers) and len(request_ids) == len(starts) == len(ends) == len(identifiers) == len(lookups) and request_ids == {event.get('requestId') for event in ends} == {event.get('requestId') for event in observations} and all(event.get('streamTerminal') == 'response.completed' and not event.get('incompleteStream') and not event.get('transportError') and event.get('status') == 200 for event in ends) and all(valid_cost(row.get('metadata', {}).get('total_cost')) for row in lookups)
known = sum(row.get('metadata', {}).get('total_cost', 0) for row in lookups if valid_cost(row.get('metadata', {}).get('total_cost')))
result = {'cell': plan['cell']['id'], 'request_starts': len(starts), 'generation_ids': len(identifiers), 'all_requests_accounted': complete, 'known_gateway_cost_usd': known, 'gateway_cost_usd': known if complete else None, 'baseline_cost_usd': plan['baseline_cost_usd'], 'jev_cost_usd': 0}
grading = out / 'grading-receipt.json'
if grading.exists():
- result['official_resolved'] = json.loads(grading.read_text()).get('resolved_instances') == 1
- result['protocol_valid'] = json.loads((out / 'receipt.json').read_text()).get('required_retrieval_observed') is True
+ grade_receipt = json.loads(grading.read_text())
+ result['official_resolved'] = grade_receipt.get('run_id') == 'jg-' + plan['cell']['id'] and grade_receipt.get('exit_code') == 0 and grade_receipt.get('resolved_instances') == 1
+ result['protocol_valid'] = protocol_valid(receipt)
result['successful_cost_win'] = complete and result['official_resolved'] and result['protocol_valid'] and known < plan['baseline_cost_usd']
write_json(out / 'generation-accounting.json', result)
print(json.dumps(result))
+def protocol_valid(receipt):
+ return (receipt.get('status') == 'completed' and receipt.get('exit_code') == 0 and
+ receipt.get('native_completed') is True and receipt.get('required_retrieval_observed') is True and
+ receipt.get('timing_valid') is True)
+
+
+def aggregate(args):
+ plan = load_cohort(args.plan.resolve())
+ rows = []
+ for item in plan['cells']:
+ out = Path(item['output'])
+ def read(name):
+ path = out / name
+ return json.loads(path.read_text()) if path.exists() else {}
+ receipt, grading, billing = read('receipt.json'), read('grading-receipt.json'), read('generation-accounting.json')
+ belongs = receipt.get('cell') == item['cell'] and receipt.get('plan_sha256') == digest(args.plan)
+ terminal = belongs and receipt.get('status') in ('completed', 'failed', 'interrupted')
+ grade_attempted = terminal and grading.get('run_id') == 'jg-' + item['cell']['id'] and type(grading.get('exit_code')) is int
+ graded = grade_attempted and grading['exit_code'] == 0
+ protocol = terminal and protocol_valid(receipt)
+ solved = graded and grading.get('resolved_instances') == 1
+ billed = (terminal and billing.get('cell') == item['cell']['id'] and
+ billing.get('all_requests_accounted') is True and valid_cost(billing.get('gateway_cost_usd')))
+ cost = billing.get('gateway_cost_usd') if billed else None
+ rows.append({'task': item['cell']['instance_id'], 'status': receipt.get('status', 'not-started'),
+ 'terminal': terminal, 'grading_attempted': grade_attempted, 'graded': graded, 'protocol_valid': protocol, 'official_resolved': solved, 'resolved': solved and protocol,
+ 'baseline_resolved': item['baseline_resolved'], 'baseline_cost_usd': item['baseline_cost_usd'],
+ 'fully_billed': billed, 'gateway_cost_usd': cost,
+ 'known_gateway_cost_usd': billing.get('known_gateway_cost_usd') if belongs and billing.get('cell') == item['cell']['id'] and valid_cost(billing.get('known_gateway_cost_usd')) else None,
+ 'successful_cost_win': solved and protocol and billed and cost < item['baseline_cost_usd']})
+ full = {row['task'] for row in rows} == {item['task'] for item in REGISTRY['tasks']}
+ preserved = sum(row['baseline_resolved'] and row['resolved'] and row['protocol_valid'] for row in rows)
+ wins = sum(row['successful_cost_win'] for row in rows)
+ complete = full and all(row['terminal'] and row['grading_attempted'] for row in rows)
+ result = {'prospective_full_cohort': full, 'complete': complete, 'baseline_solves_preserved': preserved,
+ 'fully_billed_solved_cost_wins': wins,
+ 'accepted': complete and preserved == REGISTRY['baseline_solves'] and wins >= REGISTRY['required_cost_wins'],
+ 'known_gateway_subtotal_usd': sum(row['known_gateway_cost_usd'] for row in rows if row['known_gateway_cost_usd'] is not None),
+ 'fully_billed_total_usd': sum(row['gateway_cost_usd'] for row in rows) if all(row['fully_billed'] for row in rows) else None,
+ 'cells': rows}
+ write_json(args.plan.resolve().parent / 'aggregate.json', result)
+ print(json.dumps(result))
+ return result
+
+
def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest='operation', required=True)
prep = sub.add_parser('prepare', help='Install a package in the retained image and freeze a new treatment; no model calls')
prep.add_argument('--package', type=Path, required=True)
prep.add_argument('--output', type=Path, required=True)
- prep.add_argument('--task', default=TASK)
- prep.add_argument('--baseline', type=Path, default=BASELINE)
- prep.add_argument('--inputs', type=Path, default=RUNS / 'auto-research-80/ten-agent-inputs.json')
+ prep.add_argument('--task', default='all', choices=['all', *[item['task'] for item in REGISTRY['tasks']]])
+ prep.add_argument('--evidence-root', type=Path, default=ROOT)
prep.add_argument('--skill', type=Path, default=ROOT / 'skills/jevgrep/SKILL.md')
execute = sub.add_parser('run', help='Run the paid coding-agent treatment once, or validate without calls')
execute.add_argument('--plan', type=Path, required=True)
execute.add_argument('--dry-run', action='store_true')
grading = sub.add_parser('grade', help='Run the official evaluator under a new run ID')
grading.add_argument('--plan', type=Path, required=True)
- grading.add_argument('--dataset', type=Path, default=RUNS / 'auto-research-80/evaluator-only/ten-dataset.json')
- grading.add_argument('--tooling', type=Path, default=TOOLING)
billing = sub.add_parser('account', help='Reconcile full Sol billing; never count unknown charges as zero')
billing.add_argument('--plan', type=Path, required=True)
+ summary = sub.add_parser('aggregate', help='Evaluate the prospective ten-task acceptance rule without model calls')
+ summary.add_argument('--plan', type=Path, required=True)
+ for operation in [execute, grading, billing]:
+ choice = operation.add_mutually_exclusive_group(required=True)
+ choice.add_argument('--task', choices=[item['task'] for item in REGISTRY['tasks']])
+ choice.add_argument('--all', action='store_true')
args = parser.parse_args()
- {'prepare': prepare, 'run': run, 'grade': grade, 'account': account}[args.operation](args)
+ if args.operation in ('prepare', 'aggregate'):
+ {'prepare': prepare, 'aggregate': aggregate}[args.operation](args)
+ return
+ cohort = load_cohort(args.plan.resolve())
+ tasks = [item['cell']['instance_id'] for item in cohort['cells']] if args.all else [args.task]
+ failed = False
+ for task in tasks:
+ args.task = task
+ try:
+ {'run': run, 'grade': grade, 'account': account}[args.operation](args)
+ except SystemExit:
+ failed = True
+ if failed:
+ raise SystemExit(1)
if __name__ == '__main__':
diff --git a/evals/implementation/swebench/test_installed.py b/evals/implementation/swebench/test_installed.py
index 0c1742d..f6b3b8a 100644
--- a/evals/implementation/swebench/test_installed.py
+++ b/evals/implementation/swebench/test_installed.py
@@ -34,27 +34,46 @@ class InstalledTests(unittest.TestCase):
self.assertIn('must run jg for initial repository research', prompt)
self.assertTrue(prompt.endswith(runner.baseline_prompt(row)))
- def fixture(self, root):
- baseline = root / 'baseline'; baseline.mkdir()
- row = {'instance_id':runner.TASK,'repo':'psf/requests','base_commit':'public-base','image':'official-image','problem_statement':'Fix behavior.'}
- inputs = root / 'agent-inputs.json'; runner.write_json(inputs,[row])
- cell = {'id':runner.BASELINE_ID,'instance_id':runner.TASK,'engine':'codex','arm':'baseline','invocation':'required','source_image':'official-image','source_pinned_ref':'official@sha256:retained','image':runner.RUNTIME_IMAGE,'model':'openai/gpt-5.6-sol','effort':'medium','timeout_seconds':900,'gateway_model':'openai/gpt-5.6-sol','expected_image_head':'a'*40,'expected_source_tree':'b'*40}
- (baseline/'prompt.txt').write_text(runner.baseline_prompt(row))
- runner.write_json(baseline/'receipt.json',{'image':runner.RUNTIME_IMAGE,'cell':cell,'versions':runner.VERSIONS,'provider_route':'vercel-ai-gateway','prompt_sha256':runner.digest(baseline/'prompt.txt')})
- runner.write_json(baseline/'grading-receipt.json',{'resolved_instances':1})
+ def fixture(self, root, all_tasks=False):
+ registry = json.loads(json.dumps(runner.REGISTRY))
+ entries = registry['tasks'] if all_tasks else [registry['tasks'][4]]
+ cells, rows = [], []
skill = root/'skill.md'; skill.write_text('installed skill')
package = root/'package.tgz'; package.write_bytes(b'package')
prefix = root/'installed-prefix.tar'; prefix.write_bytes(b'installed prefix')
- paths = [Path(runner.__file__),HERE/'gateway_broker.py',inputs,skill,package,prefix]
- plan = {'status':'frozen','baseline':str(baseline),'baseline_receipt_sha256':runner.digest(baseline/'receipt.json'),'baseline_cost_usd':0.2685004,'cell':{**cell,'id':'installed-new','arm':'chunks','candidate':str(prefix)},'inputs':str(inputs),'skill':str(skill),'package':str(package),'output':str(root/'attempt'),'artifacts':{str(p):runner.digest(p) for p in paths}}
- plan_path=root/'plan.json'; runner.write_json(plan_path,plan)
- return plan_path,plan
+ for index, entry in enumerate(entries):
+ baseline = root / ('baseline-' + str(index)); baseline.mkdir()
+ cell = entry['cell']
+ row = {'instance_id':entry['task'],'repo':'fixture/repo','base_commit':'public-base','image':cell['source_image'],'problem_statement':'Fix behavior.'}
+ rows.append(row)
+ (baseline/'prompt.txt').write_text(runner.baseline_prompt(row))
+ runner.write_json(baseline/'receipt.json',{'image':cell['image'],'cell':cell,'versions':runner.VERSIONS,'provider_route':'vercel-ai-gateway','prompt_sha256':runner.digest(baseline/'prompt.txt')})
+ runner.write_json(baseline/'grading-receipt.json',{'resolved_instances':int(entry['resolved'])})
+ entry['files']={name:runner.digest(baseline/name) for name in ['receipt.json','prompt.txt','grading-receipt.json']}
+ cells.append({'baseline':str(baseline),'baseline_cost_usd':entry['cost_usd'],'baseline_resolved':entry['resolved'],
+ 'cell':{**cell,'id':'installed-' + str(index),'arm':'chunks','candidate':str(prefix)},'output':str(root / ('attempt-' + str(index)))})
+ registry_path = root/'fixed-baselines.json';runner.write_json(registry_path,registry)
+ patcher=patch.object(runner,'REGISTRY',registry);patcher.start();self.addCleanup(patcher.stop)
+ inputs = root / 'agent-inputs.json'; runner.write_json(inputs,rows)
+ dataset=root/'dataset.json';dataset.write_text('fixture dataset')
+ paths = [dataset,Path(runner.__file__),HERE/'gateway_broker.py',registry_path,inputs,skill,package,prefix]
+ plan = {'schema':2,'status':'frozen','cells':cells,'runner':str(Path(runner.__file__)),'broker':str(HERE/'gateway_broker.py'),
+ 'registry':str(registry_path),'tooling':str(root/'tooling'),'dataset':str(root/'dataset.json'),
+ 'inputs':str(inputs),'skill':str(skill),'package':str(package),'artifacts':{str(p):runner.digest(p) for p in paths}}
+ plan_path=root/'plan.json';runner.write_json(plan_path,plan)
+ return plan_path,{**plan,**cells[0]}
+
+ def receipt(self, path, plan, status='completed'):
+ out=Path(plan['output']);out.mkdir(exist_ok=True)
+ runner.write_json(out/'receipt.json',{'plan_sha256':runner.digest(path),'cell':plan['cell'],'status':status,
+ 'required_retrieval_observed':True,'timing_valid':True,'native_completed':True,'exit_code':0,'host_pid':os.getpid()})
+ return out
def test_dry_run_validates_pair_without_credentials_or_agent(self):
with tempfile.TemporaryDirectory() as temporary:
path,plan=self.fixture(Path(temporary))
- with patch.object(runner,'check_image',return_value=runner.RUNTIME_IMAGE),patch.object(runner,'command',side_effect=AssertionError('No agent or paid command allowed')),patch.dict(os.environ,{},clear=True),contextlib.redirect_stdout(io.StringIO()) as stdout:
- runner.run(argparse.Namespace(plan=path,dry_run=True))
+ with patch.object(runner,'check_image',return_value=plan['cell']['image']),patch.object(runner,'command',side_effect=AssertionError('No agent or paid command allowed')),patch.dict(os.environ,{},clear=True),contextlib.redirect_stdout(io.StringIO()) as stdout:
+ runner.run(argparse.Namespace(plan=path,dry_run=True,task=None))
self.assertEqual(json.loads(stdout.getvalue())['paid_calls'],0)
self.assertFalse(Path(plan['output']).exists())
Path(plan['cell']['candidate']).write_bytes(b'changed')
@@ -63,29 +82,84 @@ class InstalledTests(unittest.TestCase):
def test_changed_model_is_not_paired_to_the_old_baseline(self):
with tempfile.TemporaryDirectory() as temporary:
path,plan=self.fixture(Path(temporary))
- plan['cell']['model']='another-model';runner.write_json(path,plan)
+ cohort=json.loads(path.read_text());cohort['cells'][0]['cell']['model']='another-model';runner.write_json(path,cohort)
with self.assertRaisesRegex(ValueError,'pairing changed'):runner.load_plan(path)
def test_grading_rejects_a_changed_official_harness_before_execution(self):
with tempfile.TemporaryDirectory() as temporary:
- root=Path(temporary);path,plan=self.fixture(root);out=Path(plan['output']);out.mkdir()
- runner.write_json(out/'receipt.json',{'plan_sha256':runner.digest(path),'cell':plan['cell']})
+ root=Path(temporary);path,plan=self.fixture(root);self.receipt(path,plan)
with patch.object(runner,'text',return_value='changed-harness'),patch.object(runner.subprocess,'run',side_effect=AssertionError('No grader launched')):
with self.assertRaisesRegex(ValueError,'harness checkout changed'):
- runner.grade(argparse.Namespace(plan=path,tooling=root/'tooling',dataset=root/'dataset.json'))
+ runner.grade(argparse.Namespace(plan=path,tooling=root/'tooling',dataset=root/'dataset.json',task=None))
def test_incomplete_billing_never_counts_missing_charges_as_free(self):
with tempfile.TemporaryDirectory() as temporary:
- path,plan=self.fixture(Path(temporary));out=Path(plan['output']);out.mkdir()
+ path,plan=self.fixture(Path(temporary));out=self.receipt(path,plan)
events=[{'kind':'codex-request-start','operation':'generation','requestId':'one'},{'kind':'codex-generation','requestId':'one','generationId':'gen_one'},{'kind':'codex-gateway','requestId':'one','status':200,'streamTerminal':'response.completed'}, {'kind':'codex-request-start','operation':'generation','requestId':'missing'}]
(out/'proxy.jsonl').write_text('\n'.join(json.dumps(e) for e in events))
- runner.write_json(out/'generation-lookups.json',[{'id':'gen_one','metadata':{'total_cost':0.1}}])
- runner.write_json(out/'grading-receipt.json',{'resolved_instances':1});runner.write_json(out/'receipt.json',{'required_retrieval_observed':True})
- with patch.object(runner.urllib.request,'urlopen',side_effect=AssertionError('No network')),contextlib.redirect_stdout(io.StringIO()):runner.account(argparse.Namespace(plan=path))
+ runner.write_json(out/'generation-lookups.json',[{'id':'gen_one','metadata':{'id':'gen_one','model':'openai/gpt-5.6-sol','total_cost':0.1}}])
+ runner.write_json(out/'grading-receipt.json',{'resolved_instances':1})
+ with patch.object(runner.urllib.request,'urlopen',side_effect=AssertionError('No network')),contextlib.redirect_stdout(io.StringIO()):runner.account(argparse.Namespace(plan=path,task=None))
result=json.loads((out/'generation-accounting.json').read_text())
self.assertEqual(result['known_gateway_cost_usd'],0.1)
self.assertIsNone(result['gateway_cost_usd']);self.assertFalse(result['successful_cost_win'])
+ def test_terminal_failures_are_retained_and_active_attempts_stop(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ path,plan=self.fixture(Path(temporary));out=self.receipt(path,plan,'failed')
+ with patch.object(runner,'text',side_effect=AssertionError('Terminal attempts need no process probe')):
+ self.assertTrue(runner.existing_attempt(plan))
+ self.receipt(path,plan,'running')
+ with patch.object(runner,'text',return_value='active-container'):
+ with self.assertRaisesRegex(ValueError,'active'):runner.existing_attempt(plan)
+ with patch.object(runner.os,'kill',side_effect=ProcessLookupError),patch.object(runner,'text',return_value=''):
+ self.assertTrue(runner.existing_attempt(plan))
+ self.assertEqual(json.loads((out/'receipt.json').read_text())['status'],'interrupted')
+
+ def test_cohort_gate_requires_eight_preserved_and_seven_fully_billed_wins(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ path,plan=self.fixture(Path(temporary),all_tasks=True)
+ for index,item in enumerate(plan['cells']):
+ out=self.receipt(path,item)
+ runner.write_json(out/'grading-receipt.json',{'run_id':'jg-'+item['cell']['id'],'exit_code':0,'resolved_instances':int(item['baseline_resolved'])})
+ runner.write_json(out/'generation-accounting.json',{'cell':item['cell']['id'],'all_requests_accounted':index<7,'gateway_cost_usd':0.01 if index<7 else None,'known_gateway_cost_usd':0.01})
+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))
+ self.assertTrue(result['accepted']);self.assertEqual(result['baseline_solves_preserved'],8)
+ self.assertEqual(result['fully_billed_solved_cost_wins'],7);self.assertIsNone(result['fully_billed_total_usd'])
+ for item in plan['cells'][8:]:
+ self.receipt(path,item,'failed')
+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))
+ self.assertTrue(result['accepted'])
+ self.assertFalse(result['cells'][-1]['protocol_valid'])
+ self.assertAlmostEqual(result['known_gateway_subtotal_usd'],0.1)
+ last=plan['cells'][-1]
+ runner.write_json(Path(last['output'])/'grading-receipt.json',{'run_id':'jg-'+last['cell']['id'],'exit_code':1})
+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))
+ self.assertTrue(result['accepted'])
+ out=Path(plan['cells'][0]['output']);runner.write_json(out/'generation-accounting.json',{'cell':plan['cells'][0]['cell']['id'],'all_requests_accounted':False,'gateway_cost_usd':None,'successful_cost_win':True})
+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))
+ self.assertFalse(result['accepted']);self.assertEqual(result['fully_billed_solved_cost_wins'],6)
+ runner.write_json(out/'grading-receipt.json',{'run_id':'jg-'+plan['cells'][0]['cell']['id'],'exit_code':0,'resolved_instances':0})
+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))
+ self.assertEqual(result['baseline_solves_preserved'],7)
+
+ def test_cohort_cannot_mix_installations_or_reuse_an_attempt_directory(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ path,_=self.fixture(Path(temporary),all_tasks=True)
+ plan=json.loads(path.read_text())
+ original=plan['cells'][1]['cell']['candidate']
+ plan['cells'][1]['cell']['candidate']='different-installation';runner.write_json(path,plan)
+ with self.assertRaisesRegex(ValueError,'one prefix'):runner.load_cohort(path)
+ plan['cells'][1]['cell']['candidate']=original
+ plan['cells'][1]['output']=plan['cells'][0]['output'];runner.write_json(path,plan)
+ with self.assertRaisesRegex(ValueError,'distinct attempt'):runner.load_cohort(path)
+
+ def test_single_task_cannot_claim_full_cohort_acceptance(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ path,_=self.fixture(Path(temporary))
+ with contextlib.redirect_stdout(io.StringIO()):result=runner.aggregate(argparse.Namespace(plan=path))
+ self.assertFalse(result['prospective_full_cohort']);self.assertFalse(result['accepted'])
+
def test_broker_captures_exact_jev_bodies_without_auth_headers(self):
with tempfile.TemporaryDirectory() as temporary:
root=Path(temporary);config=root/'gateway.json';config.write_text(json.dumps({'key':'REAL_KEY_SENTINEL','token':'BROKER_TOKEN_SENTINEL','allow_jev':True,'agent_engine':'codex'}))