OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(evals): keep expected answers hidden and preserve each case’s settings (#1261)
mldangelo-oai9efca3486da4
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.ymlindex edcf86f6f..278daeb72 100644--- a/.github/workflows/node-ci.yml+++ b/.github/workflows/node-ci.yml@@ -422,9 +422,14 @@ jobs:path: reports/python.xmlif-no-files-found: warnretention-days: 14- - name: Build SDK and type-check eval tooling+ - name: Set up Node.js for triage evalsif: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'- run: pnpm --dir sdk/typescript run build:evals+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+ with:+ node-version: "24.15.0"+ - name: Set up triage eval dependencies and host runtime+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'+ run: pnpm --dir evals/triage-finding run setup- name: Test deterministic triage eval contractsif: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'working-directory: evals/triage-findingdiff --git a/evals/secret-discovery/runtime.mts b/evals/secret-discovery/runtime.mtsindex 7a3df7440..db10ba522 100644--- a/evals/secret-discovery/runtime.mts+++ b/evals/secret-discovery/runtime.mts@@ -44,7 +44,7 @@ export async function withEvalState(let root;let home;try {- root = await mkdtemp(join(tmpdir(), "source-audit-"));+ root = await realpath(await mkdtemp(join(tmpdir(), "source-audit-")));// Share saved login updates without importing the caller's configuration.let auth;try {diff --git a/evals/secret-discovery/test_runtime.mts b/evals/secret-discovery/test_runtime.mtsindex 4a0cf5dca..00a3a0f1f 100644--- a/evals/secret-discovery/test_runtime.mts+++ b/evals/secret-discovery/test_runtime.mts@@ -776,3 +776,40 @@ async ({ root, home, signal }) => {},);}++test(+ "fixture roots are canonical when the temporary directory is a symlink",+ unixOnly,+ async (t) => {+ const directory = await mkdtemp(join(tmpdir(), "eval-aliased-temp-"));+ t.after(() => rm(directory, { recursive: true, force: true }));+ const actual = join(directory, "actual");+ const alias = join(directory, "alias");+ await mkdir(actual);+ await symlink(actual, alias);+ childProcess.execFileSync(+ process.execPath,+ [+ "--experimental-strip-types",+ "--input-type=module",+ "--eval",+ `+ import assert from "node:assert/strict";+ import { mkdtemp, realpath } from "node:fs/promises";+ import { tmpdir } from "node:os";+ import { join } from "node:path";+ import { withEvalState } from ${JSON.stringify(runtimeUrl)};+ import { gradeResult } from ${JSON.stringify(new URL("./grade.mts", import.meta.url).href)};+ await withEvalState(async () => await mkdtemp(join(tmpdir(), "home-")), join(tmpdir(), "missing-synthetic-home"), async ({root}) => {+ assert.equal(root, await realpath(root));+ const result = { findings: [], coverage: { completeness: "complete", explicitExclusions: [{ pattern: await realpath(root) }] } };+ const grade = gradeResult(result, { positives: [], files: { "fixture.txt": "synthetic" } }, root);+ assert.ok(grade.errors.includes("incomplete coverage"));+ });+ `,+ ],+ { env: { ...process.env, TMPDIR: alias }, stdio: "pipe" },+ );+ assert.deepEqual(await readdir(actual), []);+ },+);diff --git a/evals/triage-finding/README.md b/evals/triage-finding/README.mdindex ca020a8ca..22c4e4fff 100644--- a/evals/triage-finding/README.md+++ b/evals/triage-finding/README.md@@ -4,7 +4,7 @@ This Promptfoo suite verifies that `$codex-security:triage-finding` accepts theJSON result shape. It also covers bare skill invocation with no supplied finding,which should prompt the user for a finding in a supported format instead of returning triage JSON. GitHub intake cases cover repository-source selection, REST endpoints, and explicit Connector requests without querying live GitHub during the eval.-The suite uses the Promptfoo Codex SDK provider because it only needs final assistant output and deterministic assertions. Codex runs from the checkout's plugin directory and reads its triage skill directly, with ambient plugins and memory disabled. The default suite can also read the synthetic fixtures; calibration cases can read their own hydrated checkout. Dataset labels, assertions, and calibration Git history stay outside those readable roots. The eval directory owns a small pinned pnpm environment so new cases can be added and run without a separate scratch setup.+The suite uses the Promptfoo Codex SDK provider because it only needs final assistant output and deterministic assertions. The common runner stages the checkout's triage skill and policy helper in a throwaway directory, with ambient plugins and memory disabled. The default suite can also read the synthetic fixtures; calibration cases can read their own hydrated checkout. Dataset labels, assertions, and calibration Git history stay outside those readable roots. The eval directory owns a small pinned pnpm environment so new cases can be added and run without a separate scratch setup.Use Node.js 22.22.0 or newer for the eval runner. Run these commands from the repository root to install dependencies under this eval directory.@@ -18,15 +18,16 @@ pnpm --dir evals/triage-finding run setupAfter editing the tooling, rerun `pnpm --dir sdk/typescript run build:evals`. It prepares the SDK and type-checks the eval sources. Node and Promptfoo execute the TypeScript sources directly; package commands enable type stripping for Node 22.13.-Before running model evals, build the checkout's policy helper (requires the [native build prerequisites](../../plugins/codex-security/native/README.md)):+Before running the eval commands, prepare the native policy helper (requires the [native build prerequisites](../../plugins/codex-security/native/README.md)):```bashpnpm --dir plugins/codex-security/mcp-app install --frozen-lockfilenode plugins/codex-security/mcp-app/scripts/build_native.mjs-node plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs --output plugins/codex-security/mcp --native host```-Config validation and deterministic tests do not need this helper build.+The common runner builds the staged helper for validation, evaluation, and replay, then removes its temporary runtime after Promptfoo exits. The runtime extension runs after Promptfoo loads the native Codex provider, preserving its authentication precedence. A pinned Promptfoo patch runs saved `:beforeAll` startup hooks during viewer replay. Replay does not restore grading metadata, so grading hooks remain disabled.++Saved runs whose provider ID is `file://.../triage-provider.mts` need to be rerun with the current configuration before retry, resume, or replay. Their saved results remain intact. Native Codex provider IDs are unchanged.Validate the config:diff --git a/evals/triage-finding/assertions/github-rest-intake.mts b/evals/triage-finding/assertions/github-rest-intake.mtsindex f2714eef4..e57e9277f 100644--- a/evals/triage-finding/assertions/github-rest-intake.mts+++ b/evals/triage-finding/assertions/github-rest-intake.mts@@ -13,8 +13,12 @@ function endpointPattern(path: string, queryParts: string[] = []) {const repository = repositoryName(context.vars.target_repo as string);const paths = [path, path.replace("{owner}/{repo}", repository)];return (- paths.some((candidate) => escapedLiteralPattern(candidate).test(text)) &&- queryParts.every((part) => escapedLiteralPattern(part).test(text))+ paths.some((candidate) =>+ new RegExp(+ escapedLiteralPattern(candidate).source + "\\b(?![/-]|\\.\\w)",+ "i",+ ).test(text),+ ) && queryParts.every((part) => escapedLiteralPattern(part).test(text)));};}diff --git a/evals/triage-finding/assertions/triage-io.mts b/evals/triage-finding/assertions/triage-io.mtsindex c0301d9e2..f66fe2430 100644--- a/evals/triage-finding/assertions/triage-io.mts+++ b/evals/triage-finding/assertions/triage-io.mts@@ -2,24 +2,9 @@ import type { AssertionContext } from "../types.ts";import { extractTriageResult, parseExpected } from "./output.mts";function expectedScanbenchMapping(context: AssertionContext) {- const caseId = String(context.vars.case_id || "");const label = String(context.vars.expected_binary_label || "");const mappings = [];- if (caseId.endsWith("-vulnerable")) {- mappings.push({- expectedVerdict: "confirmed",- expectedBinaryLabel: "positive",- label: "vulnerable scanbench case",- });- } else if (caseId.endsWith("-fixed")) {- mappings.push({- expectedVerdict: "not_actionable",- expectedBinaryLabel: "negative",- label: "fixed scanbench case",- });- }-if (label === "positive") {mappings.push({expectedVerdict: "confirmed",@@ -106,16 +91,15 @@ export default (output: unknown, context: AssertionContext) => {`${label}: not_actionable finding must use null rank and rank_queue`,);}+ } else if (rankQueue !== finding.verdict) {+ failures.push(+ `${label}: rank_queue must match verdict ${finding.verdict}`,+ );} else if (!Number.isInteger(rank) || rank! < 1) {failures.push(`${label}: actionable or unresolved finding must use a positive integer rank`,);} else {- if (rankQueue !== finding.verdict) {- failures.push(- `${label}: rank_queue must match verdict ${finding.verdict}, got ${rankQueue}`,- );- }const ranks = ranksByQueue.get(rankQueue) || [];ranks.push({ label, rank: rank! });ranksByQueue.set(rankQueue, ranks);diff --git a/evals/triage-finding/package.json b/evals/triage-finding/package.jsonindex cf7f80d16..13d637748 100644--- a/evals/triage-finding/package.json+++ b/evals/triage-finding/package.json@@ -3,7 +3,7 @@"private": true,"description": "Promptfoo evals for the Codex Security triage-finding skill","scripts": {- "setup": "pnpm install --frozen-lockfile && pnpm rebuild better-sqlite3 && pnpm --dir ../../sdk/typescript run build:evals",+ "setup": "pnpm install --frozen-lockfile && pnpm rebuild better-sqlite3 && pnpm --dir ../../plugins/codex-security/mcp-app install --frozen-lockfile && pnpm --dir ../../sdk/typescript run build:evals && node ../../plugins/codex-security/mcp-app/scripts/build_native.mjs","calibration:generate": "node --experimental-strip-types scripts/generate-calibration-tests.mts","calibration:hydrate": "node --experimental-strip-types scripts/hydrate-calibration-repos.mts","sastbench:install": "node --experimental-strip-types sastbench/scripts/install-sastbench.mts",@@ -14,20 +14,20 @@"test:calibration-generate": "node --experimental-strip-types scripts/test-generate-calibration-tests.mts","test:calibration-hydrate": "node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts","test:dataset": "node --experimental-strip-types scripts/test-calibration-dataset.mts",- "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-intake-assertions.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-promptfoo-runtime.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts",+ "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-intake-assertions.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-promptfoo-runtime.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts && node --experimental-strip-types --test scripts/test-promptfoo-runner.mts scripts/test-triage-provider.mts","test:instructions": "node --experimental-strip-types scripts/test-app-surface-instructions.mts","test:sastbench": "node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts",- "validate": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./promptfooconfig.yaml",- "validate:calibration": "pnpm run calibration:generate && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./promptfooconfig.calibration.yaml",- "validate:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./sastbench/promptfooconfig.sastbench.yaml",- "validate:sastbench:sample": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./sastbench/promptfooconfig.sastbench-sample.yaml",- "eval": "mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./promptfooconfig.yaml -o ./artifacts/triage-finding-eval.json --no-cache --no-share",- "eval:smoke": "mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./promptfooconfig.yaml -o ./artifacts/triage-finding-eval-smoke.json --no-cache --no-share --filter-range 0:1",- "eval:calibration": "pnpm run calibration:generate && pnpm run calibration:hydrate && mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./promptfooconfig.calibration.yaml -o ./artifacts/triage-finding-calibration-eval.json --no-cache --no-share",- "eval:calibration:smoke": "node --experimental-strip-types scripts/generate-calibration-tests.mts --case oss-dompurify-ghsa-v8jm-5vwx-cfxm --variant vulnerable --output artifacts/calibration-smoke.yaml && node --experimental-strip-types scripts/hydrate-calibration-repos.mts --case oss-dompurify-ghsa-v8jm-5vwx-cfxm --variant vulnerable && mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./promptfooconfig.calibration-smoke.yaml -o ./artifacts/triage-finding-calibration-smoke.json --no-cache --no-share",- "eval:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./sastbench/promptfooconfig.sastbench.yaml --no-cache --no-share",- "eval:sastbench:sample": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./sastbench/promptfooconfig.sastbench-sample.yaml --no-cache --no-share --max-concurrency 32",- "pf:view": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo view"+ "validate": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./promptfooconfig.yaml",+ "validate:calibration": "pnpm run calibration:generate && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./promptfooconfig.calibration.yaml",+ "validate:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./sastbench/promptfooconfig.sastbench.yaml",+ "validate:sastbench:sample": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./sastbench/promptfooconfig.sastbench-sample.yaml",+ "eval": "mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./promptfooconfig.yaml -o ./artifacts/triage-finding-eval.json --no-cache --no-share",+ "eval:smoke": "mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./promptfooconfig.yaml -o ./artifacts/triage-finding-eval-smoke.json --no-cache --no-share --filter-range 0:1",+ "eval:calibration": "pnpm run calibration:generate && pnpm run calibration:hydrate && mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./promptfooconfig.calibration.yaml -o ./artifacts/triage-finding-calibration-eval.json --no-cache --no-share",+ "eval:calibration:smoke": "node --experimental-strip-types scripts/generate-calibration-tests.mts --case oss-dompurify-ghsa-v8jm-5vwx-cfxm --variant vulnerable --output artifacts/calibration-smoke.yaml && node --experimental-strip-types scripts/hydrate-calibration-repos.mts --case oss-dompurify-ghsa-v8jm-5vwx-cfxm --variant vulnerable && mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./promptfooconfig.calibration-smoke.yaml -o ./artifacts/triage-finding-calibration-smoke.json --no-cache --no-share",+ "eval:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./sastbench/promptfooconfig.sastbench.yaml --no-cache --no-share",+ "eval:sastbench:sample": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./sastbench/promptfooconfig.sastbench-sample.yaml --no-cache --no-share --max-concurrency 32",+ "pf:view": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts view"},"dependencies": {"@openai/codex-sdk": "0.162.0",diff --git a/evals/triage-finding/patches/[email protected] b/evals/triage-finding/patches/[email protected]new file mode 100644index 000000000..00e9100a9--- /dev/null+++ b/evals/triage-finding/patches/[email protected]@@ -0,0 +1,24 @@+diff --git a/dist/src/main.js b/dist/src/main.js+index 63b3515f0a43d58b4bfa0407e2749b8803f53399..1ac5c28580050c90ba60d3fc2fad0bdee03ac9e8 100755+--- a/dist/src/main.js++++ b/dist/src/main.js+@@ -7143,6 +7143,7 @@ evalRouter.post("/replay", async (req, res) => {+ label: "Replay"+ }],+ providers: [providerConfig],++ extensions: eval_.config.extensions?.filter((extension) => extension.endsWith(":beforeAll")),+ tests: [{ vars: variables || {} }]+ }, {+ maxConcurrency: 1,+diff --git a/dist/src/server/index.js b/dist/src/server/index.js+index 4d3710840e68d5570ef35011a1ff4cc105e617ee..7681d249b4d9bee02c0347b438fdf971e6909282 100644+--- a/dist/src/server/index.js++++ b/dist/src/server/index.js+@@ -61896,6 +61896,7 @@ evalRouter.post("/replay", async (req, res) => {+ label: "Replay"+ }],+ providers: [providerConfig],++ extensions: eval_.config.extensions?.filter((extension) => extension.endsWith(":beforeAll")),+ tests: [{ vars: variables || {} }]+ }, {+ maxConcurrency: 1,diff --git a/evals/triage-finding/pnpm-lock.yaml b/evals/triage-finding/pnpm-lock.yamlindex de5e32890..a6579821c 100644--- a/evals/triage-finding/pnpm-lock.yaml+++ b/evals/triage-finding/pnpm-lock.yaml@@ -15,6 +15,9 @@ overrides:[email protected]>@simple-git/argv-parser: 2.0.1[email protected]>ip-address: 10.7.2+patchedDependencies:+ [email protected]: 9b31e07e280074178b325d59241d9bfd2cd4a94238bfc5fc9bc07daf43f4d7a8+importers:.:@@ -27,7 +30,7 @@ importers:version: 1.18.34promptfoo:specifier: 0.123.1- version: 0.123.1(@aws-sdk/[email protected])(@aws-sdk/[email protected])(@cfworker/[email protected])(@smithy/[email protected])(@smithy/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])+ version: 0.123.1(patch_hash=9b31e07e280074178b325d59241d9bfd2cd4a94238bfc5fc9bc07daf43f4d7a8)(@aws-sdk/[email protected])(@aws-sdk/[email protected])(@cfworker/[email protected])(@smithy/[email protected])(@smithy/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])packages:@@ -8312,7 +8315,7 @@ snapshots:- [email protected](@aws-sdk/[email protected])(@aws-sdk/[email protected])(@cfworker/[email protected])(@smithy/[email protected])(@smithy/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]):+ [email protected](patch_hash=9b31e07e280074178b325d59241d9bfd2cd4a94238bfc5fc9bc07daf43f4d7a8)(@aws-sdk/[email protected])(@aws-sdk/[email protected])(@cfworker/[email protected])(@smithy/[email protected])(@smithy/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]):dependencies:'@anthropic-ai/sdk': 0.124.0([email protected])'@apidevtools/json-schema-ref-parser': 16.0.3(@types/[email protected])diff --git a/evals/triage-finding/pnpm-workspace.yaml b/evals/triage-finding/pnpm-workspace.yamlindex 0923d1613..4f98c09a5 100644--- a/evals/triage-finding/pnpm-workspace.yaml+++ b/evals/triage-finding/pnpm-workspace.yaml@@ -26,3 +26,6 @@ allowBuilds:onnxruntime-node: falseprotobufjs: falsesharp: false++patchedDependencies:+ [email protected]: patches/[email protected]diff --git a/evals/triage-finding/promptfooconfig.calibration-smoke.yaml b/evals/triage-finding/promptfooconfig.calibration-smoke.yamlindex c0d70238b..a9588c191 100644--- a/evals/triage-finding/promptfooconfig.calibration-smoke.yaml+++ b/evals/triage-finding/promptfooconfig.calibration-smoke.yaml@@ -1,52 +1,4 @@# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json+$ref: ./promptfooconfig.calibration.yamldescription: "Codex Security triage-finding OSS calibration smoke eval"--prompts:- - file://./prompts/triage-request.txt--providers:- - id: openai:codex-sdk:gpt-5.5- label: "Codex SDK triage-finding calibration smoke"- config:- working_dir: "{{triage_runtime_root}}"- skip_git_repo_check: true- approval_policy: never- additional_directories:- - "{{target_repo}}"- - "{{triage_node_root}}"- cli_config:- default_permissions: triage_runtime_only- permissions:- triage_runtime_only:- filesystem:- ":minimal": read- ":workspace_roots": read- network:- enabled: false- features:- memories: false- plugins: false- cli_env:- CODEX_MCP_NODE_PATH: "{{triage_node_path}}"- HTTP_PROXY: "{{env.HTTP_PROXY}}"- HTTPS_PROXY: "{{env.HTTPS_PROXY}}"- ALL_PROXY: "{{env.ALL_PROXY}}"- NO_PROXY: "{{env.NO_PROXY}}"--evaluateOptions:- maxConcurrency: 1--defaultTest:- options:- transformVars: file://./scripts/runtime-vars.mts- vars:- invocation_preamble: "Use $codex-security:triage-finding to triage the supplied existing security finding."- assert:- - type: contains-json- value: file://./schemas/triage-result-v0.schema.json- - type: javascript- value: file://./assertions/triage-io.mts- - type: javascript- value: file://./assertions/calibration-evidence.mts-tests: file://./artifacts/calibration-smoke.yamldiff --git a/evals/triage-finding/promptfooconfig.calibration.yaml b/evals/triage-finding/promptfooconfig.calibration.yamlindex 643fa6da4..230111a88 100644--- a/evals/triage-finding/promptfooconfig.calibration.yaml+++ b/evals/triage-finding/promptfooconfig.calibration.yaml@@ -4,16 +4,22 @@ description: "Codex Security triage-finding OSS calibration eval"prompts:- file://./prompts/triage-request.txt+extensions:+ - file://{{env.TRIAGE_PROVIDER_PATH}}:beforeAll+providers:- id: openai:codex-sdk:gpt-5.5label: "Codex SDK triage-finding calibration"config:- working_dir: "{{triage_runtime_root}}"skip_git_repo_check: trueapproval_policy: never+ cli_env:+ HTTP_PROXY: "{{env.HTTP_PROXY}}"+ HTTPS_PROXY: "{{env.HTTPS_PROXY}}"+ ALL_PROXY: "{{env.ALL_PROXY}}"+ NO_PROXY: "{{env.NO_PROXY}}"additional_directories:- - "{{target_repo}}"- - "{{triage_node_root}}"+ - "{% if calibration_repo_root %}{{calibration_repo_root}}{% else %}{{env.TRIAGE_CALIBRATION_ROOT}}{% endif %}/{{calibration_repo}}"cli_config:default_permissions: triage_runtime_onlypermissions:@@ -26,12 +32,6 @@ providers:features:memories: falseplugins: false- cli_env:- CODEX_MCP_NODE_PATH: "{{triage_node_path}}"- HTTP_PROXY: "{{env.HTTP_PROXY}}"- HTTPS_PROXY: "{{env.HTTPS_PROXY}}"- ALL_PROXY: "{{env.ALL_PROXY}}"- NO_PROXY: "{{env.NO_PROXY}}"evaluateOptions:maxConcurrency: 1diff --git a/evals/triage-finding/promptfooconfig.yaml b/evals/triage-finding/promptfooconfig.yamlindex 35be4ad6f..70d4ac7f1 100644--- a/evals/triage-finding/promptfooconfig.yaml+++ b/evals/triage-finding/promptfooconfig.yaml@@ -4,16 +4,20 @@ description: "Codex Security triage-finding input-type I/O eval"prompts:- file://./prompts/triage-request.txt+extensions:+ - file://{{env.TRIAGE_PROVIDER_PATH}}:beforeAll+providers:- id: openai:codex-sdk:gpt-5.5label: "Codex SDK triage-finding"config:- working_dir: "{{triage_runtime_root}}"skip_git_repo_check: trueapproval_policy: never- additional_directories:- - "{{triage_fixture_root}}"- - "{{triage_node_root}}"+ cli_env:+ HTTP_PROXY: "{{env.HTTP_PROXY}}"+ HTTPS_PROXY: "{{env.HTTPS_PROXY}}"+ ALL_PROXY: "{{env.ALL_PROXY}}"+ NO_PROXY: "{{env.NO_PROXY}}"cli_config:default_permissions: triage_runtime_onlypermissions:@@ -26,19 +30,11 @@ providers:features:memories: falseplugins: false- cli_env:- CODEX_MCP_NODE_PATH: "{{triage_node_path}}"- HTTP_PROXY: "{{env.HTTP_PROXY}}"- HTTPS_PROXY: "{{env.HTTPS_PROXY}}"- ALL_PROXY: "{{env.ALL_PROXY}}"- NO_PROXY: "{{env.NO_PROXY}}"evaluateOptions:maxConcurrency: 1defaultTest:- options:- transformVars: file://./scripts/runtime-vars.mtsvars:invocation_preamble: "Use $codex-security:triage-finding to triage the supplied existing security findings."eval_instructions: |-diff --git a/evals/triage-finding/prompts/triage-request.txt b/evals/triage-finding/prompts/triage-request.txtindex 22a7fc2fe..722a1e9b6 100644--- a/evals/triage-finding/prompts/triage-request.txt+++ b/evals/triage-finding/prompts/triage-request.txt@@ -1,4 +1,4 @@-Read and follow skills/triage-finding/SKILL.md from this checkout for the following request.+Read and follow plugins/codex-security/skills/triage-finding/SKILL.md from this checkout.{{invocation_preamble}}diff --git a/evals/triage-finding/sastbench/DESIGN.md b/evals/triage-finding/sastbench/DESIGN.mdindex 75fbcbd62..67304b8f3 100644--- a/evals/triage-finding/sastbench/DESIGN.md+++ b/evals/triage-finding/sastbench/DESIGN.md@@ -24,7 +24,7 @@ The SastBench adapter owns only benchmark-specific behavior:The model-visible prompt contains the affected checkout, an opaque case ID, and the record's `to_analyzer` data. Ground truth is stored in Promptfoo test metadata. The upstream `finding_id`, `ground_truth`, `metadata.source`, and dataset class statistics are not prompt variables.-The Codex provider runs from the checkout's plugin directory. Its permission profile also grants reads to the selected target, its label-free Git cache, the Node runtime, and Codex's minimal runtime paths; the label-bearing dataset and Promptfoo harness stay denied.+The Codex provider runs from the common runner's throwaway skill directory. Its permission profile also grants reads to the hydrated target repositories, its label-free Git cache, the Node runtime, and Codex's minimal runtime paths; the label-bearing dataset and Promptfoo harness stay denied.## Verdict mappingdiff --git a/evals/triage-finding/sastbench/README.md b/evals/triage-finding/sastbench/README.mdindex fdcfe1bcc..77ca5613a 100644--- a/evals/triage-finding/sastbench/README.md+++ b/evals/triage-finding/sastbench/README.md@@ -2,7 +2,7 @@This Promptfoo lane measures how accurately the source version of `plugins/codex-security/skills/triage-finding/SKILL.md` triages supplied scanner findings from public SastBench v0.1. It does not measure repository-wide vulnerability discovery.-SastBench contains 2,737 findings: 299 labeled true positive and 2,438 labeled false positive. The JavaScript test generator exposes the scanner claim and the exact affected checkout to Codex while keeping SastBench's label-bearing `finding_id` and ground-truth label out of the rendered prompt. Codex reads the skill directly from the checkout's plugin directory. Its permission profile also allows the selected target, the label-free Git cache, and the Node runtime; the label-bearing eval harness remains outside those roots.+SastBench contains 2,737 findings: 299 labeled true positive and 2,438 labeled false positive. The JavaScript test generator exposes the scanner claim and the exact affected checkout to Codex while keeping SastBench's label-bearing `finding_id` and ground-truth label out of the rendered prompt. The common eval runner stages the checkout skill and policy helper in a throwaway directory. Its permission profile also allows the hydrated target repositories, the label-free Git cache, and the Node runtime; the label-bearing eval harness remains outside those roots.Run commands from the repository root. The parent `evals/triage-finding/` directory owns the pinned Promptfoo and Codex SDK dependencies and the ignored `artifacts/` tree.@@ -54,7 +54,7 @@ pnpm --dir evals/triage-finding run eval:sastbench:sample \`eval:sastbench:sample` always passes `--no-cache --no-share` and defaults to 32 concurrent cases, matching the completed full benchmark's proven setting. The prior case latencies imply roughly a 35-minute run, although provider capacity and changes to the skill can change that duration. If capacity errors occur, use Promptfoo's `--retry-errors` mode at lower concurrency so successful rows stay in the same evaluation.-The `representative-100-v1` selection is deterministic. It allocates each class proportionally across scanner evidence-size buckets (`1`, `2-3`, `4-10`, and `11+` locations), then uses seeded systematic sampling ordered by repository, CWE, language, and original case index. It preserves the original opaque case IDs so baseline and candidate skill runs compare the same findings.+The `representative-100-v1` selection retains its established `en-US` collation for paired baseline comparisons. It allocates each class proportionally across scanner evidence-size buckets (`1`, `2-3`, `4-10`, and `11+` locations), then uses seeded systematic sampling ordered by repository, CWE, language, and original case index. It preserves the original opaque case IDs so baseline and candidate skill runs compare the same findings.Because positives are deliberately oversampled, each test's metadata includes a population weight. The existing SastBench metrics extension applies that weight to its additive counters, so aggregate derived metrics estimate the full 2,737-case benchmark. Per-row Promptfoo pass/fail counts and the `schema_valid`/`strict_case_correct` assertions still describe the 100 executed rows.diff --git a/evals/triage-finding/sastbench/promptfooconfig.sastbench.yaml b/evals/triage-finding/sastbench/promptfooconfig.sastbench.yamlindex 295915146..31dbda0f3 100644--- a/evals/triage-finding/sastbench/promptfooconfig.sastbench.yaml+++ b/evals/triage-finding/sastbench/promptfooconfig.sastbench.yaml@@ -10,13 +10,12 @@ providers:- id: openai:codex-sdk:gpt-5.5label: "Codex SDK triage-finding SastBench (gpt-5.5)"config:- # The plugin checkout excludes the label-bearing dataset and eval harness.- working_dir: "{{triage_runtime_root}}"- # Read the target, its label-free Git cache, and the helper runtime.+ # The provider binds working_dir to the current staged skill runtime.+ # The selected target worktree and its label-free Git cache are the only+ # additional readable workspace roots.additional_directories:+ - "{{env.SASTBENCH_GIT_CACHE_ROOT}}"- "{{target_repo}}"- - "{{sastbench_git_cache_root}}"- - "{{triage_node_root}}"skip_git_repo_check: trueapproval_policy: never# Pin a value supported by the eval's Codex SDK instead of inheriting a@@ -35,20 +34,17 @@ providers:":workspace_roots": readnetwork:enabled: false- # The source skill is loaded explicitly from the checkout.+ # The source skill is loaded explicitly from the staged runtime.# Global plugin and memory state would add machine-specific context.features:memories: falseplugins: false- cli_env:- CODEX_MCP_NODE_PATH: "{{triage_node_path}}"extensions:+ - file://{{env.TRIAGE_PROVIDER_PATH}}:beforeAll- file://./assertions/sastbench-metrics.mts:afterEachdefaultTest:- options:- transformVars: process.mainModule.require(process.cwd() + '/scripts/runtime-vars.mts').default(vars)assert:- type: contains-jsonvalue: file://../schemas/triage-result-v0.schema.jsondiff --git a/evals/triage-finding/sastbench/prompts/sastbench-triage-request.txt b/evals/triage-finding/sastbench/prompts/sastbench-triage-request.txtindex c33ce67fe..88cf6eb7d 100644--- a/evals/triage-finding/sastbench/prompts/sastbench-triage-request.txt+++ b/evals/triage-finding/sastbench/prompts/sastbench-triage-request.txt@@ -1,4 +1,4 @@-Read and follow skills/triage-finding/SKILL.md to triage the supplied scanner finding.+Read and follow plugins/codex-security/skills/triage-finding/SKILL.md to triage the supplied scanner finding.Repository checkout to inspect:{{target_repo}}diff --git a/evals/triage-finding/sastbench/scripts/install-sastbench.mts b/evals/triage-finding/sastbench/scripts/install-sastbench.mtsindex 0c6c7be94..79a36fe5d 100644--- a/evals/triage-finding/sastbench/scripts/install-sastbench.mts+++ b/evals/triage-finding/sastbench/scripts/install-sastbench.mts@@ -87,7 +87,6 @@ export function verifyInstallation(}function createInstallation(target: string) {- fs.mkdirSync(target, { recursive: true });if (fs.readdirSync(target).length !== 0) {throw new Error(`Refusing to install into non-empty directory: ${target}`);}@@ -101,7 +100,13 @@ export function installSastBench(target: string = DEFAULT_INSTALL_ROOT) {const resolvedTarget = path.resolve(target);if (!fs.existsSync(resolvedTarget)) {fs.mkdirSync(path.dirname(resolvedTarget), { recursive: true });- createInstallation(resolvedTarget);+ fs.mkdirSync(resolvedTarget);+ try {+ createInstallation(resolvedTarget);+ } catch (error) {+ fs.rmSync(resolvedTarget, { recursive: true, force: true });+ throw error;+ }}return verifyInstallation(inspectInstallation(resolvedTarget));}diff --git a/evals/triage-finding/sastbench/scripts/sastbench-lib.mts b/evals/triage-finding/sastbench/scripts/sastbench-lib.mtsindex d984191df..efa978086 100644--- a/evals/triage-finding/sastbench/scripts/sastbench-lib.mts+++ b/evals/triage-finding/sastbench/scripts/sastbench-lib.mts@@ -315,7 +315,7 @@ function allocateSampleCounts(.filter((stratum) => stratum.count < stratum.entries.length).sort((left, right) =>- deficit(right) - deficit(left) || left.name.localeCompare(right.name),+ deficit(right) - deficit(left) || compareText(left.name, right.name),);candidates[0].count += 1;remaining -= 1;@@ -323,6 +323,10 @@ function allocateSampleCounts(return allocations;}+function compareText(left: string, right: string) {+ return left.localeCompare(right, "en-US");+}+function representativeSortKey(entry: SampleEntry) {const record = entry.record;return [@@ -344,7 +348,7 @@ function deterministicSystematicSample(return [...entries];}const sortedEntries = [...entries].sort((left, right) =>- representativeSortKey(left).localeCompare(representativeSortKey(right)),+ compareText(representativeSortKey(left), representativeSortKey(right)),);const interval = sortedEntries.length / sampleCount;const seedValue = hash(@@ -394,7 +398,7 @@ export function selectRepresentativeSample();const strata = [...entriesByBucket.entries()].map(([bucket, entries]) => ({ name: `${label}:${bucket}`, entries }))- .sort((left, right) => left.name.localeCompare(right.name));+ .sort((left, right) => compareText(left.name, right.name));const allocations = allocateSampleCounts(strata, requestedCount);for (const stratum of allocations) {diff --git a/evals/triage-finding/sastbench/scripts/test-install-sastbench.mts b/evals/triage-finding/sastbench/scripts/test-install-sastbench.mtsindex 1d2ae4388..65407e22a 100644--- a/evals/triage-finding/sastbench/scripts/test-install-sastbench.mts+++ b/evals/triage-finding/sastbench/scripts/test-install-sastbench.mts@@ -1,6 +1,7 @@#!/usr/bin/env nodeimport assert from "node:assert/strict";+import { syncBuiltinESMExports } from "node:module";import { verifyInstallation } from "./install-sastbench.mts";import {EXPECTED_CASE_COUNT,@@ -44,3 +45,71 @@ assert.throws();console.log("sastbench installer verification tests passed");++import fs from "node:fs";+import path from "node:path";+import os from "node:os";+import childProcess from "node:child_process";+import { installSastBench } from "./install-sastbench.mts";+const root = fs.mkdtempSync(path.join(os.tmpdir(), "sastbench-install-"));+const target = path.join(root, "checkout");+const originalExec = childProcess.execFileSync;+let fetches = 0;+try {+ childProcess.execFileSync = ((+ _command: string,+ args: string[],+ options: { cwd: string },+ ) => {+ if (args.includes("init")) fs.mkdirSync(path.join(options.cwd, ".git"));+ if (args.includes("fetch")) {+ fetches++;+ throw new Error("synthetic fetch failed");+ }+ return "";+ }) as typeof originalExec;+ syncBuiltinESMExports();+ for (let attempt = 0; attempt < 2; attempt++) {+ assert.throws(() => installSastBench(target), /synthetic fetch failed/);+ assert.equal(fs.existsSync(target), false);+ }+ assert.equal(+ fetches,+ 2,+ "a retry must fetch again instead of inspecting an incomplete checkout",+ );+ fs.mkdirSync(target);+ fs.writeFileSync(path.join(target, "keep.txt"), "existing work");+ assert.throws(() => installSastBench(target), /not a Git checkout/);+ assert.equal(+ fs.readFileSync(path.join(target, "keep.txt"), "utf8"),+ "existing work",+ );+} finally {+ childProcess.execFileSync = originalExec;+ syncBuiltinESMExports();+ fs.rmSync(root, { recursive: true, force: true });+}++const raceRoot = fs.mkdtempSync(path.join(os.tmpdir(), "sastbench-race-"));+const raceTarget = path.join(raceRoot, "checkout");+const competingFile = path.join(raceTarget, "other-install.txt");+const originalExists = fs.existsSync;+try {+ fs.existsSync = (candidate) => {+ if (candidate === raceTarget && !originalExists(candidate)) {+ fs.mkdirSync(raceTarget);+ fs.writeFileSync(competingFile, "concurrent installation");+ return false;+ }+ return originalExists(candidate);+ };+ assert.throws(() => installSastBench(raceTarget), { code: "EEXIST" });+ assert.equal(+ fs.readFileSync(competingFile, "utf8"),+ "concurrent installation",+ );+} finally {+ fs.existsSync = originalExists;+ fs.rmSync(raceRoot, { recursive: true, force: true });+}diff --git a/evals/triage-finding/sastbench/scripts/test-promptfoo-native-harness.mts b/evals/triage-finding/sastbench/scripts/test-promptfoo-native-harness.mtsindex d077d2ad6..97f4ef377 100644--- a/evals/triage-finding/sastbench/scripts/test-promptfoo-native-harness.mts+++ b/evals/triage-finding/sastbench/scripts/test-promptfoo-native-harness.mts@@ -1,5 +1,8 @@#!/usr/bin/env nodeimport assert from "node:assert/strict";+import { execFileSync } from "node:child_process";+import os from "node:os";+import { pathToFileURL } from "node:url";import fs from "node:fs";import path from "node:path";@@ -17,7 +20,7 @@ const packageJson = JSON.parse(fs.readFileSync(path.join(evalRoot, "package.json"), "utf8"),);-assert.match(config, /working_dir:\s+"\{\{triage_runtime_root\}\}"/);+assert.doesNotMatch(config, /working_dir:/);assert.match(config, /skip_git_repo_check:\s+true/);assert.doesNotMatch(config,@@ -25,7 +28,7 @@ assert.doesNotMatch();assert.match(config,- /additional_directories:\n\s+- "\{\{target_repo\}\}"\n\s+- "\{\{sastbench_git_cache_root\}\}"/,+ /additional_directories:\n\s+- "\{\{env\.SASTBENCH_GIT_CACHE_ROOT\}\}"\n\s+- "\{\{target_repo\}\}"/,);assert.doesNotMatch(config, /sandbox_mode:/);assert.doesNotMatch(config, /network_access_enabled:/);@@ -48,7 +51,7 @@ assert.match(config, /metric:\s+schema_valid/);assert.match(config, /metric:\s+strict_case_correct/);assert.match(config,- /extensions:\n\s+- file:\/\/\.\/assertions\/sastbench-metrics\.mts:afterEach/,+ /extensions:\n\s+- file:\/\/\{\{env\.TRIAGE_PROVIDER_PATH\}\}:beforeAll\n\s+- file:\/\/\.\/assertions\/sastbench-metrics\.mts:afterEach/,);for (const metric of ["strict_precision",@@ -86,7 +89,7 @@ assert.match(assert.doesNotMatch(sampleConfig, /providers:/);assert.doesNotMatch(sampleConfig, /derivedMetrics:/);-assert.match(packageJson.scripts["eval:sastbench"], /promptfoo eval/);+assert.match(packageJson.scripts["eval:sastbench"], /run-promptfoo\.mts eval/);assert.match(packageJson.scripts["eval:sastbench"], /--no-cache/);assert.match(packageJson.scripts["eval:sastbench"], /--no-share/);assert.doesNotMatch(packageJson.scripts["eval:sastbench"], /--filter-range/);@@ -99,11 +102,11 @@ assert.doesNotMatch();assert.match(packageJson.scripts["validate:sastbench:sample"],- /promptfoo validate config -c .\/sastbench\/promptfooconfig\.sastbench-sample\.yaml/,+ /run-promptfoo\.mts validate config -c .\/sastbench\/promptfooconfig\.sastbench-sample\.yaml/,);assert.match(packageJson.scripts["eval:sastbench:sample"],- /promptfoo eval -c .\/sastbench\/promptfooconfig\.sastbench-sample\.yaml/,+ /run-promptfoo\.mts eval -c .\/sastbench\/promptfooconfig\.sastbench-sample\.yaml/,);assert.match(packageJson.scripts["eval:sastbench:sample"], /--no-cache/);assert.match(packageJson.scripts["eval:sastbench:sample"], /--no-share/);@@ -128,3 +131,44 @@ assert.equal();console.log("sastbench native Promptfoo harness tests passed");++const dispatchRoot = fs.mkdtempSync(+ path.join(os.tmpdir(), "sastbench-dispatch-"),+);+try {+ const runner = path.join(evalRoot, "scripts", "run-promptfoo.mts");+ for (const module of [runner]) {+ for (const entry of ["-", path.join(dispatchRoot, "virtual-entry.mts")]) {+ const imported = execFileSync(+ process.execPath,+ [+ "--experimental-strip-types",+ "--input-type=module",+ "--eval",+ `process.argv[1] = ${JSON.stringify(entry)}; await import(${JSON.stringify(pathToFileURL(module).href)}); console.log("imported");`,+ ],+ { encoding: "utf8" },+ );+ assert.equal(imported.trim(), "imported");+ }+ }+ if (process.platform !== "win32") {+ const link = path.join(dispatchRoot, "sastbench-runner.mts");+ fs.symlinkSync(runner, link);+ const version = execFileSync(+ process.execPath,+ ["--experimental-strip-types", link, "--version"],+ {+ encoding: "utf8",+ env: {+ ...process.env,+ PROMPTFOO_DISABLE_TELEMETRY: "1",+ PROMPTFOO_DISABLE_UPDATE: "1",+ },+ },+ );+ assert.match(version, /0\.123\.1/);+ }+} finally {+ fs.rmSync(dispatchRoot, { recursive: true, force: true });+}diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mtsindex 8cdeedce8..fb124b869 100644--- a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts@@ -196,6 +196,41 @@ const sampleSpec = {seed: "test-seed",labelCounts: { true_positive: 4, false_positive: 6 },};+// Keep the established v1 collation when comparing paired sample runs.+const collationRepositories = [+ "example/a-b",+ "example/a_b",+ "example/A",+ "example/a",+ "example/Z",+ "example/z",+];+const collationDataset = Array.from({ length: 48 }, (_, index) =>+ benchmarkRecord({+ repo_name: collationRepositories[index % collationRepositories.length],+ ground_truth: index < 24 ? "true_positive" : "false_positive",+ metadata: {+ source: "synthetic",+ cwe_id: "CWE-79",+ languages: ["JavaScript"],+ },+ to_analyzer: {+ ...positive.to_analyzer,+ locations: [+ { ...positive.to_analyzer.locations[0], file: `src/file-${index}.js` },+ ],+ },+ }),+);+assert.deepEqual(+ selectRepresentativeSample(collationDataset, {+ profile: "test-v1-collation",+ seed: "test-v1-collation",+ labelCounts: { true_positive: 6, false_positive: 6 },+ }).map(({ index }) => index),+ [0, 1, 10, 14, 21, 22, 32, 34, 36, 37, 44, 46],+);+const selectedSample = selectRepresentativeSample(sampleDataset, sampleSpec);const selectedAgain = selectRepresentativeSample(sampleDataset, sampleSpec);@@ -283,3 +318,25 @@ assert.throws();console.log("sastbench dataset and prompt tests passed");++// A locale may reorder accents and punctuation; that must not change the sample.+import { execFileSync } from "node:child_process";+const localeRecords = ["true_positive", "false_positive"].flatMap(+ (ground_truth) =>+ ["a", "ä", "z", "Z", "_", "å"].map((repo_name) =>+ benchmarkRecord({ ground_truth, repo_name }),+ ),+);+const sampleProgram = `import { selectRepresentativeSample } from ${JSON.stringify(new URL("./sastbench-lib.mts", import.meta.url).href)}; console.log(JSON.stringify(selectRepresentativeSample(${JSON.stringify(localeRecords)}, {profile:'test',seed:'test',labelCounts:{true_positive:2,false_positive:2}}).map(entry => entry.index)));`;+const localeSample = (locale: string) =>+ execFileSync(+ process.execPath,+ [+ "--experimental-strip-types",+ "--input-type=module",+ "--eval",+ sampleProgram,+ ],+ { encoding: "utf8", env: { ...process.env, LANG: locale, LC_ALL: locale } },+ );+assert.equal(localeSample("en_US.UTF-8"), localeSample("sv_SE.UTF-8"));diff --git a/evals/triage-finding/scripts/generate-calibration-tests.mts b/evals/triage-finding/scripts/generate-calibration-tests.mtsindex 33ae3f570..33bb8af51 100644--- a/evals/triage-finding/scripts/generate-calibration-tests.mts+++ b/evals/triage-finding/scripts/generate-calibration-tests.mts@@ -170,7 +170,11 @@ export function selectedVariants(return variants;}-if (import.meta.filename === fs.realpathSync(process.argv[1])) {+if (+ process.argv[1] &&+ fs.existsSync(process.argv[1]) &&+ import.meta.filename === fs.realpathSync(process.argv[1])+) {const args = parseArgs(process.argv.slice(2));const dataset = JSON.parse(fs.readFileSync(args.dataset, "utf8"));const variants = selectedVariants(dataset, args);diff --git a/evals/triage-finding/scripts/hydrate-calibration-repos.mts b/evals/triage-finding/scripts/hydrate-calibration-repos.mtsindex f69bd63cd..adb000535 100644--- a/evals/triage-finding/scripts/hydrate-calibration-repos.mts+++ b/evals/triage-finding/scripts/hydrate-calibration-repos.mts@@ -111,7 +111,11 @@ export function plannedJobs(}));}-if (import.meta.filename === fs.realpathSync(process.argv[1])) {+if (+ process.argv[1] &&+ fs.existsSync(process.argv[1]) &&+ import.meta.filename === fs.realpathSync(process.argv[1])+) {const args = parseArgs(process.argv.slice(2));const dataset = JSON.parse(fs.readFileSync(args.dataset, "utf8"));const jobs = plannedJobs(dataset, args);diff --git a/evals/triage-finding/scripts/run-promptfoo.mts b/evals/triage-finding/scripts/run-promptfoo.mtsnew file mode 100644index 000000000..a6ec4a36b--- /dev/null+++ b/evals/triage-finding/scripts/run-promptfoo.mts@@ -0,0 +1,171 @@+#!/usr/bin/env node+import childProcess, { type ChildProcess } from "node:child_process";+import fs from "node:fs";+import os from "node:os";+import path from "node:path";++const EVAL_ROOT = path.resolve(import.meta.dirname, "..");+const PLUGIN_ROOT = path.resolve(+ EVAL_ROOT,+ "..",+ "..",+ "plugins",+ "codex-security",+);+const TRIAGE_SKILL_ROOT = path.join(PLUGIN_ROOT, "skills", "triage-finding");+const PROMPTFOO_ENTRYPOINT = path.join(+ EVAL_ROOT,+ "node_modules",+ "promptfoo",+ "dist",+ "src",+ "entrypoint.js",+);++function copyDirectory(+ sourceRoot: string,+ targetRoot: string,+ excludedNames = new Set<string>(),+) {+ fs.cpSync(sourceRoot, targetRoot, {+ recursive: true,+ filter: (source) => !excludedNames.has(path.basename(source)),+ });+}++/**+ * Give Codex a throwaway working directory that contains only the skill files+ * it needs. The label-bearing dataset and Promptfoo harness stay in EVAL_ROOT.+ */+function stageSkillRuntime() {+ const runtimeRoot = fs.mkdtempSync(+ path.join(os.tmpdir(), "codex-security-triage-finding-"),+ );+ const stagedPluginRoot = path.join(runtimeRoot, "plugins", "codex-security");+ copyDirectory(+ TRIAGE_SKILL_ROOT,+ path.join(stagedPluginRoot, "skills", "triage-finding"),+ new Set(["evals"]),+ );+ for (const sharedDirectory of ["references", "schemas"]) {+ copyDirectory(+ path.join(PLUGIN_ROOT, sharedDirectory),+ path.join(stagedPluginRoot, sharedDirectory),+ );+ }+ fs.mkdirSync(path.join(stagedPluginRoot, "scripts"));+ for (const launcher of [+ "launch_codex_security_mcp",+ "launch_codex_security_mcp.cmd",+ ]) {+ fs.copyFileSync(+ path.join(PLUGIN_ROOT, "scripts", launcher),+ path.join(stagedPluginRoot, "scripts", launcher),+ );+ }+ copyDirectory(+ path.join(EVAL_ROOT, "fixtures"),+ path.join(runtimeRoot, "evals", "triage-finding", "fixtures"),+ );+ return fs.realpathSync(runtimeRoot);+}++async function runPromptfoo(+ promptfooArgs: string[],+ environment: NodeJS.ProcessEnv = {},+) {+ const runtimeRoot = stageSkillRuntime();+ const artifacts = path.join(EVAL_ROOT, "artifacts");+ const env = {+ TRIAGE_CALIBRATION_ROOT: path.join(artifacts, "calibration-repos"),+ SASTBENCH_GIT_CACHE_ROOT: path.join(artifacts, "sastbench-git-cache"),+ ...process.env,+ ...environment,+ // Promptfoo persists this stable harness path for retry, resume, and viewer replay.+ TRIAGE_PROVIDER_PATH: path.join(import.meta.dirname, "triage-provider.mts"),+ TRIAGE_RUNTIME_ROOT: runtimeRoot,+ };+ let child: ChildProcess | undefined;+ let interrupted: NodeJS.Signals | undefined;+ const handlers = (["SIGINT", "SIGTERM"] as const).map((signal) => {+ const handler = () => {+ interrupted = signal;+ if (!child?.pid) return;+ if (process.platform === "win32") {+ // Console Ctrl+C already reaches the inherited child. Node's kill()+ // would terminate it before Promptfoo can save its progress.+ if (signal !== "SIGINT") child.kill(signal);+ } else {+ try {+ process.kill(-child.pid, signal);+ } catch (error) {+ // The process group can exit before Node emits the close event.+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;+ }+ }+ };+ process.on(signal, handler);+ return [signal, handler] as const;+ });+ function runChild(args: string[], cwd: string) {+ return new Promise<number>((resolve, reject) => {+ child = childProcess.spawn(process.execPath, args, {+ cwd,+ env,+ stdio: "inherit",+ detached: process.platform !== "win32",+ });+ child.once("error", reject);+ child.once("close", (code, signal) => {+ child = undefined;+ const stoppedBy = interrupted || signal;+ resolve(+ stoppedBy === "SIGINT"+ ? 130+ : stoppedBy === "SIGTERM"+ ? 143+ : (code ?? 1),+ );+ });+ });+ }+ try {+ const built = await runChild(+ [+ path.join(PLUGIN_ROOT, "mcp-app", "scripts", "build_mcp_app.mjs"),+ "--output",+ path.join(runtimeRoot, "plugins", "codex-security", "mcp"),+ "--native",+ "host",+ ],+ path.join(PLUGIN_ROOT, "mcp-app"),+ );+ if (built !== 0) return built;+ return await runChild([PROMPTFOO_ENTRYPOINT, ...promptfooArgs], EVAL_ROOT);+ } finally {+ for (const [signal, handler] of handlers) process.off(signal, handler);+ fs.rmSync(runtimeRoot, { recursive: true, force: true });+ }+}++function runMain(filename: string, environment: NodeJS.ProcessEnv = {}) {+ try {+ if (!process.argv[1] || fs.realpathSync(process.argv[1]) !== filename)+ return;+ } catch {+ // A virtual entry point imports this module without invoking the runner.+ return;+ }+ runPromptfoo(process.argv.slice(2), environment)+ .then((code) => {+ process.exitCode = code;+ })+ .catch((error) => {+ console.error(error);+ process.exitCode = 1;+ });+}++runMain(import.meta.filename);++export { runMain, runPromptfoo, stageSkillRuntime };diff --git a/evals/triage-finding/scripts/runtime-vars.mts b/evals/triage-finding/scripts/runtime-vars.mtsindex 1cda16dd0..7f9020d43 100644--- a/evals/triage-finding/scripts/runtime-vars.mts+++ b/evals/triage-finding/scripts/runtime-vars.mts@@ -4,13 +4,16 @@ import path from "node:path";const evalRoot = path.resolve(import.meta.dirname, "..");const sourceRoot = path.resolve(evalRoot, "../..");-// Resolve paths for each execution, including retries of persisted evaluations.+// Preserve the stable bindings used by saved native-provider configs, including+// viewer replay, which does not rerun transforms. Current configs bind their+// temporary runtime through the beforeAll extension instead.export default (vars: Record<string, unknown>) => {const nodePath = fs.realpathSync(process.execPath);let targetRepo = vars.target_repo;if (vars.calibration_repo) {targetRepo = path.join((vars.calibration_repo_root ||+ process.env.TRIAGE_CALIBRATION_ROOT ||path.join(evalRoot, "artifacts", "calibration-repos")) as string,vars.calibration_repo as string,);diff --git a/evals/triage-finding/scripts/test-generate-calibration-tests.mts b/evals/triage-finding/scripts/test-generate-calibration-tests.mtsindex 9f8cc7d7e..ec29f511f 100644--- a/evals/triage-finding/scripts/test-generate-calibration-tests.mts+++ b/evals/triage-finding/scripts/test-generate-calibration-tests.mts@@ -4,6 +4,8 @@ import assert from "node:assert/strict";import { execFileSync } from "node:child_process";import fs from "node:fs";import os from "node:os";+import { createRequire } from "node:module";+import runtimeVars from "./runtime-vars.mts";import path from "node:path";import { findingInput, variantCaseId } from "./generate-calibration-tests.mts";import type { CalibrationCase } from "../types.ts";@@ -16,6 +18,8 @@ const generator = path.join("generate-calibration-tests.mts",);const dataset = path.join(evalDir, "datasets", "triage-calibration-seed.json");+const require = createRequire(import.meta.url);+const { parse } = createRequire(require.resolve("promptfoo"))("yaml");const trackedTests = path.join(evalDir, "tests", "calibration-oss.yaml");const tmpDir = fs.realpathSync(@@ -48,6 +52,18 @@ assert.equal(trackedYaml,"tracked calibration tests are stale; run calibration:generate",);+const rows: {+ metadata: Record<string, string>;+ vars: Record<string, string>;+}[] = parse(generatedYaml);+assert.equal(rows.length, 16);+assert.equal(new Set(rows.map((row) => row.vars.case_id)).size, rows.length);+for (const row of rows) {+ assert.equal(row.metadata.case_id, row.vars.case_id);+ assert.equal(row.vars.expected_ids, row.vars.case_id);+ assert.equal(row.vars.calibration_repo, row.vars.case_id);+ assert.ok(row.vars.finding_input.includes(`input_id: ${row.vars.case_id}`));+}const cases: CalibrationCase[] = JSON.parse(fs.readFileSync(dataset, "utf8"),).cases;@@ -138,6 +154,29 @@ assert.ok()}`,),);+const smokeRows = parse(smokeYaml) as typeof rows;+const hydration = execFileSync(+ process.execPath,+ [+ "--experimental-strip-types",+ path.join(evalDir, "scripts", "hydrate-calibration-repos.mts"),+ "--case",+ "oss-dompurify-ghsa-v8jm-5vwx-cfxm",+ "--variant",+ "vulnerable",+ "--repo-root",+ path.join(tmpDir, "custom targets # space"),+ "--dry-run",+ ],+ { encoding: "utf8" },+);+assert.deepEqual(+ smokeRows.map((row) => runtimeVars(row.vars).target_repo),+ hydration+ .split("\n")+ .filter((line) => line.startsWith(" "))+ .map((line) => line.slice(2)),+);fs.rmSync(tmpDir, { recursive: true, force: true });console.log(diff --git a/evals/triage-finding/scripts/test-intake-assertions.mts b/evals/triage-finding/scripts/test-intake-assertions.mtsindex b8f188fa1..078ecd58b 100644--- a/evals/triage-finding/scripts/test-intake-assertions.mts+++ b/evals/triage-finding/scripts/test-intake-assertions.mts@@ -162,6 +162,46 @@ for (const repository of ["{owner}/{repo}", "example/project"]) {);}}+// Match the complete collection endpoint, allowing prose delimiters.+for (const repository of ["{owner}/{repo}", "example/project"]) {+ for (const [behavior, output] of Object.entries({+ dependabot_malware: `GET /repos/${repository}/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: advisory`,+ advisories_private_reports: `GET /repos/${repository}/security-advisories?per_page=100, with separate state=triage, state=draft, state=published, state=closed requests. Triage includes private vulnerability reports. source_type: advisory`,+ })) {+ for (const suffix of [+ "/42/instances",+ "-wrong",+ ".json",+ "_wrong",+ "Extra",+ ]) {+ expectPass(+ github,+ output.replaceAll("?", `${suffix}?`),+ githubContext(behavior),+ false,+ );+ }+ for (const delimiter of [+ "\n",+ "` ",+ '\" ',+ " ",+ ") ",+ ", ",+ ". ",+ "; ",+ ": ",+ "] ",+ ]) {+ expectPass(+ github,+ output.replaceAll("?", `${delimiter}?`),+ githubContext(behavior),+ );+ }+ }+}const codeScanning = {alerts: {path: "/repos/example/project/code-scanning/alerts",diff --git a/evals/triage-finding/scripts/test-promptfoo-runner.mts b/evals/triage-finding/scripts/test-promptfoo-runner.mtsnew file mode 100644index 000000000..23121c2e2--- /dev/null+++ b/evals/triage-finding/scripts/test-promptfoo-runner.mts@@ -0,0 +1,191 @@+import assert from "node:assert/strict";+import { spawn } from "node:child_process";+import fs from "node:fs";+import path from "node:path";+import test from "node:test";+import { stageSkillRuntime } from "./run-promptfoo.mts";++test("runtime contains checkout skill and fixtures without calibration labels", () => {+ const root = stageSkillRuntime();+ try {+ const skill = "plugins/codex-security/skills/triage-finding/SKILL.md";+ assert.equal(+ fs.readFileSync(path.join(root, skill), "utf8"),+ fs.readFileSync(+ path.resolve(import.meta.dirname, "../../..", skill),+ "utf8",+ ),+ );+ assert.ok(+ fs.existsSync(+ path.join(root, "evals/triage-finding/fixtures/repo/src/server.js"),+ ),+ );+ for (const name of ["datasets", "tests", "artifacts"])+ assert.equal(+ fs.existsSync(path.join(root, "evals/triage-finding", name)),+ false,+ );+ } finally {+ fs.rmSync(root, { recursive: true, force: true });+ }+});++test(+ "Windows console Ctrl+C lets the child save progress before runtime cleanup",+ { timeout: 15000 },+ async (t) => {+ const target = `+ const fs = require('node:fs');+ let received = 0;+ process.on('SIGINT', () => {+ received++;+ if (received > 1) process.exit(99);+ setTimeout(() => {+ console.log('saved:' + received + ':' + fs.existsSync(process.env.TRIAGE_RUNTIME_ROOT));+ process.exit(0);+ }, 150);+ });+ // IPC delivers the console event without using Windows kill(SIGINT).+ process.on('message', () => process.emit('SIGINT'));+ console.log(JSON.stringify({ root: process.env.TRIAGE_RUNTIME_ROOT }));+ `;+ const driver = `+ const cp = require('node:child_process');+ const spawn = cp.spawn;+ process.env.TEMP = require('node:os').tmpdir();+ Object.defineProperty(process, 'platform', { value: 'win32' });+ let child;+ cp.spawn = (command, args, options) => {+ if (args[0].endsWith('build_mcp_app.mjs'))+ return spawn(command, ['--eval', ''], options);+ child = spawn(command, ['--eval', ${JSON.stringify(target)}], {+ ...options, stdio: ['inherit', 'inherit', 'inherit', 'ipc'],+ });+ const kill = child.kill.bind(child);+ child.kill = signal => {+ console.log('forced:' + signal);+ return kill('SIGKILL');+ };+ return child;+ };+ process.on('message', () => {+ child.send('console-interrupt');+ process.emit('SIGINT');+ });+ import(${JSON.stringify(new URL("./run-promptfoo.mts", import.meta.url).href)}).then(({ runPromptfoo }) => runPromptfoo([])).then(code => { process.exitCode = code; process.disconnect(); });+ `;+ const child = spawn(+ process.execPath,+ ["--experimental-strip-types", "--eval", driver],+ { stdio: ["ignore", "pipe", "pipe", "ipc"] },+ );+ t.after(() => {+ if (child.exitCode === null) child.kill("SIGKILL");+ });+ let stderr = "";+ child.stderr!.on("data", (chunk) => {+ stderr += chunk;+ });+ const exited = new Promise<number | null>((resolve, reject) => {+ child.once("error", reject);+ child.once("close", resolve);+ });+ let output = "";+ const ready = new Promise<{ root: string }>((resolve) => {+ child.stdout!.on("data", (chunk) => {+ output += chunk;+ if (output.includes("\n")) resolve(JSON.parse(output.split("\n")[0]));+ });+ });+ const { root } = await Promise.race([+ ready,+ exited.then(() => {+ throw new Error(`runner exited early: ${stderr}`);+ }),+ ]);+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ assert.ok(fs.existsSync(root));+ child.send("console-interrupt");+ assert.equal(await exited, 130, stderr);+ assert.match(output, /saved:1:true/);+ assert.doesNotMatch(output, /forced:/);+ assert.equal(fs.existsSync(root), false);+ },+);++for (const [signal, exitCode] of [+ ["SIGINT", 130],+ ["SIGTERM", 143],+] as const) {+ for (const delivery of ["pid", "group"]) {+ test(+ `runner forwards ${signal} once (${delivery}) and removes its runtime after the child exits`,+ { skip: process.platform === "win32", timeout: 15000 },+ async (t) => {+ const target = `+ let received = 0;+ process.on(${JSON.stringify(signal)}, () => {+ received++;+ if (received > 1) process.exit(99);+ setTimeout(() => { console.log("graceful:" + received); process.exit(0); }, 150);+ });+ console.log(JSON.stringify({ root: process.env.TRIAGE_RUNTIME_ROOT }));+ setInterval(() => {}, 1000);+ `;+ const driver = `+ const cp = require('node:child_process');+ const spawn = cp.spawn;+ cp.spawn = (command, args, options) => spawn(process.execPath, ['--eval', args[0].endsWith("build_mcp_app.mjs") ? "" : ${JSON.stringify(target)}], options);+ import(${JSON.stringify(new URL("./run-promptfoo.mts", import.meta.url).href)}).then(({ runPromptfoo }) => runPromptfoo([])).then(code => { process.exitCode = code; });+ `;+ const child = spawn(+ process.execPath,+ ["--experimental-strip-types", "--eval", driver],+ {+ stdio: ["ignore", "pipe", "pipe"],+ detached: true,+ },+ );+ t.after(() => {+ if (child.exitCode === null) child.kill("SIGKILL");+ });+ let stderr = "";+ child.stderr.on("data", (chunk) => {+ stderr += chunk;+ });+ const exited = new Promise<{+ code: number | null;+ signal: NodeJS.Signals | null;+ }>((resolve, reject) => {+ child.once("error", reject);+ child.once("exit", (code, signal) => resolve({ code, signal }));+ });+ let text = "";+ const ready = new Promise<{ root: string }>((resolve) => {+ child.stdout.on("data", (chunk) => {+ text += chunk;+ if (text.includes("\n")) resolve(JSON.parse(text.split("\n")[0]));+ });+ });+ const { root } = await Promise.race([+ ready,+ exited.then(() => {+ throw new Error(`runner exited early: ${stderr}`);+ }),+ ]);+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ assert.ok(fs.existsSync(root));+ if (delivery === "group") process.kill(-child.pid!, signal);+ else child.kill(signal);+ assert.deepEqual(+ await exited,+ { code: exitCode, signal: null },+ stderr,+ );+ assert.match(text, /graceful:1/);+ assert.equal(fs.existsSync(root), false);+ },+ );+ }+}diff --git a/evals/triage-finding/scripts/test-promptfoo-runtime.mts b/evals/triage-finding/scripts/test-promptfoo-runtime.mtsindex 9b82ca927..a2c7c1adb 100644--- a/evals/triage-finding/scripts/test-promptfoo-runtime.mts+++ b/evals/triage-finding/scripts/test-promptfoo-runtime.mts@@ -4,27 +4,51 @@ import assert from "node:assert/strict";import fs from "node:fs";import path from "node:path";import runtimeVars from "./runtime-vars.mts";-import { DEFAULT_CACHE_ROOT } from "../sastbench/scripts/hydrate-sastbench-repos.mts";const evalRoot = path.resolve(import.meta.dirname, "..");-const sourceRoot = path.resolve(evalRoot, "../..");-const pluginRoot = path.join(sourceRoot, "plugins", "codex-security");-const vars = runtimeVars({});-assert.equal(vars.triage_runtime_root, pluginRoot);-assert.equal(vars.triage_fixture_root, path.join(evalRoot, "fixtures"));-assert.equal(vars.triage_node_path, fs.realpathSync(process.execPath));-assert.equal(vars.triage_node_root, path.dirname(vars.triage_node_path));-assert.equal(vars.sastbench_git_cache_root, DEFAULT_CACHE_ROOT);-assert.ok(- fs.existsSync(path.join(pluginRoot, "skills/triage-finding/SKILL.md")),-);-for (const fixture of ["repo", "policy-repo"]) {- const target = runtimeVars({- target_repo: `evals/triage-finding/fixtures/${fixture}`,- }).target_repo;- assert.equal(target, path.join(evalRoot, "fixtures", fixture));- assert.ok(fs.existsSync(target as string));+// Saved native configs have no startup hook during viewer replay. Their+// plugin-relative working directory and fixture bindings must stay usable.+const originalRuntime = process.env.TRIAGE_RUNTIME_ROOT;+try {+ for (const staged of [+ undefined,+ path.join(evalRoot, "synthetic staged runtime"),+ ] as const) {+ if (staged === undefined) delete process.env.TRIAGE_RUNTIME_ROOT;+ else process.env.TRIAGE_RUNTIME_ROOT = staged;+ const root = path.resolve(evalRoot, "../..");+ const bindings = runtimeVars({ triage_runtime_root: "old runtime" });+ assert.equal(+ bindings.triage_runtime_root,+ path.join(root, "plugins", "codex-security"),+ );+ assert.equal(+ bindings.triage_fixture_root,+ path.join(root, "evals", "triage-finding", "fixtures"),+ );+ assert.equal(bindings.triage_node_path, fs.realpathSync(process.execPath));+ assert.equal(+ bindings.triage_node_root,+ path.dirname(bindings.triage_node_path),+ );+ assert.equal(+ bindings.sastbench_git_cache_root,+ path.join(evalRoot, "artifacts", "sastbench-git-cache"),+ );+ for (const target of [+ "evals/triage-finding/fixtures/repo",+ "evals/triage-finding/fixtures/policy-repo",+ ]) {+ assert.equal(+ runtimeVars({ target_repo: target }).target_repo,+ path.join(root, target),+ );+ }+ }+} finally {+ if (originalRuntime === undefined) delete process.env.TRIAGE_RUNTIME_ROOT;+ else process.env.TRIAGE_RUNTIME_ROOT = originalRuntime;}for (const target of [path.join(evalRoot, "artifacts", "sastbench-targets", "opaque-target"),@@ -35,46 +59,72 @@ for (const target of [}const calibrationRepo = "calibration-0123456789abcdef";-for (const customRoot of ["", path.join(evalRoot, "custom targets # space")]) {- const replayed = runtimeVars({- calibration_repo: calibrationRepo,- calibration_repo_root: customRoot,- target_repo: "old checkout",- triage_runtime_root: "old plugin",- });- assert.equal(replayed.triage_runtime_root, pluginRoot);- assert.equal(- replayed.target_repo,- path.join(- customRoot || path.join(evalRoot, "artifacts", "calibration-repos"),- calibrationRepo,- ),- );+const originalCalibrationRoot = process.env.TRIAGE_CALIBRATION_ROOT;+const defaultCalibrationRoot = path.join(+ evalRoot,+ "artifacts",+ "calibration-repos",+);+const environmentCalibrationRoot = path.join(+ evalRoot,+ "environment targets # space",+);+const explicitCalibrationRoot = path.join(evalRoot, "case targets # space");+try {+ for (const [environmentRoot, caseRoot, expectedRoot] of [+ ["", "", defaultCalibrationRoot],+ [environmentCalibrationRoot, "", environmentCalibrationRoot],+ ["", explicitCalibrationRoot, explicitCalibrationRoot],+ [+ environmentCalibrationRoot,+ explicitCalibrationRoot,+ explicitCalibrationRoot,+ ],+ ]) {+ process.env.TRIAGE_CALIBRATION_ROOT = environmentRoot;+ const replayed = runtimeVars({+ calibration_repo: calibrationRepo,+ calibration_repo_root: caseRoot,+ target_repo: "old checkout",+ });+ assert.equal(+ replayed.target_repo,+ path.join(expectedRoot, calibrationRepo),+ );+ }+} finally {+ if (originalCalibrationRoot === undefined)+ delete process.env.TRIAGE_CALIBRATION_ROOT;+ else process.env.TRIAGE_CALIBRATION_ROOT = originalCalibrationRoot;}for (const name of ["promptfooconfig.yaml","promptfooconfig.calibration.yaml",- "promptfooconfig.calibration-smoke.yaml","sastbench/promptfooconfig.sastbench.yaml",]) {const config = fs.readFileSync(path.join(evalRoot, name), "utf8");- assert.match(config, /working_dir: "\{\{triage_runtime_root\}\}"/);assert.match(config, /":minimal": read\n\s+":workspace_roots": read/);assert.match(config, /plugins: false/);assert.match(config, /memories: false/);- assert.ok(config.includes('"{{triage_node_root}}"'));- assert.ok(- config.includes(- name === "promptfooconfig.yaml"- ? '"{{triage_fixture_root}}"'- : '"{{target_repo}}"',- ),- );+ assert.doesNotMatch(config, /working_dir:|CODEX_MCP_NODE_PATH:/);+ if (name === "promptfooconfig.calibration.yaml") {+ assert.ok(config.includes("{{calibration_repo}}"));+ assert.match(config, /transformVars:/);+ }}+assert.match(+ fs.readFileSync(+ path.join(evalRoot, "promptfooconfig.calibration-smoke.yaml"),+ "utf8",+ ),+ /\$ref: \.\/promptfooconfig\.calibration\.yaml/,+);assert.ok(fs.readFileSync(path.join(evalRoot, "prompts/triage-request.txt"), "utf8")- .startsWith("Read and follow skills/triage-finding/SKILL.md"),+ .startsWith(+ "Read and follow plugins/codex-security/skills/triage-finding/SKILL.md",+ ),);-console.log("triage Promptfoo checkout path tests passed");+console.log("triage Promptfoo target path tests passed");diff --git a/evals/triage-finding/scripts/test-triage-io.mts b/evals/triage-finding/scripts/test-triage-io.mtsindex e4f4f9fe1..9af46dc92 100644--- a/evals/triage-finding/scripts/test-triage-io.mts+++ b/evals/triage-finding/scripts/test-triage-io.mts@@ -81,13 +81,13 @@ const sourceType = "cve";assertPasses("vulnerable scanbench cases map to confirmed/positive",outputFor({- inputId: "GHSA-example-000-vulnerable",+ inputId: "input-001",verdict: "confirmed",}),{vars: {- case_id: "ghsa-example-vulnerable",- expected_ids: "GHSA-example-000-vulnerable",+ case_id: "case-001",+ expected_ids: "input-001",expected_source_types: sourceType,expected_verdicts: "confirmed",expected_binary_label: "positive",@@ -98,13 +98,13 @@ assertPasses(assertPasses("fixed scanbench cases map to not_actionable/negative",outputFor({- inputId: "GHSA-example-000-fixed",+ inputId: "input-002",verdict: "not_actionable",}),{vars: {- case_id: "ghsa-example-fixed",- expected_ids: "GHSA-example-000-fixed",+ case_id: "case-002",+ expected_ids: "input-002",expected_source_types: sourceType,expected_verdicts: "not_actionable",expected_binary_label: "negative",@@ -115,16 +115,16 @@ assertPasses(assertFails("fixed scanbench cases cannot be labeled confirmed",outputFor({- inputId: "GHSA-example-000-fixed",+ inputId: "input-002",verdict: "confirmed",}),{vars: {- case_id: "ghsa-example-fixed",- expected_ids: "GHSA-example-000-fixed",+ case_id: "case-002",+ expected_ids: "input-002",expected_source_types: sourceType,expected_verdicts: "confirmed",- expected_binary_label: "positive",+ expected_binary_label: "negative",},},/fixed.*not_actionable|negative/,diff --git a/evals/triage-finding/scripts/test-triage-provider.mts b/evals/triage-finding/scripts/test-triage-provider.mtsnew file mode 100644index 000000000..7bbfda637--- /dev/null+++ b/evals/triage-finding/scripts/test-triage-provider.mts@@ -0,0 +1,1427 @@+import assert from "node:assert/strict";+import { spawn } from "node:child_process";+import fs from "node:fs";+import { createRequire } from "node:module";+import os from "node:os";+import { createServer } from "node:net";+import path from "node:path";+import test from "node:test";+import { DatabaseSync } from "node:sqlite";+import { beforeAll } from "./triage-provider.mts";+const require = createRequire(import.meta.url);+const { parse } = createRequire(require.resolve("promptfoo"))("yaml");++const evalRoot = path.resolve(import.meta.dirname, "..");+const runner = path.join(import.meta.dirname, "run-promptfoo.mts");+const extensions = [+ `file://${path.join(import.meta.dirname, "triage-provider.mts")}:beforeAll`,+];+interface Capture {+ cwd: string;+ policy: string;+ proxies: Record<string, string>;+ nodePath: string;+ nodeTarget: string;+ directories: string[];+ overrides: string[];+}++const proxies = {+ HTTP_PROXY: "http://http.example.test:8080",+ HTTPS_PROXY: "http://https.example.test:8080",+ ALL_PROXY: "http://all.example.test:8080",+ NO_PROXY: "localhost,.example.test",+};++function invoke(args: string[], environment: NodeJS.ProcessEnv) {+ return new Promise<{ code: number | null; output: string }>(+ (resolve, reject) => {+ const child = spawn(+ process.execPath,+ ["--experimental-strip-types", runner, ...args],+ {+ cwd: evalRoot,+ env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },+ stdio: ["ignore", "pipe", "pipe"],+ },+ );+ let output = "";+ child.stdout.on("data", (chunk) => {+ output += chunk;+ });+ child.stderr.on("data", (chunk) => {+ output += chunk;+ });+ child.once("error", reject);+ child.once("close", (code) => resolve({ code, output }));+ },+ );+}++test(+ "runtime extension preserves native credential precedence and wraps providers once",+ { skip: process.platform === "win32" },+ async (t) => {+ const root = fs.realpathSync(+ fs.mkdtempSync(path.join(os.tmpdir(), "triage-auth-")),+ );+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const priorRuntime = process.env.TRIAGE_RUNTIME_ROOT;+ process.env.TRIAGE_RUNTIME_ROOT = root;+ t.after(() => {+ if (priorRuntime === undefined) delete process.env.TRIAGE_RUNTIME_ROOT;+ else process.env.TRIAGE_RUNTIME_ROOT = priorRuntime;+ });+ const fakeCodex = path.join(root, "codex");+ fs.writeFileSync(+ fakeCodex,+ `#!${process.execPath}+console.log(JSON.stringify({type:'thread.started',thread_id:'synthetic-auth'}));+console.log(JSON.stringify({type:'item.completed',item:{id:'message',type:'agent_message',text:JSON.stringify({apiKey:process.env.CODEX_API_KEY,cwd:process.argv[process.argv.indexOf('--cd')+1],directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])})}}));+console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_tokens:1}}));+`,+ { mode: 0o755 },+ );+ const { loadApiProvider } = await import("promptfoo");+ const cases = [+ {+ env: { OPENAI_API_KEY: "synthetic-global" },+ providerEnv: { CODEX_API_KEY: "synthetic-provider" },+ expected: "synthetic-provider",+ },+ {+ env: { CODEX_API_KEY: "synthetic-global" },+ providerEnv: { OPENAI_API_KEY: "synthetic-provider" },+ expected: "synthetic-provider",+ },+ {+ env: { OPENAI_API_KEY: "synthetic-global" },+ providerEnv: {+ OPENAI_API_KEY: "synthetic-openai",+ CODEX_API_KEY: "synthetic-codex",+ },+ expected: "synthetic-openai",+ },+ {+ env: { OPENAI_API_KEY: "synthetic-global" },+ providerEnv: { CODEX_API_KEY: "synthetic-provider" },+ apiKey: "synthetic-explicit",+ expected: "synthetic-explicit",+ },+ {+ env: { OPENAI_API_KEY: "synthetic-global" },+ providerEnv: { CODEX_API_KEY: "" },+ expected: "synthetic-global",+ },+ ];+ for (const entry of cases) {+ const load = () =>+ loadApiProvider("openai:codex-sdk:gpt-5.5", {+ basePath: root,+ env: entry.env,+ options: {+ env: entry.providerEnv,+ config: {+ working_dir: root,+ codex_path_override: fakeCodex,+ skip_git_repo_check: true,+ model: "gpt-5.5",+ maxRetries: 0,+ apiKey: entry.apiKey,+ cli_env: { CODEX_MCP_NODE_PATH: process.execPath },+ },+ },+ });+ const native = await load();+ const adapted = await load();+ try {+ const context = { suite: { providers: [adapted] } };+ beforeAll(context);+ beforeAll(context);+ const original = await native.callApi("synthetic");+ const result = await adapted.callApi("synthetic");+ assert.equal(original.error, undefined);+ assert.equal(result.error, undefined);+ const originalOutput = JSON.parse(String(original.output));+ const output = JSON.parse(String(result.output));+ assert.equal(originalOutput.apiKey, entry.expected);+ assert.equal(output.apiKey, originalOutput.apiKey);+ assert.equal(output.cwd, root);+ assert.equal(adapted.id(), native.id());+ assert.deepEqual(output.directories, [+ path.dirname(fs.realpathSync(process.execPath)),+ path.dirname(process.execPath),+ path.dirname(fakeCodex),+ ]);+ } finally {+ await native.cleanup?.();+ await adapted.cleanup?.();+ }+ }+ },+);++// Exercise the pinned provider and Codex SDK subprocess without a model request.+// The normal setup command supplies the host helper build used by the runner.+test(+ "provider preserves proxies and rebinds saved evaluations to fresh runtimes",+ { skip: process.platform === "win32", timeout: 120000 },+ async (t) => {+ const root = fs.realpathSync(+ fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-")),+ );+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const capture = path.join(root, "captures.jsonl");+ const fakeCodex = path.join(root, "codex");+ const fail = path.join(root, "fail");+ fs.writeFileSync(+ fakeCodex,+ `#!${process.execPath}+const fs = require('node:fs');+const path = require('node:path');+const cp = require('node:child_process');+const cwd = process.argv[process.argv.indexOf('--cd') + 1];+const launcher = path.join(cwd, 'plugins/codex-security/scripts/launch_codex_security_mcp');+const fixture = path.join(cwd, 'evals/triage-finding/fixtures/repo');+fs.writeFileSync(path.join(fixture, 'SECURITY.md'), '# Synthetic policy\\n');+const policy = cp.execFileSync(launcher, ['--helper', 'resolve-security-md', '--repo', fixture, '--scope', 'src/server.js', '--out', '-'], {encoding:'utf8'});+const directories = process.argv.flatMap((arg, index) => arg === '--add-dir' ? [process.argv[index + 1]] : []);+const overrides = process.argv.flatMap((arg, index) => arg === '--config' ? [process.argv[index + 1]] : []);+fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy,directories,overrides,nodePath:process.env.CODEX_MCP_NODE_PATH,nodeTarget:fs.realpathSync.native(process.env.CODEX_MCP_NODE_PATH), proxies: Object.fromEntries(${JSON.stringify(Object.keys(proxies))}.map(key => [key,process.env[key]]))}) + '\\n');+console.log(JSON.stringify({type:'thread.started', thread_id:'synthetic-thread'}));+if (fs.existsSync(${JSON.stringify(fail)})) {+ console.log(JSON.stringify({type:'turn.failed',error:{message:'synthetic retryable failure'}}));+} else {+ console.log(JSON.stringify({type:'item.completed', item:{id:'message',type:'agent_message',text:'ok'}}));+ console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,cached_input_tokens:0,output_tokens:1}}));+}+`,+ { mode: 0o755 },+ );+ const ambientNode = path.join(root, "ambient-node", "bin", "node");+ fs.mkdirSync(path.dirname(ambientNode), { recursive: true });+ fs.copyFileSync(+ process.execPath,+ ambientNode,+ fs.constants.COPYFILE_FICLONE,+ );+ const environment = {+ ...proxies,+ CODEX_MCP_NODE_PATH: ambientNode,+ OPENAI_API_KEY: "synthetic-test-key",+ PROMPTFOO_CONFIG_DIR: path.join(root, "state"),+ PROMPTFOO_DISABLE_WAL_MODE: "true",+ PROMPTFOO_DISABLE_TELEMETRY: "1",+ PROMPTFOO_DISABLE_UPDATE: "1",+ };+ const configPath = path.join(root, "config.json");+ const provider = parse(+ fs.readFileSync(path.join(evalRoot, "promptfooconfig.yaml"), "utf8"),+ ).providers[0];+ provider.config.codex_path_override = fakeCodex;+ fs.writeFileSync(+ configPath,+ JSON.stringify({+ extensions,+ providers: [provider],+ prompts: ["hello"],+ tests: [{ assert: [{ type: "equals", value: "ok" }] }],+ }),+ );+ const calibration = parse(+ fs.readFileSync(+ path.join(evalRoot, "promptfooconfig.calibration.yaml"),+ "utf8",+ ),+ ).providers[0];+ calibration.id = provider.id;+ calibration.config.codex_path_override = fakeCodex;+ const customNode = path.join(root, "custom-node", "bin", "node");+ fs.mkdirSync(path.dirname(customNode), { recursive: true });+ fs.copyFileSync(+ process.execPath,+ customNode,+ fs.constants.COPYFILE_FICLONE,+ );+ calibration.config.cli_env.CODEX_MCP_NODE_PATH = customNode;+ const selectedNode = path.join(+ root,+ "selected-node",+ "bin",+ "synthetic-node",+ );+ fs.mkdirSync(path.dirname(selectedNode), { recursive: true });+ fs.copyFileSync(+ process.execPath,+ selectedNode,+ fs.constants.COPYFILE_FICLONE,+ );+ const nodeAlias = path.join(root, "node-alias");+ const nodeDispatcher = path.join(root, "node-runtime", "dispatcher");+ fs.mkdirSync(path.dirname(nodeDispatcher));+ fs.writeFileSync(+ nodeDispatcher,+ `#!/bin/sh+case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "$@" ;; *) echo 'synthetic dispatcher requires node-alias' >&2; exit 42 ;; esac+`,+ { mode: 0o755 },+ );+ fs.symlinkSync(nodeDispatcher, nodeAlias);+ const nodeChoices = [+ customNode,+ path.relative(path.join(os.tmpdir(), "runtime-placeholder"), customNode),+ "node",+ nodeAlias,+ path.join(root, "missing-node"),+ "synthetic-node",+ ];+ const calibrationConfig = path.join(root, "calibration.json");+ fs.mkdirSync(path.join(root, "case"));+ fs.writeFileSync(+ calibrationConfig,+ JSON.stringify({+ extensions,+ defaultTest: {+ options: {+ transformVars: `file://${path.join(import.meta.dirname, "runtime-vars.mts")}`,+ },+ },+ providers: nodeChoices.map((nodePath, index) => ({+ ...calibration,+ label: `custom-runtime-${index}`,+ config: {+ ...calibration.config,+ cli_env: {+ ...calibration.config.cli_env,+ CODEX_MCP_NODE_PATH: nodePath,+ PATH: `${path.dirname(selectedNode)}${path.delimiter}${path.dirname(customNode)}${path.delimiter}${process.env.PATH}`,+ },+ },+ })),+ prompts: ["hello"],+ tests: [+ {+ vars: { calibration_repo_root: root, calibration_repo: "case" },+ assert: [{ type: "equals", value: "ok" }],+ },+ ],+ }),+ );+ const simultaneous = await Promise.all(+ [configPath, calibrationConfig, configPath].map((config, index) =>+ invoke(+ [+ "eval",+ "-c",+ config,+ "--no-cache",+ "--no-share",+ "--no-progress-bar",+ ],+ {+ ...environment,+ CODEX_MCP_NODE_PATH: index === 0 ? undefined : ambientNode,+ PROMPTFOO_CONFIG_DIR: path.join(root, `concurrent-${index}`),+ },+ ),+ ),+ );+ for (const result of simultaneous)+ assert.equal(result.code, 0, result.output);+ const rows = fs+ .readFileSync(capture, "utf8")+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line) as Capture);+ assert.equal(rows.length, 8);+ assert.equal(new Set(rows.map((row) => row.cwd)).size, 3);+ for (const row of rows) {+ assert.deepEqual(row.proxies, proxies);+ assert.match(row.policy, /Synthetic policy/);+ assert.equal(fs.existsSync(row.cwd), false);+ assert.match(+ row.overrides.join("\n"),+ /permissions\.triage_runtime_only\.filesystem\.:workspace_roots="read"/,+ );+ assert.deepEqual(row.directories, [+ ...([customNode, selectedNode, nodeDispatcher].includes(row.nodeTarget)+ ? [path.join(root, "case")]+ : []),+ path.dirname(row.nodeTarget),+ path.dirname(row.nodePath),+ path.dirname(fakeCodex),+ ]);+ }+ assert.equal(rows.filter((row) => row.nodeTarget === customNode).length, 4);+ assert.equal(rows.filter((row) => row.nodePath === nodeAlias).length, 1);+ assert.equal(rows.filter((row) => row.nodePath === selectedNode).length, 1);+ assert.equal(rows.filter((row) => row.nodePath === ambientNode).length, 1);+ assert.equal(+ rows.filter((row) => row.nodePath === fs.realpathSync(process.execPath))+ .length,+ 1,+ );++ // Prompt-specific settings merge and render separately for each concurrent+ // case, including the original configuration saved for retry and resume.+ const templatedConfig = path.join(root, "templated.json");+ fs.writeFileSync(+ templatedConfig,+ JSON.stringify({+ extensions,+ providers: [+ {+ ...provider,+ config: {+ ...provider.config,+ cli_env: {+ ...provider.config.cli_env,+ CODEX_MCP_NODE_PATH: "{{node_path}}",+ },+ additional_directories: ["{{permission_root}}"],+ },+ },+ ],+ prompts: [+ "hello",+ {+ id: "hello with an override",+ label: "per-case runtime override",+ config: {+ cli_env: {+ ...provider.config.cli_env,+ CODEX_MCP_NODE_PATH: "{{override_node_path}}",+ },+ additional_directories: ["{{override_permission_root}}"],+ },+ },+ ],+ tests: [customNode, selectedNode].map((nodePath) => ({+ vars: {+ node_path: nodePath,+ permission_root: path.dirname(nodePath),+ override_node_path:+ nodePath === customNode ? selectedNode : customNode,+ override_permission_root: path.dirname(+ nodePath === customNode ? selectedNode : customNode,+ ),+ },+ assert: [{ type: "equals", value: "ok" }],+ })),+ }),+ );+ const templatedEnvironment = {+ ...environment,+ PROMPTFOO_CONFIG_DIR: path.join(root, "templated-state"),+ };+ const startingRows = rows.length;+ fs.writeFileSync(fail, "");+ const templatedInitial = await invoke(+ [+ "eval",+ "-c",+ templatedConfig,+ "--max-concurrency",+ "2",+ "--no-cache",+ "--no-share",+ "--no-progress-bar",+ ],+ templatedEnvironment,+ );+ assert.notEqual(templatedInitial.code, 0, templatedInitial.output);+ assert.match(templatedInitial.output, /synthetic retryable failure/);+ fs.rmSync(fail);+ const templatedRetry = await invoke(+ [+ "eval",+ "--retry-errors",+ "--no-cache",+ "--no-share",+ "--no-progress-bar",+ ],+ templatedEnvironment,+ );+ assert.equal(templatedRetry.code, 0, templatedRetry.output);+ const templatedDatabase = new DatabaseSync(+ path.join(templatedEnvironment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),+ { enableForeignKeyConstraints: false },+ );+ templatedDatabase.prepare("DELETE FROM eval_results").run();+ templatedDatabase.close();+ const templatedResume = await invoke(+ ["eval", "--resume", "--no-cache", "--no-share", "--no-progress-bar"],+ templatedEnvironment,+ );+ assert.equal(templatedResume.code, 0, templatedResume.output);+ const templatedRows = fs+ .readFileSync(capture, "utf8")+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line) as Capture)+ .slice(startingRows);+ assert.equal(templatedRows.length, 12);+ assert.equal(new Set(templatedRows.map((row) => row.cwd)).size, 3);+ assert.equal(+ templatedRows.filter((row) => row.nodePath === customNode).length,+ 6,+ );+ assert.equal(+ templatedRows.filter((row) => row.nodePath === selectedNode).length,+ 6,+ );+ for (const row of templatedRows) {+ assert.deepEqual(row.proxies, proxies);+ assert.equal(fs.existsSync(row.cwd), false);+ assert.deepEqual(row.directories, [+ path.dirname(row.nodePath),+ path.dirname(row.nodePath),+ path.dirname(row.nodePath),+ path.dirname(fakeCodex),+ ]);+ }+ },+);++test(+ "provider resolves each call's Node template without leaking sibling configuration",+ { skip: process.platform === "win32", timeout: 120000 },+ async (t) => {+ const root = fs.realpathSync(+ fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-vars-")),+ );+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const nodes = [+ "first & node/node",+ "second/node",+ "literal/{{custom_node}}",+ ].map((name) => path.join(root, name)) as [string, string, string];+ for (const node of nodes) {+ fs.mkdirSync(path.dirname(node), { recursive: true });+ fs.writeFileSync(node, "#!/bin/sh\nexit 0\n", { mode: 0o755 });+ }+ const extra = path.join(root, "extra");+ fs.mkdirSync(extra);+ const fakeCodex = path.join(root, "codex");+ fs.writeFileSync(+ fakeCodex,+ `#!${process.execPath}+const details = {executable:process.env.CODEX_TEST_EXECUTABLE || require('node:fs').realpathSync(process.argv[1]), path:process.env.PATH, node:process.env.CODEX_MCP_NODE_PATH, marker:process.env.EXTRA_MARKER, directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])};+console.log(JSON.stringify({type:'thread.started',thread_id:'synthetic-thread'}));+console.log(JSON.stringify({type:'item.completed',item:{id:'message',type:'agent_message',text:JSON.stringify(details)}}));+console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,cached_input_tokens:0,output_tokens:1}}));+`,+ { mode: 0o755 },+ );+ const environment = {+ TRIAGE_RUNTIME_ROOT: root,+ PROMPTFOO_CONFIG_DIR: path.join(root, "state"),+ PROMPTFOO_DISABLE_TELEMETRY: "1",+ PROMPTFOO_DISABLE_UPDATE: "1",+ PROMPTFOO_DISABLE_WAL_MODE: "true",+ PROMPTFOO_DISABLE_TEMPLATING: "",+ NODE_BASENAME: "node",+ CODEX_MCP_NODE_PATH: "",+ OPENAI_API_KEY: "synthetic-test-key",+ };+ const original = new Map(+ Object.keys(environment).map((key) => [key, process.env[key]]),+ );+ t.after(() => {+ for (const [key, value] of original) {+ if (value === undefined) delete process.env[key];+ else process.env[key] = value;+ }+ });+ Object.assign(process.env, environment);+ const { loadApiProvider } = await import("promptfoo");+ const load = (+ template: string | undefined,+ command = fakeCodex,+ cliEnvironment: Record<string, string> = {},+ ) =>+ loadApiProvider("openai:codex-sdk:gpt-5.5", {+ basePath: root,+ options: {+ config: {+ working_dir: root,+ codex_path_override: command,+ skip_git_repo_check: true,+ model: "gpt-5.5",+ maxRetries: 0,+ cli_env: {+ ...cliEnvironment,+ EXTRA_MARKER: "{{marker}}",+ ...(template === undefined+ ? {}+ : { CODEX_MCP_NODE_PATH: template }),+ },+ },+ },+ }).then((provider) => {+ beforeAll({ suite: { providers: [provider] } });+ return provider;+ });+ for (const template of [+ "{{custom_node}}",+ '{{custom_node | replace("second", "second")}}',+ "{{custom_node_json | load}}",+ nodes[0],+ ]) {+ const provider = await load(template);+ const overrides = template === nodes[0];+ const invoke = async (node: string) => {+ const context = {+ vars: {+ custom_node: node,+ custom_node_json: JSON.stringify(node),+ marker: node,+ },+ prompt: { raw: "synthetic", label: "synthetic" },+ ...(overrides+ ? {+ prompt: {+ raw: "synthetic",+ label: "synthetic",+ config: {+ cli_env: {+ CODEX_MCP_NODE_PATH: "{{custom_node}}",+ EXTRA_MARKER: "{{marker}}",+ },+ additional_directories: [extra],+ },+ },+ }+ : {}),+ };+ const saved = structuredClone(context);+ const result = await provider.callApi(+ "Return synthetic launch details",+ context as Parameters<typeof provider.callApi>[1],+ );+ assert.equal(result.error, undefined);+ const captured = JSON.parse(String(result.output));+ assert.equal(captured.node, node);+ assert.deepEqual(captured.directories, [+ ...(overrides ? [extra] : []),+ path.dirname(node),+ path.dirname(node),+ path.dirname(fakeCodex),+ ]);+ if (overrides) assert.equal(captured.marker, node);+ assert.deepEqual(context, saved);+ };+ try {+ for (const node of nodes.slice(0, 2)) await invoke(node);+ await Promise.all(nodes.slice(0, 2).map(invoke));+ } finally {+ await provider.cleanup?.();+ }+ }+ for (const sibling of [false, true]) {+ const nodeRoot = path.join(+ root,+ sibling ? "node-sibling" : "node-missing",+ );+ const target = path.join(nodeRoot, "target", "node");+ const alias = path.join(nodeRoot, "alias");+ fs.mkdirSync(path.join(nodeRoot, "target", "child"), { recursive: true });+ fs.copyFileSync(nodes[0], target);+ fs.symlinkSync(path.join(nodeRoot, "target", "child"), alias);+ if (sibling) fs.copyFileSync(nodes[1], path.join(nodeRoot, "node"));+ for (const [command, expected, searchPath] of [+ [`${alias}/../node`, target, process.env.PATH],+ [`${path.relative(root, alias)}/../node`, target, process.env.PATH],+ ["node", target, `${alias}/..`],+ ["node", target, `${path.relative(root, alias)}/..`],+ [+ path.join(nodeRoot, "target") + "/child/../node",+ target,+ process.env.PATH,+ ],+ [path.join(nodeRoot, "missing"), nodes[0], path.dirname(nodes[0])],+ ] as const) {+ const provider = await load(command, fakeCodex, { PATH: searchPath! });+ try {+ const result = await provider.callApi("synthetic");+ assert.equal(result.error, undefined);+ const captured = JSON.parse(String(result.output));+ assert.equal(+ fs.realpathSync.native(captured.node),+ expected,+ command,+ );+ assert.deepEqual(captured.directories, [+ path.dirname(expected),+ path.dirname(captured.node),+ path.dirname(fakeCodex),+ ]);+ } finally {+ await provider.cleanup?.();+ }+ }+ }+ const alias = path.join(root, "codex-alias");+ fs.symlinkSync(fakeCodex, alias);+ const selectedPath = `${root}${path.delimiter}${process.env.PATH}`;+ const builtinCodex = path.join(root, "test");+ fs.copyFileSync(fakeCodex, builtinCodex);+ const literalCodex = path.join(root, String.raw`codex\probe`);+ fs.copyFileSync(fakeCodex, literalCodex);+ const targetRoot = path.join(root, "path-target");+ fs.mkdirSync(path.join(targetRoot, "child"), { recursive: true });+ const directoryAlias = path.join(root, "path-alias");+ fs.symlinkSync(path.join(targetRoot, "child"), directoryAlias);+ const targetCodex = path.join(targetRoot, "codex");+ const shellQuote = (value: string) =>+ "'" + value.replaceAll("'", "'\\''") + "'";+ fs.writeFileSync(+ targetCodex,+ `#!/bin/sh\nCODEX_TEST_EXECUTABLE=${shellQuote(targetCodex)} exec ${shellQuote(process.execPath)} ${shellQuote(fakeCodex)} "$@"\n`,+ { mode: 0o755 },+ );+ const relativePath = path.relative(process.cwd(), root);+ for (const [command, executable, searchPath] of [+ [fakeCodex, fakeCodex, selectedPath],+ [path.relative(process.cwd(), fakeCodex), fakeCodex, selectedPath],+ [path.basename(fakeCodex), fakeCodex, selectedPath],+ [alias, fakeCodex, selectedPath],+ ["test", builtinCodex, selectedPath],+ [path.basename(literalCodex), literalCodex, selectedPath],+ ["codex", targetCodex, `${directoryAlias}/..`],+ [+ "codex",+ targetCodex,+ `${path.relative(process.cwd(), directoryAlias)}/..`,+ ],+ [`${directoryAlias}/../codex`, targetCodex, selectedPath],+ ["test", builtinCodex, relativePath],+ ["test", builtinCodex, `${path.delimiter}${relativePath}`],+ ] as const) {+ const provider = await load(nodes[0], command, { PATH: searchPath });+ try {+ const result = await provider.callApi("synthetic");+ assert.equal(result.error, undefined);+ const captured = JSON.parse(String(result.output));+ assert.equal(captured.executable, executable);+ assert.equal(captured.path, searchPath);+ assert.deepEqual(captured.directories, [+ path.dirname(nodes[0]),+ path.dirname(nodes[0]),+ path.dirname(executable),+ ]);+ } finally {+ await provider.cleanup?.();+ }+ }+ const otherCodex = path.join(root, "other", "codex");+ fs.mkdirSync(path.dirname(otherCodex));+ fs.copyFileSync(fakeCodex, otherCodex);+ const selectedCodex = await load(nodes[0], "{{selected_codex}}");+ try {+ await Promise.all(+ [fakeCodex, otherCodex, fakeCodex].map(async (command) => {+ const context = {+ vars: { selected_codex: command },+ prompt: { raw: "synthetic", label: "synthetic" },+ };+ const result = await selectedCodex.callApi("synthetic", context);+ assert.equal(result.error, undefined);+ const captured = JSON.parse(String(result.output));+ assert.equal(captured.executable, command);+ assert.deepEqual(captured.directories, [+ path.dirname(nodes[0]),+ path.dirname(nodes[0]),+ path.dirname(command),+ ]);+ assert.deepEqual(context, {+ vars: { selected_codex: command },+ prompt: { raw: "synthetic", label: "synthetic" },+ });+ }),+ );+ } finally {+ await selectedCodex.cleanup?.();+ }+ const packages = ["first", "second", "vendored"].map((name) => {+ const modules = path.join(root, name, "node_modules", "@openai");+ const packageRoot = path.join(modules, "codex");+ const nativePackage =+ name === "vendored"+ ? packageRoot+ : path.join(modules, `codex-${process.platform}-${process.arch}`);+ const nativeRoot = path.join(nativePackage, "vendor", "synthetic-target");+ const native = path.join(nativeRoot, "bin", "codex");+ const tools = path.join(nativeRoot, "codex-path");+ const launcher = path.join(packageRoot, "bin", "codex.js");+ fs.mkdirSync(path.dirname(launcher), { recursive: true });+ fs.mkdirSync(path.dirname(native), { recursive: true });+ fs.mkdirSync(tools);+ fs.writeFileSync(+ path.join(packageRoot, "package.json"),+ JSON.stringify({+ name: "@openai/codex",+ bin: { codex: "bin/codex.js" },+ }),+ );+ if (name !== "vendored")+ fs.writeFileSync(path.join(nativePackage, "package.json"), "{}");+ fs.copyFileSync(fakeCodex, native);+ fs.writeFileSync(+ launcher,+ `#!${process.execPath}+const child = require('node:child_process').spawnSync(${JSON.stringify(native)}, process.argv.slice(2), {stdio:'inherit',env:process.env});+process.exit(child.status ?? 1);+`,+ { mode: 0o755 },+ );+ return {+ launcher,+ native,+ tools,+ configured: name === "vendored" ? [path.dirname(native), tools] : [],+ };+ });+ const packaged = await load(nodes[0], "{{selected_codex}}");+ try {+ await Promise.all(+ packages.map(async ({ launcher, native, tools, configured }) => {+ const result = await packaged.callApi("synthetic", {+ vars: { selected_codex: launcher },+ prompt: {+ raw: "synthetic",+ label: "synthetic",+ config: { additional_directories: configured },+ },+ });+ assert.equal(result.error, undefined);+ const captured = JSON.parse(String(result.output));+ assert.equal(captured.executable, native);+ assert.equal(captured.path, process.env.PATH);+ assert.deepEqual(captured.directories, [+ ...configured,+ path.dirname(nodes[0]),+ path.dirname(nodes[0]),+ path.dirname(launcher),+ ...(configured.length ? [] : [path.dirname(native), tools]),+ ]);+ }),+ );+ } finally {+ await packaged.cleanup?.();+ }+ const mixed = await load("{{prefix}}/{{env.NODE_BASENAME}}");+ try {+ const result = await mixed.callApi("synthetic", {+ vars: { prefix: path.dirname(nodes[1]) },+ prompt: { raw: "synthetic", label: "synthetic" },+ });+ assert.equal(JSON.parse(String(result.output)).node, nodes[1]);+ } finally {+ await mixed.cleanup?.();+ }+ for (const [ambient, template, value, expected] of [+ [undefined, undefined, "", process.execPath],+ ["", undefined, "", process.execPath],+ [nodes[0], "", "", process.execPath],+ [nodes[0], "{{custom_node}}", "", process.execPath],+ [undefined, "{{missing_node}}", "", process.execPath],+ [nodes[0], undefined, "", nodes[0]],+ [nodes[0], "{{custom_node}}", nodes[1]!, nodes[1]],+ ] as const) {+ if (ambient === undefined) delete process.env.CODEX_MCP_NODE_PATH;+ else process.env.CODEX_MCP_NODE_PATH = ambient;+ const provider = await load(template);+ try {+ const result = await provider.callApi("synthetic", {+ vars: { custom_node: value },+ prompt: { raw: "synthetic", label: "synthetic" },+ });+ assert.equal(result.error, undefined);+ const captured = JSON.parse(String(result.output));+ const selected = fs.realpathSync(expected!);+ assert.equal(captured.node, selected);+ assert.deepEqual(captured.directories, [+ path.dirname(selected),+ path.dirname(captured.node),+ path.dirname(fakeCodex),+ ]);+ } finally {+ await provider.cleanup?.();+ }+ }+ const withoutContext = await load(nodes[0]);+ try {+ const result = await withoutContext.callApi("synthetic");+ const captured = JSON.parse(String(result.output));+ assert.equal(captured.node, nodes[0]);+ assert.equal(captured.marker, "{{marker}}");+ } finally {+ await withoutContext.cleanup?.();+ }+ process.env.PROMPTFOO_DISABLE_TEMPLATING = "true";+ const disabled = await load(nodes[2]!);+ try {+ const result = await disabled.callApi("synthetic", {+ vars: { custom_node: "other" },+ prompt: { raw: "synthetic", label: "synthetic" },+ });+ assert.equal(JSON.parse(String(result.output)).node, nodes[2]);+ } finally {+ await disabled.cleanup?.();+ }+ },+);++test(+ "saved native-provider configs retain legacy paths through retry, resume, and viewer replay",+ { skip: process.platform === "win32", timeout: 120000 },+ async (t) => {+ const root = fs.realpathSync(+ fs.mkdtempSync(path.join(os.tmpdir(), "legacy-provider-")),+ );+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ // Keep the saved source layout independent of the checkout and its path.+ const legacySourceRoot = path.join(root, "source.checkout");+ const legacyEvalRoot = path.join(+ legacySourceRoot,+ "evals",+ "triage-finding",+ );+ const legacyTransform = path.join(+ legacyEvalRoot,+ "scripts",+ "runtime-vars.mts",+ );+ fs.mkdirSync(path.dirname(legacyTransform), { recursive: true });+ fs.copyFileSync(+ path.join(import.meta.dirname, "runtime-vars.mts"),+ legacyTransform,+ );+ const skill = path.join(+ legacySourceRoot,+ "plugins",+ "codex-security",+ "skills",+ "triage-finding",+ "SKILL.md",+ );+ fs.mkdirSync(path.dirname(skill), { recursive: true });+ fs.writeFileSync(skill, "Synthetic legacy skill fixture.\n");+ const fixture = path.join(+ legacyEvalRoot,+ "fixtures",+ "repo",+ "src",+ "server.js",+ );+ fs.mkdirSync(path.dirname(fixture), { recursive: true });+ fs.writeFileSync(fixture, "// Synthetic repository fixture.\n");+ const capture = path.join(root, "captures.jsonl");+ const failure = path.join(root, "failure");+ const fakeCodex = path.join(root, "codex");+ fs.writeFileSync(+ fakeCodex,+ `#!${process.execPath}+const fs = require('node:fs');+const path = require('node:path');+const cwd = process.argv[process.argv.indexOf('--cd') + 1];+const prompt = fs.readFileSync(0, 'utf8');+const target = prompt.match(/^TARGET=(.+)$/m)[1];+const skill = fs.readFileSync(path.join(cwd, 'skills/triage-finding/SKILL.md'), 'utf8');+fs.readFileSync(path.join(target, 'src/server.js'), 'utf8');+fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,target,skillLoaded:skill.length>0,nodePath:process.env.CODEX_MCP_NODE_PATH,directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])})+'\\n');+console.log(JSON.stringify({type:'thread.started',thread_id:'synthetic-legacy'}));+if (fs.existsSync(${JSON.stringify(failure)})) {+ console.log(JSON.stringify({type:'turn.failed',error:{message:'synthetic legacy retryable failure'}}));+} else {+ console.log(JSON.stringify({type:'item.completed',item:{id:'message',type:'agent_message',text:'ok'}}));+ console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_tokens:1}}));+}+`,+ { mode: 0o755 },+ );+ const configPath = path.join(root, "config.json");+ // These are the native-provider placeholders saved by the previous configs.+ // No startup extension is available when Promptfoo reloads those records.+ const provider = {+ id: "openai:codex-sdk:gpt-5.5",+ config: {+ working_dir: "{{triage_runtime_root}}",+ codex_path_override: fakeCodex,+ skip_git_repo_check: true,+ approval_policy: "never",+ additional_directories: [+ "{{triage_fixture_root}}",+ "{{triage_node_root}}",+ ],+ cli_env: { CODEX_MCP_NODE_PATH: "{{triage_node_path}}" },+ },+ };+ fs.writeFileSync(+ configPath,+ JSON.stringify({+ providers: [provider],+ prompts: [+ "Read and follow skills/triage-finding/SKILL.md.\nTARGET={{target_repo}}",+ ],+ defaultTest: {+ options: {+ transformVars: `file://${legacyTransform}`,+ },+ },+ tests: [+ {+ vars: { target_repo: "evals/triage-finding/fixtures/repo" },+ assert: [{ type: "equals", value: "ok" }],+ },+ ],+ }),+ );+ const environment = {+ OPENAI_API_KEY: "synthetic-test-key",+ PROMPTFOO_CONFIG_DIR: path.join(root, "state"),+ PROMPTFOO_DISABLE_WAL_MODE: "true",+ PROMPTFOO_DISABLE_TELEMETRY: "1",+ PROMPTFOO_DISABLE_UPDATE: "1",+ };+ fs.writeFileSync(failure, "");+ const initial = await invoke(+ [+ "eval",+ "-c",+ configPath,+ "--no-cache",+ "--no-share",+ "--no-progress-bar",+ ],+ environment,+ );+ assert.notEqual(initial.code, 0, initial.output);+ assert.match(initial.output, /synthetic legacy retryable failure/);+ fs.rmSync(failure);+ const retry = await invoke(+ [+ "eval",+ "--retry-errors",+ "--no-cache",+ "--no-share",+ "--no-progress-bar",+ ],+ environment,+ );+ assert.equal(retry.code, 0, retry.output);+ const database = new DatabaseSync(+ path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),+ { enableForeignKeyConstraints: false },+ );+ const stored = database+ .prepare("SELECT id, config FROM evals ORDER BY created_at DESC LIMIT 1")+ .get();+ assert.ok(stored && typeof stored.config === "string");+ const saved = JSON.parse(stored.config);+ assert.equal(saved.providers[0].id, provider.id);+ assert.equal(+ saved.providers[0].config.working_dir,+ "{{triage_runtime_root}}",+ );+ assert.equal(saved.extensions?.length || 0, 0);+ database.exec("DELETE FROM eval_results");+ database.close();+ const resumed = await invoke(+ ["eval", "--resume", "--no-cache", "--no-share", "--no-progress-bar"],+ environment,+ );+ assert.equal(resumed.code, 0, resumed.output);+ const resumedDatabase = new DatabaseSync(+ path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),+ { enableForeignKeyConstraints: false },+ );+ const resumedResult = resumedDatabase+ .prepare(+ "SELECT metadata, test_case FROM eval_results WHERE eval_id = ? LIMIT 1",+ )+ .get(String(stored.id));+ resumedDatabase.close();+ assert.ok(resumedResult && typeof resumedResult.test_case === "string");+ const replayVariables =+ JSON.parse((resumedResult.metadata as string) || "{}").inputVars ||+ JSON.parse(resumedResult.test_case).vars;+ const listener = createServer();+ await new Promise<void>((resolve) =>+ listener.listen(0, "127.0.0.1", resolve),+ );+ const address = listener.address();+ assert.ok(address && typeof address !== "string");+ const port = address.port;+ await new Promise<void>((resolve, reject) =>+ listener.close((error) => (error ? reject(error) : resolve())),+ );+ const viewer = spawn(+ process.execPath,+ [+ "--experimental-strip-types",+ runner,+ "view",+ "--port",+ String(port),+ "--no",+ ],+ {+ cwd: evalRoot,+ env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },+ stdio: ["ignore", "pipe", "pipe"],+ },+ );+ let viewerOutput = "";+ for (const stream of [viewer.stdout, viewer.stderr])+ stream.on("data", (chunk) => {+ viewerOutput += chunk;+ });+ const closed = new Promise<number | null>((resolve, reject) => {+ viewer.once("error", reject);+ viewer.once("close", resolve);+ });+ t.after(async () => {+ if (viewer.exitCode === null) viewer.kill("SIGTERM");+ await closed;+ });+ const url = `http://127.0.0.1:${port}`;+ while (true) {+ try {+ if ((await fetch(`${url}/api/eval`)).ok) break;+ } catch {}+ assert.equal(viewer.exitCode, null, viewerOutput);+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ const replay = await fetch(`${url}/api/eval/replay`, {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({+ evaluationId: stored.id,+ testIndex: 0,+ prompt:+ "Read and follow skills/triage-finding/SKILL.md.\nTARGET=" ++ replayVariables.target_repo,+ variables: replayVariables,+ }),+ });+ assert.equal(replay.status, 200, viewerOutput);+ const replayResult = (await replay.json()) as { output: unknown };+ assert.equal(replayResult.output, "ok", JSON.stringify(replayResult));+ viewer.kill("SIGTERM");+ await closed;+ const rows = fs+ .readFileSync(capture, "utf8")+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line));+ assert.equal(rows.length, 4);+ assert.equal(new Set(rows.map((row) => row.cwd)).size, 1);+ const nodePath = fs.realpathSync(process.execPath);+ for (const row of rows) {+ const runtimeRoot = legacySourceRoot;+ const fixtureRoot = path.join(+ runtimeRoot,+ "evals",+ "triage-finding",+ "fixtures",+ );+ assert.equal(+ row.cwd,+ path.join(runtimeRoot, "plugins", "codex-security"),+ );+ assert.equal(row.target, path.join(fixtureRoot, "repo"));+ assert.equal(row.nodePath, nodePath);+ assert.deepEqual(row.directories, [fixtureRoot, path.dirname(nodePath)]);+ assert.equal(row.skillLoaded, true);+ assert.equal(fs.existsSync(row.cwd), true);+ }+ },+);++for (const suite of ["SAST", "calibration"] as const) {+ test(+ `${suite} provider survives persisted retry, resume, and viewer replay`,+ { skip: process.platform === "win32", timeout: 120000 },+ async (t) => {+ const root = fs.realpathSync(+ fs.mkdtempSync(path.join(os.tmpdir(), "sast-provider-replay-")),+ );+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const targetRoot = fs.realpathSync(+ fs.mkdtempSync(path.join(os.tmpdir(), "sast-custom-target-")),+ );+ t.after(() => fs.rmSync(targetRoot, { recursive: true, force: true }));+ const targetRepository = path.join(targetRoot, "selected-case");+ const siblingRepository = path.join(targetRoot, "other-case");+ fs.mkdirSync(targetRepository);+ fs.mkdirSync(siblingRepository);+ const capture = path.join(root, "captures.jsonl");+ const failure = path.join(root, "failure");+ const fakeCodex = path.join(root, "codex");+ fs.writeFileSync(+ fakeCodex,+ `#!${process.execPath}+const fs = require('node:fs');+const cwd = process.argv[process.argv.indexOf('--cd') + 1];+fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({ cwd, directories: process.argv.flatMap((arg, index) => arg === '--add-dir' ? [process.argv[index + 1]] : []) }) + '\\n');+console.log(JSON.stringify({ type: 'thread.started', thread_id: 'synthetic-sast-replay' }));+if (fs.existsSync(${JSON.stringify(failure)})) {+ console.log(JSON.stringify({ type: 'turn.failed', error: { message: 'synthetic retryable failure' } }));+} else {+ console.log(JSON.stringify({ type: 'item.completed', item: { id: 'message', type: 'agent_message', text: 'ok' } }));+ console.log(JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } }));+}+`,+ { mode: 0o755 },+ );+ const environment = {+ OPENAI_API_KEY: "synthetic-test-key",+ CODEX_MCP_NODE_PATH: process.execPath,+ PROMPTFOO_CONFIG_DIR: path.join(root, "state"),+ PROMPTFOO_DISABLE_WAL_MODE: "true",+ PROMPTFOO_DISABLE_TELEMETRY: "1",+ PROMPTFOO_DISABLE_UPDATE: "1",+ };+ const template = parse(+ fs.readFileSync(+ path.join(+ evalRoot,+ suite === "SAST"+ ? "sastbench/promptfooconfig.sastbench.yaml"+ : "promptfooconfig.calibration.yaml",+ ),+ "utf8",+ ),+ );+ const provider = template.providers[0];+ provider.config.codex_path_override = fakeCodex;+ // Keep the real YAML provider reference and its original configuration directory.+ const configPath = path.join(+ evalRoot,+ suite === "SAST" ? "sastbench" : ".",+ `.provider-replay-${process.pid}.json`,+ );+ t.after(() => fs.rmSync(configPath, { force: true }));+ fs.writeFileSync(+ configPath,+ JSON.stringify({+ extensions: template.extensions,+ providers: [provider],+ prompts: ["hello"],+ defaultTest:+ suite === "calibration"+ ? {+ options: {+ transformVars: `file://${path.join(import.meta.dirname, "runtime-vars.mts")}`,+ },+ }+ : undefined,+ tests: [+ {+ metadata:+ suite === "SAST"+ ? { ground_truth: "true_positive" }+ : undefined,+ vars:+ suite === "SAST"+ ? { target_repo: targetRepository }+ : {+ calibration_repo: "calibration-0123456789abcdef",+ calibration_repo_root: "",+ },+ assert: [{ type: "equals", value: "ok" }],+ },+ ],+ }),+ );+ fs.writeFileSync(failure, "");+ const initial = await invoke(+ [+ "eval",+ "-c",+ configPath,+ "--no-cache",+ "--no-share",+ "--no-progress-bar",+ ],+ environment,+ );+ assert.notEqual(initial.code, 0, initial.output);+ assert.match(initial.output, /synthetic retryable failure/);+ if (suite === "SAST") {+ const initialDatabase = new DatabaseSync(+ path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),+ { enableForeignKeyConstraints: false },+ );+ const initialResult = initialDatabase+ .prepare("SELECT metadata FROM eval_results LIMIT 1")+ .get();+ initialDatabase.close();+ assert.ok(+ JSON.parse((initialResult?.metadata as string) || "{}").sastbench,+ initial.output,+ );+ }+ fs.rmSync(failure);+ const retry = await invoke(+ [+ "eval",+ "--retry-errors",+ "--no-cache",+ "--no-share",+ "--no-progress-bar",+ ],+ environment,+ );+ assert.equal(retry.code, 0, retry.output);+ const database = new DatabaseSync(+ path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),+ { enableForeignKeyConstraints: false },+ );+ const stored = database+ .prepare(+ "SELECT id, config FROM evals ORDER BY created_at DESC LIMIT 1",+ )+ .get();+ assert.ok(stored && typeof stored.config === "string");+ assert.equal(JSON.parse(stored.config).providers[0].id, provider.id);+ assert.deepEqual(+ JSON.parse(stored.config).extensions,+ template.extensions.map((extension: string) =>+ extension.replace(+ "{{env.TRIAGE_PROVIDER_PATH}}",+ path.join(import.meta.dirname, "triage-provider.mts"),+ ),+ ),+ );+ database.prepare("DELETE FROM eval_results").run();+ database.close();+ const resumed = await invoke(+ ["eval", "--resume", "--no-cache", "--no-share", "--no-progress-bar"],+ environment,+ );+ assert.equal(resumed.code, 0, resumed.output);++ const resumedDatabase = new DatabaseSync(+ path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),+ { enableForeignKeyConstraints: false },+ );+ const resumedResult = resumedDatabase+ .prepare(+ "SELECT metadata, test_case FROM eval_results WHERE eval_id = ? LIMIT 1",+ )+ .get(String(stored.id));+ resumedDatabase.close();+ assert.ok(resumedResult && typeof resumedResult.test_case === "string");+ // Match the viewer: prefer saved inputVars, then the resolved test-case vars.+ const replayVariables =+ JSON.parse((resumedResult.metadata as string | null) ?? "{}")+ .inputVars || JSON.parse(resumedResult.test_case).vars;+ assert.equal(typeof replayVariables.target_repo, "string");++ const listener = createServer();+ await new Promise<void>((resolve) =>+ listener.listen(0, "127.0.0.1", resolve),+ );+ const address = listener.address();+ assert.ok(address && typeof address !== "string");+ const port = address.port;+ await new Promise<void>((resolve, reject) =>+ listener.close((error) => (error ? reject(error) : resolve())),+ );+ const viewer = spawn(+ process.execPath,+ [+ "--experimental-strip-types",+ runner,+ "view",+ "--port",+ String(port),+ "--no",+ ],+ {+ cwd: evalRoot,+ env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },+ stdio: ["ignore", "pipe", "pipe"],+ },+ );+ let viewerOutput = "";+ for (const stream of [viewer.stdout, viewer.stderr])+ stream.on("data", (chunk) => {+ viewerOutput += chunk;+ });+ const closed = new Promise<number | null>((resolve, reject) => {+ viewer.once("error", reject);+ viewer.once("close", resolve);+ });+ t.after(async () => {+ if (viewer.exitCode === null) viewer.kill("SIGTERM");+ await closed;+ });+ const url = `http://127.0.0.1:${port}`;+ while (true) {+ try {+ const response = await fetch(`${url}/api/eval`);+ if (response.ok) break;+ } catch {+ /* The actual viewer is still building its staged runtime. */+ }+ assert.equal(viewer.exitCode, null, viewerOutput);+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ const replay = await fetch(`${url}/api/eval/replay`, {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({+ evaluationId: stored.id,+ testIndex: 0,+ prompt: "hello",+ variables: replayVariables,+ }),+ });+ assert.equal(replay.status, 200, viewerOutput);+ const replayResult = (await replay.json()) as { output: unknown };+ assert.equal(replayResult.output, "ok", JSON.stringify(replayResult));+ viewer.kill("SIGTERM");+ await closed;+ assert.doesNotMatch(viewerOutput, /afterEach extension hook failed/);+ const rows = fs+ .readFileSync(capture, "utf8")+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line));+ assert.equal(rows.length, 4);+ assert.equal(new Set(rows.map((row) => row.cwd)).size, 4);+ for (const row of rows) {+ assert.equal(fs.existsSync(row.cwd), false);+ assert.deepEqual(row.directories, [+ ...(suite === "SAST"+ ? [+ path.join(evalRoot, "artifacts", "sastbench-git-cache"),+ targetRepository,+ ]+ : [+ path.join(+ evalRoot,+ "artifacts",+ "calibration-repos",+ "calibration-0123456789abcdef",+ ),+ ]),+ path.dirname(fs.realpathSync(process.execPath)),+ path.dirname(process.execPath),+ path.dirname(fakeCodex),+ ]);+ assert.ok(+ row.directories.every(+ (directory: string) =>+ siblingRepository !== directory &&+ !siblingRepository.startsWith(directory + path.sep),+ ),+ );+ }+ },+ );+}diff --git a/evals/triage-finding/scripts/triage-provider.mts b/evals/triage-finding/scripts/triage-provider.mtsnew file mode 100644index 000000000..f8fa3ac73--- /dev/null+++ b/evals/triage-finding/scripts/triage-provider.mts@@ -0,0 +1,163 @@+import { execFileSync } from "node:child_process";+import {+ accessSync,+ constants,+ readFileSync,+ realpathSync,+ statSync,+} from "node:fs";+import path from "node:path";+import type { ApiProvider, ProviderOptions } from "promptfoo";+import {+ bundledCodexSdkEnvironment,+ resolveBundledCodexExecutable,+} from "../../../sdk/typescript/dist/codex-sdk-environment.js";++type CodexProvider = ApiProvider & {+ getCodexInstanceForTurn(+ environment: NodeJS.ProcessEnv,+ config: ProviderOptions["config"],+ apiKey?: string,+ ): Promise<{+ activeInstance: { exec: { executablePath: string; pathDirs: string[] } };+ }>;+ callApiInternal(+ prompt: Parameters<ApiProvider["callApi"]>[0],+ context: Parameters<ApiProvider["callApi"]>[1],+ options: Parameters<ApiProvider["callApi"]>[2],+ config: ProviderOptions["config"],+ ): ReturnType<ApiProvider["callApi"]>;+};++function resolveCommand(+ command: string,+ cwd: string,+ environment: NodeJS.ProcessEnv,+) {+ if (+ command.includes("/") ||+ (process.platform === "win32" && command.includes("\\"))+ )+ return process.platform === "win32"+ ? path.resolve(cwd, command)+ : path.isAbsolute(command)+ ? command+ : `${cwd}/${command}`;+ if (process.platform === "win32") {+ return execFileSync(+ path.join(process.env.SystemRoot!, "System32", "where.exe"),+ [command],+ { cwd, env: environment, encoding: "utf8" },+ )+ .trim()+ .split(/\r?\n/)[0];+ }+ for (const directory of (environment.PATH ?? "/usr/bin:/bin").split(+ path.delimiter,+ )) {+ const candidate = `${path.isAbsolute(directory) ? directory : `${cwd}/${directory}`}/${command}`;+ try {+ accessSync(candidate, constants.X_OK);+ if (statSync(candidate).isFile()) return candidate;+ } catch {+ // Continue the same executable search used by native spawn.+ }+ }+ throw new Error(`Executable not found on PATH: ${command}`);+}++const wrappedProviders = new WeakSet<ApiProvider>();++// Keep native provider loading and authentication unchanged. Bind the staged+// runtime after Promptfoo has created each provider for this evaluation.+export function beforeAll({ suite }: { suite: { providers: ApiProvider[] } }) {+ for (const apiProvider of suite.providers) {+ const provider = apiProvider as CodexProvider;+ if (+ typeof provider.getCodexInstanceForTurn !== "function" ||+ wrappedProviders.has(provider)+ )+ continue;+ const getInstance = provider.getCodexInstanceForTurn.bind(provider);+ provider.getCodexInstanceForTurn = async (environment, config, apiKey) => {+ const instance = await getInstance(environment, config, apiKey);+ // The pinned provider exposes its selected SDK executor before creating+ // thread options. Extend this call's config without changing selection+ // or the SDK's bundled-tool PATH handling.+ const { executablePath, pathDirs } = instance.activeInstance.exec;+ const executable = realpathSync.native(+ resolveCommand(executablePath, process.cwd(), environment),+ );+ const directories = [path.dirname(executable), ...pathDirs];+ if (path.basename(executable) === "codex.js") {+ try {+ const packageJson = path.join(+ path.dirname(executable),+ "..",+ "package.json",+ );+ if (+ JSON.parse(readFileSync(packageJson, "utf8")).name ===+ "@openai/codex"+ ) {+ const native = resolveBundledCodexExecutable(packageJson);+ directories.push(+ path.dirname(native),+ ...Object.values(bundledCodexSdkEnvironment(native, {})),+ );+ }+ } catch {+ // Other launcher layouts keep their configured directories and+ // report their own dependency errors when the SDK starts them.+ }+ }+ config.additional_directories = [+ ...(config.additional_directories ?? []),+ ...directories,+ ];+ return instance;+ };+ const callApiInternal = provider.callApiInternal.bind(provider);+ // The pinned Codex provider merges prompt overrides and renders case+ // variables before this call. Keep that upstream behavior for each case.+ provider.callApiInternal = (prompt, context, options, config) => {+ const runtimeRoot = process.env.TRIAGE_RUNTIME_ROOT;+ if (!runtimeRoot) {+ throw new Error(+ "Run this evaluation through scripts/run-promptfoo.mts.",+ );+ }+ const requestedNode =+ (config.cli_env?.CODEX_MCP_NODE_PATH ??+ process.env.CODEX_MCP_NODE_PATH ??+ process.execPath) ||+ process.execPath;+ const environment = { ...process.env, ...config.cli_env };+ let nodeCommand: string;+ try {+ nodeCommand = resolveCommand(requestedNode, runtimeRoot, environment);+ accessSync(+ nodeCommand,+ process.platform === "win32" ? constants.F_OK : constants.X_OK,+ );+ } catch {+ nodeCommand = resolveCommand("node", runtimeRoot, environment);+ }+ const nodePath =+ process.platform === "win32"+ ? realpathSync(nodeCommand)+ : realpathSync.native(nodeCommand);+ return callApiInternal(prompt, context, options, {+ ...config,+ working_dir: runtimeRoot,+ cli_env: { ...config.cli_env, CODEX_MCP_NODE_PATH: nodeCommand },+ additional_directories: [+ ...(config.additional_directories ?? []),+ path.dirname(nodePath),+ path.dirname(nodeCommand),+ ],+ });+ };+ wrappedProviders.add(provider);+ }+}diff --git a/plugins/codex-security/mcp-app/scripts/build_native_wrappers.mjs b/plugins/codex-security/mcp-app/scripts/build_native_wrappers.mjsindex c22cd0994..747d85d1e 100644--- a/plugins/codex-security/mcp-app/scripts/build_native_wrappers.mjs+++ b/plugins/codex-security/mcp-app/scripts/build_native_wrappers.mjs@@ -1,9 +1,11 @@+import { randomUUID } from "node:crypto";+import { rename, rm, writeFile } from "node:fs/promises";import { resolve } from "node:path";import { build } from "esbuild";export async function buildNativeWrappers() {const root = resolve(import.meta.dirname, "../../native");- await build({+ const result = await build({absWorkingDir: root,entryPoints: ["*.mts"],outdir: root,@@ -11,5 +13,18 @@ export async function buildNativeWrappers() {format: "esm",platform: "node",target: "node20",+ write: false,});+ // Concurrent runtime builds must never import a partially written wrapper.+ await Promise.all(+ result.outputFiles.map(async (file) => {+ const temporary = `${file.path}.${randomUUID()}.tmp`;+ try {+ await writeFile(temporary, file.contents);+ await rename(temporary, file.path);+ } finally {+ await rm(temporary, { force: true });+ }+ }),+ );}diff --git a/plugins/codex-security/mcp-app/tests/support/reducer-paging/deep-reducer-paging.ts b/plugins/codex-security/mcp-app/tests/support/reducer-paging/deep-reducer-paging.tsindex 1c23832ee..879431970 100644--- a/plugins/codex-security/mcp-app/tests/support/reducer-paging/deep-reducer-paging.ts+++ b/plugins/codex-security/mcp-app/tests/support/reducer-paging/deep-reducer-paging.ts@@ -21,16 +21,34 @@ import {gradeReducerPagingResult,} from "./deep-reducer-paging-fixture.ts";+interface ReducerPagingEvalOptions {+ root: string;+ mode?: "deterministic" | "model";+ model?: string;+}+/** Run one reducer through real code-mode IPC and the production artifact tools. */-export async function runReducerPagingEval({+export async function runReducerPagingEval(options: ReducerPagingEvalOptions) {+ try {+ const result = await runReducer(options);+ await writeJson(path.join(options.root, "report.json"), result.report);+ return result;+ } catch (error) {+ await writeJson(path.join(options.root, "report.json"), {+ mode: options.mode ?? "deterministic",+ ...(options.model ? { model: options.model } : {}),+ error: error instanceof Error ? error.stack : String(error),+ }).catch(() => {});+ throw error;+ }+}++async function runReducer({root,mode = "deterministic",model,-}: {- root: string;- mode?: "deterministic" | "model";- model?: string;-}) {+}: ReducerPagingEvalOptions) {+ assert.ok(mode === "deterministic" || mode === "model");const fixture = await createReducerPagingFixture(root);const tracePath = path.join(root, "tool-trace.jsonl");const serverConfigPath = path.join(root, "server-config.json");@@ -111,6 +129,7 @@ export async function runReducerPagingEval({const trace = (await readFile(tracePath, "utf8")).trim().split("\n")+ .filter(Boolean).map((line) => JSON.parse(line));const report = {mode,@@ -121,10 +140,6 @@ export async function runReducerPagingEval({...(await gradeReducerPagingResult(fixture)),...(usage ? { usage } : {}),};- await writeFile(- path.join(root, "report.json"),- JSON.stringify(report, null, 2),- );return { report, fixture };}@@ -133,6 +148,7 @@ export function gradeReducerPagingTrace(trace: ReducerTraceEvent[],ipcFrameLimitBytes: number,) {+ assert.ok(trace.length > 0, "The reducer produced no tool trace.");const reads = trace.filter((event) =>event.event === "request" &&diff --git a/plugins/codex-security/mcp-app/tests/support/temporary-directories.ts b/plugins/codex-security/mcp-app/tests/support/temporary-directories.tsindex 00d490a5c..63711e67a 100644--- a/plugins/codex-security/mcp-app/tests/support/temporary-directories.ts+++ b/plugins/codex-security/mcp-app/tests/support/temporary-directories.ts@@ -2,7 +2,11 @@ import { mkdtemp, realpath, rm } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";-export function createTemporaryDirectories(canonicalize = false) {+export function createTemporaryDirectories(+ canonicalize = false,+ removeDirectory = (root: string) =>+ rm(root, { recursive: true, force: true }),+) {const roots: string[] = [];return {track(root: string): void {@@ -14,11 +18,7 @@ export function createTemporaryDirectories(canonicalize = false) {return root;},async cleanup(): Promise<void> {- await Promise.all(- roots- .splice(0)- .map((root) => rm(root, { recursive: true, force: true })),- );+ await Promise.all(roots.splice(0).map(removeDirectory));},};}diff --git a/plugins/codex-security/mcp-app/tests/test_build_native_wrappers.ts b/plugins/codex-security/mcp-app/tests/test_build_native_wrappers.tsnew file mode 100644index 000000000..2eb070ae6--- /dev/null+++ b/plugins/codex-security/mcp-app/tests/test_build_native_wrappers.ts@@ -0,0 +1,56 @@+import assert from "node:assert/strict";+import { spawn } from "node:child_process";+import test from "node:test";++const builder = new URL(+ "../scripts/build_native_wrappers.mjs",+ import.meta.url,+);+const platform = new URL("../../native/platform.mjs", import.meta.url);+const { buildNativeWrappers } = await import(builder.href);++test("concurrent runtime builds preserve complete native wrapper exports", async () => {+ await buildNativeWrappers();+ const { nativeTarget } = await import(platform.href);+ assert.equal(typeof nativeTarget, "string");+ const source = `+ import assert from 'node:assert/strict';+ import { buildNativeWrappers } from ${JSON.stringify(builder.href)};+ for (let iteration = 0; iteration < 15; iteration++) {+ await buildNativeWrappers();+ const platform = await import(${JSON.stringify(platform.href)} + '?build=' + iteration);+ assert.equal(platform.nativeTarget, ${JSON.stringify(nativeTarget)});+ }+ `;+ await Promise.all(+ Array.from(+ { length: 3 },+ () =>+ new Promise<void>((resolve, reject) => {+ const child = spawn(+ process.execPath,+ ["--input-type=module", "--eval", source],+ {+ stdio: ["ignore", "pipe", "pipe"],+ },+ );+ let output = "";+ child.stdout.on("data", (data) => {+ output += data;+ });+ child.stderr.on("data", (data) => {+ output += data;+ });+ child.once("error", reject);+ child.once("close", (code) => {+ try {+ assert.equal(code, 0, output);+ resolve();+ } catch (error) {+ reject(error);+ }+ });+ }),+ ),+ );+});diff --git a/plugins/codex-security/mcp-app/tests/test_deep_reducer_paging_eval.ts b/plugins/codex-security/mcp-app/tests/test_deep_reducer_paging_eval.tsindex f6dd2e697..caaac04a3 100644--- a/plugins/codex-security/mcp-app/tests/test_deep_reducer_paging_eval.ts+++ b/plugins/codex-security/mcp-app/tests/test_deep_reducer_paging_eval.ts@@ -1,6 +1,6 @@import { readJson, readJsonLines } from "./support/json.ts";import assert from "node:assert/strict";-import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";import test from "node:test";@@ -67,3 +67,53 @@ test("a reducer recovers from the real IPC frame limit and records all sources",/distinct|independent/,);});++for (const failure of ["synthetic model failure", "empty trace"]) {+ test(`retains a diagnostic report for ${failure}`, async (t) => {+ const { Codex } = await import("@openai/codex-sdk");+ const root = await mkdtemp(+ path.join(tmpdir(), "deep-reducer-failed-eval-"),+ );+ t.after(() => rm(root, { recursive: true, force: true }));+ const thread = {+ run: async () => {+ if (failure !== "empty trace") throw new Error(failure);+ return { finalResponse: "No tools called." };+ },+ } as unknown as ReturnType<typeof Codex.prototype.startThread>;+ t.mock.method(Codex.prototype, "startThread", () => thread);+ const expected =+ failure === "empty trace"+ ? /produced no tool trace/+ : /synthetic model failure/;+ await assert.rejects(+ runReducerPagingEval({ root, mode: "model" }),+ expected,+ );+ const report = JSON.parse(+ await readFile(path.join(root, "report.json"), "utf8"),+ );+ assert.equal(report.mode, "model");+ assert.match(report.error, expected);+ assert.equal(report.accountedSourceCount, undefined);+ });+}++test("preserves the reducer error when writing its report also fails", async (t) => {+ const root = await mkdtemp(+ path.join(tmpdir(), "deep-reducer-report-failure-"),+ );+ t.after(() => rm(root, { recursive: true, force: true }));+ const { Codex } = await import("@openai/codex-sdk");+ const failure = new Error("Synthetic reducer failure");+ t.mock.method(Codex.prototype, "startThread", () => ({+ run: async () => {+ await rm(root, { recursive: true, force: true });+ throw failure;+ },+ }));+ await assert.rejects(+ runReducerPagingEval({ root, mode: "model" }),+ (error) => error === failure,+ );+});diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.tsindex 22d4b3aa5..157b2b289 100644--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts@@ -1351,6 +1351,7 @@ async function testWorkerRuntimeSettings() {"GIT_SSH_COMMAND","GIT_CONFIG_GLOBAL","CODEX_CLI_PATH",+ "CODEX_MCP_NODE_PATH","CODEX_HOME","CODEX_SECURITY_CONFIG_PATH","CODEX_SECURITY_PLUGIN_ROOT",@@ -1742,6 +1743,10 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,);}) as typeof childProcess.spawn;syncBuiltinESMExports();+ const nodePaths = settings.map((_, index) =>+ path.join(fixture.root, `selected node ${index}`, "node"),+ );+const executors = settings.map((modelSettings, index) =>new CodexSdkWorkerExecutor({@@ -1762,6 +1767,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,await Promise.all(executors.map((executor, index) => {// Each concurrent launch snapshots its own scan environment.+ process.env.CODEX_MCP_NODE_PATH = nodePaths[index];process.env.CODEX_SECURITY_CONFIG_PATH =workerConfigurations[index].path;process.env.CODEX_SECURITY_KNOWLEDGE_BASE =@@ -1885,7 +1891,10 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,);assert.equal(invocation.codexHome, await realpath(codexHome));assert.equal(invocation.providerKey, providerKeys[index]);+ assert.equal(invocation.mcpNodePath, nodePaths[index]);+ assert.equal(process.env.CODEX_MCP_NODE_PATH, nodePaths.at(-1));assert.equal(invocation.providerHeader, providerHeaders[index]);+assert.equal(workerLaunch.environment!.CODEX_API_KEY, undefined);assert.equal(workerLaunch.environment!.CODEX_SQLITE_HOME,@@ -2002,6 +2011,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,}assert.equal(invocation.python, python);assertConfigOverrides(invocation.argv, {+ "mcp_servers.cs_artifacts.command": process.execPath,"mcp_servers.cs_artifacts.env.CODEX_SECURITY_PYTHON_COMMAND": `${helperPython}-${index} `,"mcp_servers.cs_artifacts.args.0": path.join(fixture.root,@@ -3321,8 +3331,9 @@ if (pythonProbe && pythonProbe.status !== 0) throw new Error(pythonProbe.stderrconst pythonRuntime = pythonProbe ? JSON.parse(pythonProbe.stdout) : undefined;const knowledgePath = stdin.includes('synthetic worker configuration fixture') ? process.env.CODEX_SECURITY_KNOWLEDGE_BASE : undefined;const knowledgeDocuments = knowledgePath === undefined ? undefined : Object.fromEntries(readdirSync(knowledgePath).map(name => [name, readFileSync(join(knowledgePath, name), 'utf8')]));-writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ argv: process.argv.slice(2), stdin, cwd: process.cwd(), knowledgePath, knowledgeDocuments, codexHome: process.env.CODEX_HOME, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, bundledTool: toolProbe?.stdout.trim(), pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(profileContents === undefined ? {} : { profileContents }), ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) }));+writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ argv: process.argv.slice(2), stdin, cwd: process.cwd(), knowledgePath, knowledgeDocuments, codexHome: process.env.CODEX_HOME, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, mcpNodePath: process.env.CODEX_MCP_NODE_PATH, bundledTool: toolProbe?.stdout.trim(), pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(profileContents === undefined ? {} : { profileContents }), ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) }));if (stdin.includes('COMPLETE_THEN_HANG')) process.on('SIGTERM', () => { if (!stdin.includes('IGNORE_TERMINATION')) setTimeout(() => process.exit(0), 100); });+if (stdin.includes('THREAD_START_CONFIG_ERROR')) { console.error('Error: thread/start: thread/start failed: agents.max_threads cannot be set when features.multi_agent_v2 is enabled (code -32600)'); process.exit(1); }if (stdin.includes('CONFIG_ERROR')) { console.error('failed to load configuration: invalid value'); process.exit(2); }if (stdin.includes('MCP_STARTUP_TIMEOUT') || stdin.includes('CATALOG_AUTH_ONLY') || stdin.includes('SYNC_AUTH_ONLY')) {diff --git a/plugins/codex-security/scripts/launch_codex_security_mcp.cmd b/plugins/codex-security/scripts/launch_codex_security_mcp.cmdindex 4618e5583..c5bad6823 100644--- a/plugins/codex-security/scripts/launch_codex_security_mcp.cmd+++ b/plugins/codex-security/scripts/launch_codex_security_mcp.cmd@@ -15,8 +15,12 @@ if errorlevel 1 ():launch+if defined CODEX_MCP_NODE_PATH if exist "%CODEX_MCP_NODE_PATH%" (+ set "CODEX_SECURITY_MCP_NODE=%CODEX_MCP_NODE_PATH%"+ goto run+)rem WindowsApps can expose a Node path that exists but cannot be executed.-rem Prefer relocated user-writable runtimes before probing packaged paths.+rem Without an explicit override, prefer relocated user-writable runtimes.if defined LOCALAPPDATA for /d %%D in ("%LOCALAPPDATA%\OpenAI\Codex\runtimes\cua_node\*") do if exist "%%~fD\bin\node.exe" (set "CODEX_SECURITY_MCP_NODE=%%~fD\bin\node.exe"goto run@@ -29,10 +33,6 @@ if defined USERPROFILE if exist "%USERPROFILE%\.cache\codex-runtimes\codex-primaset "CODEX_SECURITY_MCP_NODE=%USERPROFILE%\.cache\codex-runtimes\codex-primary-runtime\dependencies\node\bin\node.exe"goto run)-if defined CODEX_MCP_NODE_PATH if exist "%CODEX_MCP_NODE_PATH%" (- set "CODEX_SECURITY_MCP_NODE=%CODEX_MCP_NODE_PATH%"- goto run-)if defined CODEX_BROWSER_USE_NODE_PATH if exist "%CODEX_BROWSER_USE_NODE_PATH%" (set "CODEX_SECURITY_MCP_NODE=%CODEX_BROWSER_USE_NODE_PATH%"goto rundiff --git a/sdk/typescript/src/codex-sdk-environment.ts b/sdk/typescript/src/codex-sdk-environment.tsindex d57316a7b..a1436d411 100644--- a/sdk/typescript/src/codex-sdk-environment.ts+++ b/sdk/typescript/src/codex-sdk-environment.ts@@ -1,5 +1,7 @@-import { realpathSync, statSync } from "node:fs";+import { existsSync, readdirSync, realpathSync, statSync } from "node:fs";+import { createRequire } from "node:module";import { delimiter, dirname, join } from "node:path";+import { PluginBootstrapError } from "./errors.js";export function bundledCodexSdkEnvironment(command: string,@@ -34,3 +36,40 @@ export function bundledCodexSdkEnvironment(result[pathKey] = [toolsDirectory, ...entries].join(delimiter);return result;}++export function resolveBundledCodexExecutable(+ codexPackageJson?: string,+): string {+ const platform = process.platform === "android" ? "linux" : process.platform;+ const packageName = `@openai/codex-${platform}-${process.arch}`;+ let packageJson: string;+ try {+ codexPackageJson ??= createRequire(import.meta.url).resolve(+ "@openai/codex/package.json",+ );+ packageJson = createRequire(codexPackageJson).resolve(+ `${packageName}/package.json`,+ );+ } catch (error) {+ throw new PluginBootstrapError(+ `The bundled Codex executable could not be resolved from ${packageName}. Reinstall @openai/codex with optional dependencies enabled, or set CODEX_CLI_PATH to an installed Codex executable.`,+ { cause: error },+ );+ }+ const vendor = join(dirname(packageJson), "vendor");+ const target = readdirSync(vendor, { withFileTypes: true }).find((entry) =>+ entry.isDirectory(),+ );+ const command = join(+ vendor,+ target?.name ?? "",+ "bin",+ process.platform === "win32" ? "codex.exe" : "codex",+ );+ if (target === undefined || !existsSync(command)) {+ throw new PluginBootstrapError(+ `The ${packageName} package does not contain the Codex executable. Reinstall @openai/codex with optional dependencies enabled, or set CODEX_CLI_PATH to an installed Codex executable.`,+ );+ }+ return command;+}diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.tsindex 285cb6bda..2ca651fcf 100644--- a/sdk/typescript/src/runtime.ts+++ b/sdk/typescript/src/runtime.ts@@ -1,3 +1,4 @@+import { resolveBundledCodexExecutable } from "./codex-sdk-environment.js";import { gitProtectionRoots } from "./targets.js";import { isNonEmptyString, notify } from "./value.js";import { execFile as execFileCallback, spawn } from "node:child_process";@@ -6,8 +7,6 @@ import {chmodSync,constants,createWriteStream,- existsSync,- readdirSync,type BigIntStats,type Stats,} from "node:fs";@@ -2824,37 +2823,7 @@ export function resolveCodexCommand(};}- const platform = process.platform === "android" ? "linux" : process.platform;- const packageName = `@openai/codex-${platform}-${process.arch}`;- let packageJson: string;- try {- const require = createRequire(import.meta.url);- const codexPackageJson = require.resolve("@openai/codex/package.json");- packageJson = createRequire(codexPackageJson).resolve(- `${packageName}/package.json`,- );- } catch (error) {- throw new PluginBootstrapError(- `The bundled Codex executable could not be resolved from ${packageName}. Reinstall @openai/codex with optional dependencies enabled, or set CODEX_CLI_PATH to an installed Codex executable.`,- { cause: error },- );- }- const vendor = join(dirname(packageJson), "vendor");- const target = readdirSync(vendor, { withFileTypes: true }).find((entry) =>- entry.isDirectory(),- );- const command = join(- vendor,- target?.name ?? "",- "bin",- process.platform === "win32" ? "codex.exe" : "codex",- );- if (target === undefined || !existsSync(command)) {- throw new PluginBootstrapError(- `The ${packageName} package does not contain the Codex executable. Reinstall @openai/codex with optional dependencies enabled, or set CODEX_CLI_PATH to an installed Codex executable.`,- );- }- return { command };+ return { command: resolveBundledCodexExecutable() };}export function executablePathForSpawn(command: string): string {diff --git a/sdk/typescript/tests-ts/mcp-launcher.test.ts b/sdk/typescript/tests-ts/mcp-launcher.test.tsindex 2339d887a..2cd297597 100644--- a/sdk/typescript/tests-ts/mcp-launcher.test.ts+++ b/sdk/typescript/tests-ts/mcp-launcher.test.ts@@ -392,9 +392,16 @@ setInterval(() => {}, 1000););}-test.each(["server", "helper"] as const)(- "starts the packaged %s with managed Node and an empty PATH",- async (mode) => {+test.each(+ (["server", "helper"] as const).flatMap((mode) =>+ ["none", "LOCALAPPDATA", "XDG_CACHE_HOME", "USERPROFILE"].map((cache) => ({+ mode,+ cache,+ })),+ ),+)(+ "starts the packaged $mode with managed Node before $cache cache and an empty PATH",+ async ({ mode, cache }) => {const node = Bun.which("node");if (node === null)throw new Error("Node is required for the MCP smoke test.");@@ -429,6 +436,32 @@ test.each(["server", "helper"] as const)(`@echo used>"${marker}"\r\n@"${node}" %*\r\n`,);}+ if (cache !== "none") {+ const cacheNode = join(+ root,+ ...(cache === "LOCALAPPDATA"+ ? [+ "OpenAI",+ "Codex",+ "runtimes",+ "cua_node",+ "fixture",+ "bin",+ "node.exe",+ ]+ : [+ ...(cache === "USERPROFILE" ? [".cache"] : []),+ "codex-runtimes",+ "codex-primary-runtime",+ "dependencies",+ "node",+ "bin",+ "node.exe",+ ]),+ );+ await mkdir(dirname(cacheNode), { recursive: true });+ await copyFile(node, cacheNode);+ }const launcher = join(PLUGIN_ROOT, config.command);const windows = process.platform === "win32";const args =diff --git a/sdk/typescript/tests-ts/provider-profile.test.ts b/sdk/typescript/tests-ts/provider-profile.test.tsindex 8129d36cd..486f6ef4a 100644--- a/sdk/typescript/tests-ts/provider-profile.test.ts+++ b/sdk/typescript/tests-ts/provider-profile.test.ts@@ -1,4 +1,5 @@import { existsSync } from "node:fs";+import * as fsPromises from "node:fs/promises";import {chmod,copyFile,@@ -10,7 +11,7 @@ import {} from "node:fs/promises";import { join, sep } from "node:path";import { pathToFileURL } from "node:url";-import { afterEach, expect, test } from "bun:test";+import { afterEach, expect, spyOn, test } from "bun:test";import type { CodexOptions } from "@openai/codex-sdk";import { parse } from "smol-toml";import {@@ -288,6 +289,31 @@ test("native profile turns omit optional null fields and retain array errors", a);});+test("API fixture cleanup retries a transient busy directory", async () => {+ const root = await temporaryDirectory();+ await writeFile(join(root, "child.txt"), "synthetic fixture");+ const remove = fsPromises.rm;+ let busy = true;+ const removal = spyOn(fsPromises, "rm").mockImplementation(+ async (path, options) => {+ if (path === root && busy) {+ busy = false;+ throw Object.assign(new Error("Synthetic busy directory"), {+ code: "EBUSY",+ });+ }+ await remove(path, options);+ },+ );+ try {+ await cleanup();+ await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" });+ } finally {+ removal.mockRestore();+ await remove(root, { recursive: true, force: true });+ }+});+test("native profile launches retain bundled tools through executable path aliases", async () => {const root = await temporaryDirectory();const bin = join(root, "vendor", "bin");diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.tsindex f136e01fc..04e2a10f9 100644--- a/sdk/typescript/tests-ts/skeleton.test.ts+++ b/sdk/typescript/tests-ts/skeleton.test.ts@@ -263,7 +263,8 @@ describe("TypeScript package skeleton", () => {expect(nativeCoverageStep).not.toHaveProperty("continue-on-error");for (const name of ["Install plugin dependencies",- "Build SDK and type-check eval tooling",+ "Set up Node.js for triage evals",+ "Set up triage eval dependencies and host runtime",]) {expect(job.steps!.find((step) => step.name === name)?.if).toBe("matrix.os == 'ubuntu-latest' && matrix.python == '3.12'",diff --git a/sdk/typescript/tests-ts/support/temporary-directories.ts b/sdk/typescript/tests-ts/support/temporary-directories.tsindex b5ef19c74..80ac0a7a8 100644--- a/sdk/typescript/tests-ts/support/temporary-directories.ts+++ b/sdk/typescript/tests-ts/support/temporary-directories.ts@@ -27,7 +27,10 @@ export function createApiTestFixtures(prefix = "codex-security-api-",canonicalize = true,) {- const temporaryDirectories = createTemporaryDirectories(canonicalize);+ const temporaryDirectories = createTemporaryDirectories(+ canonicalize,+ removeTemporaryDirectory,+ );return {temporaryDirectories,cleanup: temporaryDirectories.cleanup,