OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test(evals): share intake grading and discovery fixtures (#2184)
Michael D'Angeloa43a30db9f34
diff --git a/evals/secret-discovery/fixtures.mts b/evals/secret-discovery/fixtures.mtsindex 4a55a1463..2eec7efe5 100644--- a/evals/secret-discovery/fixtures.mts+++ b/evals/secret-discovery/fixtures.mts@@ -80,8 +80,8 @@ CONTENT_SHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdid: string,path: string,marker: string,- cwes: string[],consumerMarker?: string,+ cwes = ["CWE-798", "CWE-312", "CWE-313", "CWE-522"],endMarker = marker,) => {const lines = files[path].split("\n");@@ -104,23 +104,16 @@ CONTENT_SHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcd"active-source","src/client.py","SERVICE_TOKEN =",- ["CWE-798", "CWE-312", "CWE-313", "CWE-522"],'"Authorization": "Bearer " + SERVICE_TOKEN',),- positive("unused-source", "src/retired.py", "LEGACY_SERVICE_TOKEN =", [- "CWE-798",- "CWE-312",- "CWE-313",- "CWE-522",- ]),+ positive("unused-source", "src/retired.py", "LEGACY_SERVICE_TOKEN ="),positive("integration-source","tests/test_delivery.py","INTEGRATION_TOKEN =",- ["CWE-798", "CWE-312", "CWE-313", "CWE-522"],'"Authorization": "Bearer " + INTEGRATION_TOKEN',),- positive("dotenv-url", "config/.env", "DATABASE_URL=", [+ positive("dotenv-url", "config/.env", "DATABASE_URL=", undefined, ["CWE-798","CWE-256","CWE-259",@@ -129,18 +122,13 @@ CONTENT_SHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcd"CWE-313","CWE-522",]),- positive("account-config", "config/service.json", '"client_secret"', [- "CWE-798",- "CWE-312",- "CWE-313",- "CWE-522",- ]),+ positive("account-config", "config/service.json", '"client_secret"'),positive("private-key","src/signing.mjs","const signingKey",- ["CWE-321", "CWE-798", "CWE-312", "CWE-313"],"return sign(",+ ["CWE-321", "CWE-798", "CWE-312", "CWE-313"],privateKey.trim().split("\n").at(-2),),],diff --git a/evals/secret-discovery/test_discovery.mts b/evals/secret-discovery/test_discovery.mtsindex 8ac38e58c..2027e7b03 100644--- a/evals/secret-discovery/test_discovery.mts+++ b/evals/secret-discovery/test_discovery.mts@@ -832,23 +832,15 @@ test("stages production prompt unchanged and no labels; grades only the final SD);return {events: (async function* () {- yield {- type: "item.completed",- item: {- type: "agent_message",- text: JSON.stringify(retainedResult(prepared.fixture)),- },- };- yield {- type: "item.completed",- item: {- type: "agent_message",- text: JSON.stringify({- findings: [],- coverage: {},- }),- },- };+ for (const result of [+ retainedResult(prepared.fixture),+ { findings: [], coverage: {} },+ ]) {+ yield {+ type: "item.completed",+ item: { type: "agent_message", text: JSON.stringify(result) },+ };+ }yield {type: "turn.completed",usage: { input_tokens: 1, output_tokens: 1 },diff --git a/evals/secret-discovery/test_runtime.mts b/evals/secret-discovery/test_runtime.mtsindex bfb178497..3516913e2 100644--- a/evals/secret-discovery/test_runtime.mts+++ b/evals/secret-discovery/test_runtime.mts@@ -21,7 +21,7 @@ import { tmpdir } from "node:os";import { syncBuiltinESMExports } from "node:module";import { delimiter, dirname, join, win32 } from "node:path";import { setTimeout as delay } from "node:timers/promises";-import test from "node:test";+import test, { type TestContext } from "node:test";import { Codex } from "../../sdk/typescript/node_modules/@openai/codex-sdk/dist/index.js";import {codexSettings,@@ -153,28 +153,12 @@ test("an eval does not overwrite a later replacement login", async (t) => {);});-test("the SDK launches long Windows paths with the native namespace prefix", async (t) => {- const executable = win32.join(- "C:\\synthetic",- ...Array(12).fill("nested executable directory"),- "bin",- "codex.exe",- );- assert.ok(executable.length > 260);- const platform = Object.getOwnPropertyDescriptor(process, "platform");- let settings;- try {- Object.defineProperty(process, "platform", { value: "win32" });- settings = codexSettings("C:\\synthetic-home", executable, {});- } finally {- Object.defineProperty(process, "platform", platform!);- }- assert.equal(settings.env.CODEX_CLI_PATH, executable);-+async function captureSdkLaunch(+ t: TestContext,+ settings: ReturnType<typeof codexSettings>,+) {const intercepted = new Error("Synthetic launch intercepted");- let launched;- const spawn = t.mock.method(childProcess, "spawn", (file: string) => {- launched = file;+ const spawn = t.mock.method(childProcess, "spawn", () => {throw intercepted;});syncBuiltinESMExports();@@ -189,11 +173,33 @@ test("the SDK launches long Windows paths with the native namespace prefix", asy(error) => error === intercepted,);assert.equal(spawn.mock.callCount(), 1);- assert.equal(launched, win32.toNamespacedPath(executable));+ return spawn.mock.calls[0].arguments;} finally {spawn.mock.restore();syncBuiltinESMExports();}+}++test("the SDK launches long Windows paths with the native namespace prefix", async (t) => {+ const executable = win32.join(+ "C:\\synthetic",+ ...Array(12).fill("nested executable directory"),+ "bin",+ "codex.exe",+ );+ assert.ok(executable.length > 260);+ const platform = Object.getOwnPropertyDescriptor(process, "platform");+ let settings;+ try {+ Object.defineProperty(process, "platform", { value: "win32" });+ settings = codexSettings("C:\\synthetic-home", executable, {});+ } finally {+ Object.defineProperty(process, "platform", platform!);+ }+ assert.equal(settings.env.CODEX_CLI_PATH, executable);++ const [launched] = await captureSdkLaunch(t, settings);+ assert.equal(launched, win32.toNamespacedPath(executable));});test("the SDK executable override preserves bundled tools at the child boundary", async (t) => {@@ -209,57 +215,31 @@ test("the SDK executable override preserves bundled tools at the child boundary"PATH: [tools, join(directory, "unix tools"), tools].join(delimiter),};const original = { ...inherited };- const intercepted = new Error("Synthetic launch intercepted");- const environments: NodeJS.ProcessEnv[] = [];- const spawn = t.mock.method(- childProcess,- "spawn",- (- _file: string,- _args: string[],- options: import("node:child_process").SpawnOptions,- ) => {- environments.push(options.env!);- throw intercepted;- },- );- syncBuiltinESMExports();- try {- for (const environment of [{}, inherited]) {- await assert.rejects(- async () => {- const settings = codexSettings(directory, executable, environment);- const { events } = await new Codex(settings)- .startThread({ skipGitRepoCheck: true })- .runStreamed("Offline synthetic launch check");- for await (const event of events) void event;- },- (error) => error === intercepted,- );- }- assert.deepEqual(- environments.map((environment) =>- Object.fromEntries(- Object.entries(environment).filter(- ([key]) => key.toLowerCase() === "path",- ),+ const environments = [];+ for (const environment of [{}, inherited]) {+ const settings = codexSettings(directory, executable, environment);+ const [, , options] = await captureSdkLaunch(t, settings);+ environments.push(options!.env!);+ }+ assert.deepEqual(+ environments.map((environment) =>+ Object.fromEntries(+ Object.entries(environment).filter(+ ([key]) => key.toLowerCase() === "path",),),- [- { PATH: tools },- process.platform === "win32"- ? { Path: [tools, join(directory, "other tools")].join(delimiter) }- : {- ...inherited,- PATH: [tools, join(directory, "unix tools")].join(delimiter),- },- ],- );- assert.deepEqual(inherited, original);- } finally {- spawn.mock.restore();- syncBuiltinESMExports();- }+ ),+ [+ { PATH: tools },+ process.platform === "win32"+ ? { Path: [tools, join(directory, "other tools")].join(delimiter) }+ : {+ ...inherited,+ PATH: [tools, join(directory, "unix tools")].join(delimiter),+ },+ ],+ );+ assert.deepEqual(inherited, original);});async function waitForFile(path: string) {@@ -276,7 +256,7 @@ async function waitForFile(path: string) {}async function nativeFixture(- t: import("node:test").TestContext,+ t: TestContext,mode = "complete",account: { type: string } | null = null,) {@@ -398,24 +378,18 @@ if (kind === "preflight") {OPENAI_API_KEY: "synthetic-env-key",DATABASE_URL: "synthetic-unrelated-secret",});- return { directory, prepared, settings, executable, result };+ const preflight = (initial = settings) =>+ preflightEval(prepared, initial, new AbortController().signal);+ return { directory, prepared, settings, executable, result, preflight };}test("native preflight and SDK exec share effective settings and fallback auth",unixOnly,async (t) => {- const {- directory,- prepared,- settings: original,- result,- } = await nativeFixture(t);- const settings = await preflightEval(- prepared,- original,- new AbortController().signal,- );+ const native = await nativeFixture(t);+ const { directory, prepared, settings: original, result } = native;+ const settings = await native.preflight();const { report, semanticResult } = await runPreparedEval(prepared,new Codex(settings),@@ -508,25 +482,21 @@ for (const {} of [{name: "missing file login",- account: null,expectedKey: "synthetic-env-key",},{name: "empty auth file",authFile: "",- account: null,expectedKey: "synthetic-env-key",},{name: "malformed auth file",authFile: "{",- account: null,expectedKey: "synthetic-env-key",},{name: "auth file without an account",authFile: "{}",- account: null,expectedKey: "synthetic-env-key",},{@@ -541,7 +511,6 @@ for (const {},{name: "blank Codex key",- account: null,codexApiKey: " \t ",expectedKey: "synthetic-env-key",},@@ -552,14 +521,11 @@ for (const {codexApiKey: "synthetic-codex-key",expectedKey: "synthetic-codex-key",},- { name: "blank OpenAI key", account: null, openAiApiKey: " \t " },+ { name: "blank OpenAI key", openAiApiKey: " \t " },]) {test(`native auth precedence with ${name}`, unixOnly, async (t) => {- const {- directory,- prepared,- settings: original,- } = await nativeFixture(t, "complete", account);+ const native = await nativeFixture(t, "complete", account);+ const { directory, prepared, settings: original } = native;const ambient = original.env.CODEX_HOME;if (authFile !== undefined)await writeFile(join(ambient, "auth.json"), authFile);@@ -569,11 +535,7 @@ for (const {CODEX_API_KEY: codexApiKey,OPENAI_API_KEY: openAiApiKey,});- const settings = await preflightEval(- prepared,- initialSettings,- new AbortController().signal,- );+ const settings = await native.preflight(initialSettings);const { report } = await runPreparedEval(prepared, new Codex(settings));assert.equal(report.passed, true);assert.equal(initialSettings.apiKey, undefined);@@ -609,11 +571,8 @@ for (const codexApiKey of [undefined, "synthetic-explicit-key"]) {`Windows mixed-case auth snapshot with ${codexApiKey ? "explicit" : "fallback"} key`,unixOnly,async (t) => {- const {- directory,- prepared,- settings: original,- } = await nativeFixture(t);+ const native = await nativeFixture(t);+ const { directory, prepared, settings: original } = native;const {OPENAI_API_KEY,Path: ignoredPathAlias,@@ -632,11 +591,7 @@ for (const codexApiKey of [undefined, "synthetic-explicit-key"]) {);assert.equal(snapshot.env.OPENAI_API_KEY, undefined);assert.equal(snapshot.env.OpenAI_API_Key, environment.OpenAI_API_Key);- settings = await preflightEval(- prepared,- snapshot,- new AbortController().signal,- );+ settings = await native.preflight(snapshot);} finally {Object.defineProperty(process, "platform", platform!);}@@ -661,13 +616,10 @@ for (const [mode, expectedError] of [["account-error", /account\/read/],] as const) {test(`${mode} prevents starting the SDK exec turn`, unixOnly, async (t) => {- const { directory, prepared, settings } = await nativeFixture(t, mode);+ const native = await nativeFixture(t, mode);+ const { directory, prepared } = native;await assert.rejects(async () => {- const resolved = await preflightEval(- prepared,- settings,- new AbortController().signal,- );+ const resolved = await native.preflight();await runPreparedEval(prepared, new Codex(resolved));}, expectedError);await assert.rejects(access(join(directory, "exec.json")), {@@ -682,12 +634,9 @@ for (const mode of ["fallback-item", "fallback-error"]) {`${mode} aborts the real SDK turn and discards its response`,unixOnly,async (t) => {- const { directory, prepared, settings } = await nativeFixture(t, mode);- const resolved = await preflightEval(- prepared,- settings,- new AbortController().signal,- );+ const native = await nativeFixture(t, mode);+ const { directory, prepared } = native;+ const resolved = await native.preflight();await assert.rejects(runPreparedEval(prepared, new Codex(resolved)),/results were discarded/,diff --git a/evals/triage-finding/assertions/github-rest-intake.mts b/evals/triage-finding/assertions/github-rest-intake.mtsindex fc4fbe41e..4a6d00b4f 100644--- a/evals/triage-finding/assertions/github-rest-intake.mts+++ b/evals/triage-finding/assertions/github-rest-intake.mts@@ -1,4 +1,8 @@-import { outputText as textFor, hasTriageJson } from "./output.mts";+import {+ gradeFailures,+ outputText as textFor,+ hasTriageJson,+} from "./output.mts";import type { AssertionContext } from "../types.ts";function repositoryName(value: string) {@@ -259,12 +263,5 @@ export default (output: unknown, context: AssertionContext) => {? check(text, context): [`unknown expected_github_rest_behavior: ${behavior}`];- return {- pass: failures.length === 0,- score: failures.length === 0 ? 1 : 0,- reason:- failures.length === 0- ? "GitHub REST intake behavior matched."- : failures.join("; "),- };+ return gradeFailures(failures, "GitHub REST intake behavior matched.");};diff --git a/evals/triage-finding/assertions/missing-input.mts b/evals/triage-finding/assertions/missing-input.mtsindex a18b25cb6..5dad8c24c 100644--- a/evals/triage-finding/assertions/missing-input.mts+++ b/evals/triage-finding/assertions/missing-input.mts@@ -1,4 +1,4 @@-import { outputText, hasTriageJson } from "./output.mts";+import { gradeFailures, outputText, hasTriageJson } from "./output.mts";function matchedFormatNames(text: string) {const supportedFormats: [string, RegExp][] = [@@ -46,12 +46,8 @@ export default (output: unknown) => {);}- return {- pass: failures.length === 0,- score: failures.length === 0 ? 1 : 0,- reason:- failures.length === 0- ? "Missing-input response asks for a finding and names supported formats without triage JSON."- : failures.join("; "),- };+ return gradeFailures(+ failures,+ "Missing-input response asks for a finding and names supported formats without triage JSON.",+ );};diff --git a/evals/triage-finding/assertions/output.mts b/evals/triage-finding/assertions/output.mtsindex 59d81b0c6..c660037eb 100644--- a/evals/triage-finding/assertions/output.mts+++ b/evals/triage-finding/assertions/output.mts@@ -1,5 +1,14 @@import type { TriageResult } from "../types.ts";+export function gradeFailures(failures: string[], successReason: string) {+ const pass = failures.length === 0;+ return {+ pass,+ score: pass ? 1 : 0,+ reason: pass ? successReason : failures.join("; "),+ };+}+export function outputText(output: unknown) {return typeof output === "string" ? output : JSON.stringify(output);}diff --git a/evals/triage-finding/assertions/ticket-intake.mts b/evals/triage-finding/assertions/ticket-intake.mtsindex b03a5fe55..c70c063c1 100644--- a/evals/triage-finding/assertions/ticket-intake.mts+++ b/evals/triage-finding/assertions/ticket-intake.mts@@ -1,4 +1,4 @@-import { outputText, hasTriageJson } from "./output.mts";+import { gradeFailures, outputText, hasTriageJson } from "./output.mts";import type { AssertionContext } from "../types.ts";const expectedPatterns: Record<string, RegExp[]> = {@@ -52,33 +52,27 @@ export default (output: unknown, context: AssertionContext) => {const subissueBehavior = String(context.vars.expected_linear_subissues || "");const patterns = Object.hasOwn(expectedPatterns, behavior)? expectedPatterns[behavior]- : [];+ : undefined;const subissuePatterns = Object.hasOwn(expectedSubissuePatterns,subissueBehavior,)? expectedSubissuePatterns[subissueBehavior]- : [];+ : undefined;const failures = [];- if (- (!behavior && !subissueBehavior) ||- (behavior && !Object.hasOwn(expectedPatterns, behavior))- ) {+ if ((!behavior && !subissueBehavior) || (behavior && !patterns)) {failures.push(`unknown expected_ticket_failure: ${behavior}`);}- if (- subissueBehavior &&- !Object.hasOwn(expectedSubissuePatterns, subissueBehavior)- ) {+ if (subissueBehavior && !subissuePatterns) {failures.push(`unknown expected_linear_subissues: ${subissueBehavior}`);}- for (const pattern of patterns) {+ for (const pattern of patterns ?? []) {if (!pattern.test(text))failures.push(`missing recovery detail matching ${pattern}`);}- for (const pattern of subissuePatterns) {+ for (const pattern of subissuePatterns ?? []) {if (!pattern.test(text))failures.push(`missing Linear sub-issue detail matching ${pattern}`);}@@ -92,12 +86,8 @@ export default (output: unknown, context: AssertionContext) => {failures.push("must stop before repository analysis or verdicting");}- return {- pass: failures.length === 0,- score: failures.length === 0 ? 1 : 0,- reason:- failures.length === 0- ? "Ticket connector failure produced recovery guidance without triage output."- : failures.join("; "),- };+ return gradeFailures(+ failures,+ "Ticket connector failure produced recovery guidance without triage output.",+ );};diff --git a/evals/triage-finding/assertions/triage-io.mts b/evals/triage-finding/assertions/triage-io.mtsindex f66fe2430..be06c91ef 100644--- a/evals/triage-finding/assertions/triage-io.mts+++ b/evals/triage-finding/assertions/triage-io.mts@@ -1,26 +1,9 @@import type { AssertionContext } from "../types.ts";-import { extractTriageResult, parseExpected } from "./output.mts";--function expectedScanbenchMapping(context: AssertionContext) {- const label = String(context.vars.expected_binary_label || "");- const mappings = [];-- if (label === "positive") {- mappings.push({- expectedVerdict: "confirmed",- expectedBinaryLabel: "positive",- label: "positive scanbench label",- });- } else if (label === "negative") {- mappings.push({- expectedVerdict: "not_actionable",- expectedBinaryLabel: "negative",- label: "negative scanbench label",- });- }-- return mappings;-}+import {+ gradeFailures,+ extractTriageResult,+ parseExpected,+} from "./output.mts";export default (output: unknown, context: AssertionContext) => {const result = extractTriageResult(@@ -32,21 +15,24 @@ export default (output: unknown, context: AssertionContext) => {const expectedVerdicts = parseExpected(context.vars.expected_verdicts);const failures = [];- for (const mapping of expectedScanbenchMapping(context)) {+ const binaryLabel = String(context.vars.expected_binary_label || "");+ if (binaryLabel === "positive" || binaryLabel === "negative") {+ const expectedVerdict =+ binaryLabel === "positive" ? "confirmed" : "not_actionable";if (expectedVerdicts.length === 1 &&- expectedVerdicts[0] !== mapping.expectedVerdict+ expectedVerdicts[0] !== expectedVerdict) {failures.push(- `${mapping.label}: expected_verdicts must be ${mapping.expectedVerdict}, got ${expectedVerdicts[0]}`,+ `${binaryLabel} scanbench label: expected_verdicts must be ${expectedVerdict}, got ${expectedVerdicts[0]}`,);}if (context.vars.expected_binary_label &&- context.vars.expected_binary_label !== mapping.expectedBinaryLabel+ context.vars.expected_binary_label !== binaryLabel) {failures.push(- `${mapping.label}: expected_binary_label must be ${mapping.expectedBinaryLabel}, got ${context.vars.expected_binary_label}`,+ `${binaryLabel} scanbench label: expected_binary_label must be ${binaryLabel}, got ${context.vars.expected_binary_label}`,);}}@@ -67,20 +53,16 @@ export default (output: unknown, context: AssertionContext) => {findings.forEach((finding, index) => {const label = expectedIds[index] || `index ${index}`;- if (finding.input_id !== expectedIds[index]) {- failures.push(- `${label}: expected input_id ${expectedIds[index]}, got ${finding.input_id}`,- );- }- if (finding.source_type !== expectedSourceTypes[index]) {- failures.push(- `${label}: expected source_type ${expectedSourceTypes[index]}, got ${finding.source_type}`,- );- }- if (finding.verdict !== expectedVerdicts[index]) {- failures.push(- `${label}: expected verdict ${expectedVerdicts[index]}, got ${finding.verdict}`,- );+ for (const [field, expected] of [+ ["input_id", expectedIds[index]],+ ["source_type", expectedSourceTypes[index]],+ ["verdict", expectedVerdicts[index]],+ ] as const) {+ if (finding[field] !== expected) {+ failures.push(+ `${label}: expected ${field} ${expected}, got ${finding[field]}`,+ );+ }}const rank = finding.exploitability_stack_rank?.rank;@@ -146,12 +128,8 @@ export default (output: unknown, context: AssertionContext) => {});}- return {- pass: failures.length === 0,- score: failures.length === 0 ? 1 : 0,- reason:- failures.length === 0- ? "Triage I/O matched expected source types, order, and verdicts."- : failures.join("; "),- };+ return gradeFailures(+ failures,+ "Triage I/O matched expected source types, order, and verdicts.",+ );};diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts b/evals/triage-finding/sastbench/scripts/test-sastbench-result.mtsindex cb2bdfca3..d8e751488 100644--- a/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts@@ -48,11 +48,7 @@ assert.throws(() => extractTriageResult(truncated), {message: "Could not find a parseable triage-finding/v0 JSON result",});const truncatedMetrics = addSastBenchMetrics(- extensionContext({- caseId: "sastbench-000000",- expectedGroundTruth: "true_positive",- output: truncated,- }),+ extensionContext("sastbench-000000", "true_positive", { output: truncated }),).result;assert.equal(truncatedMetrics.metadata.sastbench.status, "invalid_output");assert.equal(truncatedMetrics.namedScores.invalid_output, 1);@@ -71,11 +67,7 @@ for (const output of ["confirmed",);const result = addSastBenchMetrics(- extensionContext({- caseId: "sastbench-000000",- expectedGroundTruth: "true_positive",- output,- }),+ extensionContext("sastbench-000000", "true_positive", { output }),).result;assert.equal(result.metadata.sastbench.status, "ok");assert.equal(result.namedScores.strict_tp, 1);@@ -121,23 +113,16 @@ assert.equal(false,);-function extensionContext({- caseId,- expectedGroundTruth,- output,- error,- sampleWeight,-}: {- caseId: string;- expectedGroundTruth: string;- output?: string;- error?: string;- sampleWeight?: number;-}) {+function extensionContext(+ caseId: string,+ groundTruth: string,+ { output, error }: { output?: string; error?: string },+ sampleWeight?: number,+) {return {test: {metadata: {- ground_truth: expectedGroundTruth,+ ground_truth: groundTruth,...(sampleWeight === undefined ? {} : { sample_weight: sampleWeight }),},},@@ -159,43 +144,16 @@ function extensionContext({};}-const outcomes = [- {- caseId: "tp-confirmed",- expectedGroundTruth: "true_positive",- verdict: "confirmed",- },- {- caseId: "tp-closed",- expectedGroundTruth: "true_positive",- verdict: "not_actionable",- },- {- caseId: "tp-review",- expectedGroundTruth: "true_positive",- verdict: "needs_review",- },- {- caseId: "fp-closed",- expectedGroundTruth: "false_positive",- verdict: "not_actionable",- },- {- caseId: "fp-confirmed",- expectedGroundTruth: "false_positive",- verdict: "confirmed",- },- {- caseId: "fp-review",- expectedGroundTruth: "false_positive",- verdict: "needs_review",- },-];-const nativeContexts = outcomes.map((outcome) =>- extensionContext({- caseId: outcome.caseId,- expectedGroundTruth: outcome.expectedGroundTruth,- output: JSON.stringify(triageResult(outcome.verdict, outcome.caseId)),+const nativeContexts = [+ ["tp-confirmed", "true_positive", "confirmed"],+ ["tp-closed", "true_positive", "not_actionable"],+ ["tp-review", "true_positive", "needs_review"],+ ["fp-closed", "false_positive", "not_actionable"],+ ["fp-confirmed", "false_positive", "confirmed"],+ ["fp-review", "false_positive", "needs_review"],+].map(([caseId, groundTruth, verdict]) =>+ extensionContext(caseId, groundTruth, {+ output: JSON.stringify(triageResult(verdict, caseId)),}),);const nativeMetricRows = nativeContexts.map(addSastBenchMetrics);@@ -239,54 +197,33 @@ assert.equal(totals.execution_or_parse_error, 0);assert.equal(totals.existing_metric, 6);const positiveProviderError = addSastBenchMetrics(- extensionContext({- caseId: "tp-error",- expectedGroundTruth: "true_positive",- error: "provider failed",- }),-);-assert.equal(positiveProviderError.result.namedScores.strict_fn, 1);-assert.equal(- positiveProviderError.result.namedScores.execution_or_parse_error,- 1,-);-assert.equal(- positiveProviderError.result.namedScores.remaining_analyst_work,- 1,-);-assert.equal(- positiveProviderError.result.metadata.sastbench.status,- "model_error",-);+ extensionContext("tp-error", "true_positive", { error: "provider failed" }),+).result;+assert.equal(positiveProviderError.namedScores.strict_fn, 1);+assert.equal(positiveProviderError.namedScores.execution_or_parse_error, 1);+assert.equal(positiveProviderError.namedScores.remaining_analyst_work, 1);+assert.equal(positiveProviderError.metadata.sastbench.status, "model_error");const negativeInvalidOutput = addSastBenchMetrics(- extensionContext({- caseId: "fp-invalid",- expectedGroundTruth: "false_positive",+ extensionContext("fp-invalid", "false_positive", {output: "not a triage result",}),-);-assert.equal(negativeInvalidOutput.result.namedScores.strict_fp, 1);-assert.equal(- negativeInvalidOutput.result.namedScores.execution_or_parse_error,- 1,-);-assert.equal(- negativeInvalidOutput.result.metadata.sastbench.status,- "invalid_output",-);+).result;+assert.equal(negativeInvalidOutput.namedScores.strict_fp, 1);+assert.equal(negativeInvalidOutput.namedScores.execution_or_parse_error, 1);+assert.equal(negativeInvalidOutput.metadata.sastbench.status, "invalid_output");const weightedPositive = addSastBenchMetrics(- extensionContext({- caseId: "weighted-tp",- expectedGroundTruth: "true_positive",- output: JSON.stringify(triageResult("confirmed", "weighted-tp")),- sampleWeight: 2.5,- }),-);-assert.equal(weightedPositive.result.namedScores.strict_tp, 2.5);-assert.equal(weightedPositive.result.namedScores.positive_cases, 2.5);-assert.equal(weightedPositive.result.namedScores.existing_metric, 1);+ extensionContext(+ "weighted-tp",+ "true_positive",+ { output: JSON.stringify(triageResult("confirmed", "weighted-tp")) },+ 2.5,+ ),+).result;+assert.equal(weightedPositive.namedScores.strict_tp, 2.5);+assert.equal(weightedPositive.namedScores.positive_cases, 2.5);+assert.equal(weightedPositive.namedScores.existing_metric, 1);const firstResult = nativeMetricRows[0].result;assert.equal(firstResult.metadata.sastbench.status, "ok");@@ -295,12 +232,9 @@ assert.equal(firstResult.latencyMs, 1234);assert.equal(firstResult.cost, 0.25);assert.equal(firstResult.response, nativeContexts[0].result.response);assert.match(- negativeInvalidOutput.result.metadata.sastbench.error!,+ negativeInvalidOutput.metadata.sastbench.error!,/triage-finding\/v0/,);-assert.equal(- positiveProviderError.result.metadata.sastbench.error,- "provider failed",-);+assert.equal(positiveProviderError.metadata.sastbench.error, "provider failed");console.log("sastbench result and native metric tests passed");diff --git a/evals/triage-finding/scripts/test-intake-assertions.mts b/evals/triage-finding/scripts/test-intake-assertions.mtsindex 7895fb554..e3e733623 100644--- a/evals/triage-finding/scripts/test-intake-assertions.mts+++ b/evals/triage-finding/scripts/test-intake-assertions.mts@@ -138,6 +138,10 @@ const githubContext = (behavior: string) => ({target_repo: "https://github.com/example/project",},});+function expectGitHub(behavior: string, text: string, pass = true) {+ expectPass(github, text, githubContext(behavior), pass);+}+for (const repository of ["{owner}/{repo}", "example/project"]) {for (const sourceType of ["source_type: advisory",@@ -150,15 +154,13 @@ for (const repository of ["{owner}/{repo}", "example/project"]) {"source_type: advisory.","source_type: advisory!",]) {- expectPass(- github,+ expectGitHub(+ "dependabot_malware",`GET /repos/${repository}/dependabot/alerts?classification=malware&state=open&per_page=100. ${sourceType}`,- githubContext("dependabot_malware"),);- expectPass(- github,+ expectGitHub(+ "advisories_private_reports",`GET /repos/${repository}/security-advisories?per_page=100, with separate state=triage, state=draft, state=published, state=closed requests. Triage is for private vulnerability reports. ${sourceType}`,- githubContext("advisories_private_reports"),);}}@@ -175,12 +177,7 @@ for (const repository of ["{owner}/{repo}", "example/project"]) {"_wrong","Extra",]) {- expectPass(- github,- output.replaceAll("?", `${suffix}?`),- githubContext(behavior),- false,- );+ expectGitHub(behavior, output.replaceAll("?", `${suffix}?`), false);}for (const delimiter of ["\n",@@ -194,11 +191,7 @@ for (const repository of ["{owner}/{repo}", "example/project"]) {": ","] ",]) {- expectPass(- github,- output.replaceAll("?", `${delimiter}?`),- githubContext(behavior),- );+ expectGitHub(behavior, output.replaceAll("?", `${delimiter}?`));}}}@@ -217,16 +210,15 @@ for (const repository of ["{owner}/{repo}", "example/project"]) {const answer = JSON.parse(JSON.stringify(codeScanning).replaceAll("example/project", repository),);- expectPass(- github,+ expectGitHub(+ "code_scanning",JSON.stringify(answer).replaceAll(":100", ':"100"'),- githubContext("code_scanning"),);for (const text of [JSON.stringify(answer),`\`\`\`json\n${JSON.stringify(answer)}\n\`\`\``,]) {- expectPass(github, text, githubContext("code_scanning"));+ expectGitHub("code_scanning", text);}}for (const invalid of [@@ -261,19 +253,13 @@ for (const invalid of [alerts: { ...codeScanning.alerts, parameters: { state: "open" } },},]) {- expectPass(- github,- JSON.stringify(invalid),- githubContext("code_scanning"),- false,- );+ expectGitHub("code_scanning", JSON.stringify(invalid), false);}-expectPass(github, "not JSON", githubContext("code_scanning"), false);+expectGitHub("code_scanning", "not JSON", false);-expectPass(- github,+expectGitHub(+ "explicit_issue",'GitHub Issues require an explicit issue and are not included in all sources. source_type: "freeform"',- githubContext("explicit_issue"),);for (const text of ['GET /repos/example/other/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: "advisory"',@@ -287,7 +273,7 @@ for (const text of ['GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type_suffix":"advisory"}',"GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: advisory_other",]) {- expectPass(github, text, githubContext("dependabot_malware"), false);+ expectGitHub("dependabot_malware", text, false);}for (const emphasis of ["**", "*", "__", "_", "***"]) {@@ -296,10 +282,9 @@ for (const emphasis of ["**", "*", "__", "_", "***"]) {`Normalize as ${emphasis}advisory${emphasis}`,`${emphasis}source_type${emphasis}: advisory`,]) {- expectPass(- github,+ expectGitHub(+ "dependabot_malware",`GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. ${normalization}.`,- githubContext("dependabot_malware"),);}for (const normalization of [@@ -314,16 +299,14 @@ for (const emphasis of ["**", "*", "__", "_", "***"]) {"Normalize as advisory: preserve the identifier","Normalization: source_type: advisory",]) {- expectPass(- github,+ expectGitHub(+ "dependabot_malware",`GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. ${emphasis}${normalization}${emphasis}.`,- githubContext("dependabot_malware"),);}- expectPass(- github,+ expectGitHub(+ "dependabot_malware",`GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. ${emphasis}source_type: advisory_other${emphasis}`,- githubContext("dependabot_malware"),false,);}@@ -338,7 +321,7 @@ const decision = {};const connectorJson = JSON.stringify(decision);for (const text of [connectorJson, `\`\`\`json\n${connectorJson}\n\`\`\``]) {- expectPass(github, text, githubContext("explicit_connector"));+ expectGitHub("explicit_connector", text);}for (const invalid of [null,@@ -361,22 +344,17 @@ for (const invalid of [{ ...decision, explanation: "additional field" },{ ...decision, scope: { ...decision.scope, extra: true } },]) {- expectPass(- github,- JSON.stringify(invalid),- githubContext("explicit_connector"),- false,- );+ expectGitHub("explicit_connector", JSON.stringify(invalid), false);}for (const text of ["not JSON",`\`\`\`json\n${connectorJson}\n\`\`\`\n\`\`\`json\n{}\n\`\`\``,]) {- expectPass(github, text, githubContext("explicit_connector"), false);+ expectGitHub("explicit_connector", text, false);}const restJson = JSON.stringify({ transport: "rest" });for (const text of [restJson, `\`\`\`json\n${restJson}\n\`\`\``]) {- expectPass(github, text, githubContext("default_rest"));+ expectGitHub("default_rest", text);}for (const invalid of [null,@@ -386,12 +364,7 @@ for (const invalid of [{ transport: "other" },{ transport: "rest", findings: [] },]) {- expectPass(- github,- JSON.stringify(invalid),- githubContext("default_rest"),- false,- );+ expectGitHub("default_rest", JSON.stringify(invalid), false);}for (const [behavior, answer] of [["explicit_connector", connectorJson],@@ -411,13 +384,13 @@ for (const [behavior, answer] of [`${answer}\n${extra}`,`\`\`\`json\n${answer}\n\`\`\`\n\`\`\`json\n${extra}\n\`\`\``,]) {- expectPass(github, text, githubContext(behavior), false);+ expectGitHub(behavior, text, false);}}}for (const behavior of ["", "code_scaning", "constructor"]) {- expectPass(github, "arbitrary text", githubContext(behavior), false);+ expectGitHub(behavior, "arbitrary text", false);}for (const repository of [diff --git a/evals/triage-finding/scripts/test-triage-io.mts b/evals/triage-finding/scripts/test-triage-io.mtsindex 14be2c5f7..202e6586c 100644--- a/evals/triage-finding/scripts/test-triage-io.mts+++ b/evals/triage-finding/scripts/test-triage-io.mts@@ -3,55 +3,50 @@ import assert from "assert";import fs from "node:fs";import path from "node:path";import vm from "node:vm";+import { parse } from "yaml";import triageIo from "../assertions/triage-io.mts";import type { TriageFinding } from "../types.ts";-function outputFor({- inputId,- verdict,- sourceType = "cve",-}: {- inputId: string;- verdict: string;- sourceType?: string;-}) {- const stackRank = {- rank_queue: verdict === "not_actionable" ? null : verdict,- rank: verdict === "not_actionable" ? null : 1,- rationale:- verdict === "not_actionable" ? "not actionable" : "Example ranking",- drivers: [],+function findingFor(+ inputId: string,+ verdict: string,+ rank: number | null = verdict === "not_actionable" ? null : 1,+ rankQueue: string | null = verdict === "not_actionable" ? null : verdict,+) {+ return {+ triage_item_id: "triage-001",+ input_id: inputId,+ source_type: "cve",+ title: "Example finding",+ normalized_input: {},+ verdict,+ confidence: "high",+ exploitability_stack_rank: {+ rank,+ rank_queue: rankQueue,+ rationale:+ verdict === "not_actionable" ? "not actionable" : "Example ranking",+ drivers: [],+ },+ affected_locations: [],+ reachable_path: [],+ evidence: [],+ counterevidence: [],+ proof_gaps: [],+ recommended_next_step: "No action",+ fix_finding_handoff: verdict === "confirmed" ? "Fix handoff" : null,};- return `\`\`\`json-{- "schema_version": "triage-finding/v0",- "repository": {- "path": "/tmp/repo",- "revision": "abc123"- },- "findings": [- {- "triage_item_id": "triage-001",- "input_id": "${inputId}",- "source_type": "${sourceType}",- "title": "Example finding",- "normalized_input": {},- "verdict": "${verdict}",- "confidence": "high",- "exploitability_stack_rank": ${JSON.stringify(stackRank)},- "affected_locations": [],- "reachable_path": [],- "evidence": [],- "counterevidence": [],- "proof_gaps": [],- "recommended_next_step": "No action",- "fix_finding_handoff": ${verdict === "confirmed" ? JSON.stringify("Fix handoff") : "null"}- }- ]-}-\`\`\``;}+const documentFor = (findings: TriageFinding[]) => ({+ schema_version: "triage-finding/v0",+ repository: { path: "/tmp/repo", revision: "abc123" },+ findings,+});++const queueOutput = (findings: TriageFinding[]) =>+ JSON.stringify({ schema_version: "triage-finding/v0", findings });+function assertPasses(name: string,output: unknown,@@ -76,75 +71,42 @@ function assertFails();}-const sourceType = "cve";--assertPasses(- "vulnerable scanbench cases map to confirmed/positive",- outputFor({- inputId: "input-001",- verdict: "confirmed",- }).replace('"evidence": []', '"evidence": ["Inline ``` fence in source."]'),- {- vars: {- case_id: "case-001",- expected_ids: "input-001",- expected_source_types: sourceType,- expected_verdicts: "confirmed",- expected_binary_label: "positive",- },- },-);--assertPasses(- "fixed scanbench cases map to not_actionable/negative",- outputFor({- inputId: "input-002",- verdict: "not_actionable",- }),- {- vars: {- case_id: "case-002",- expected_ids: "input-002",- expected_source_types: sourceType,- expected_verdicts: "not_actionable",- expected_binary_label: "negative",- },- },-);--assertFails(- "fixed scanbench cases cannot be labeled confirmed",- outputFor({- inputId: "input-002",- verdict: "confirmed",- }),- {+for (const [id, verdict, label, error] of [+ ["input-001", "confirmed", "positive", null],+ ["input-002", "not_actionable", "negative", null],+ ["input-002", "confirmed", "negative", /fixed.*not_actionable|negative/],+] as const) {+ const finding = findingFor(id, verdict);+ const output =+ "```json\n" ++ JSON.stringify(+ documentFor([+ {+ ...finding,+ evidence: label === "positive" ? ["Inline ``` fence in source."] : [],+ },+ ]),+ null,+ 2,+ ) ++ "\n```";+ const context = {vars: {- case_id: "case-002",- expected_ids: "input-002",- expected_source_types: sourceType,- expected_verdicts: "confirmed",- expected_binary_label: "negative",+ case_id: label === "positive" ? "case-001" : "case-002",+ expected_ids: id,+ expected_source_types: "cve",+ expected_verdicts: verdict,+ expected_binary_label: label,},- },- /fixed.*not_actionable|negative/,-);--function findingFor(- inputId: string,- verdict: string,- rank: number | null,- rankQueue: string | null = verdict,-) {- const finding: TriageFinding = JSON.parse(- outputFor({ inputId, verdict }).split("```json")[1].split("```")[0],- ).findings[0];- finding.exploitability_stack_rank = {- ...finding.exploitability_stack_rank,- rank,- rank_queue: rankQueue,};- return finding;+ if (error)+ assertFails(+ "fixed scanbench case cannot be confirmed",+ output,+ context,+ error,+ );+ else assertPasses(`${verdict} maps to ${label}`, output, context);}const queueFindings = [@@ -161,8 +123,6 @@ const contextFor = (findings: TriageFinding[]) => ({},});const queueContext = contextFor(queueFindings);-const queueOutput = (findings: TriageFinding[]) =>- JSON.stringify({ schema_version: "triage-finding/v0", findings });assertPasses("independent verdict queues",queueOutput(queueFindings),@@ -242,39 +202,29 @@ const inputCases = fs.readFileSync(path.resolve(import.meta.dirname, "../tests/input-types.yaml"),"utf8",);-const scannerCase = inputCases.match(- /case_id: scanner-query[\s\S]*?(?=\n- description:|$)/,-)![0];-const scannerVars: Record<string, string> = { case_id: "scanner-query" };-const scannerContext = { vars: scannerVars };-for (const field of [- "expected_ids",- "expected_source_types",- "expected_verdicts",-]) {- scannerVars[field] = scannerCase.match(new RegExp(field + ": (.+)"))![1];-}-function queryVerdict(inputId: string, verdict: string) {- return JSON.parse(- outputFor({ inputId, sourceType: "scanner_ticket", verdict }).match(- /```json\n([\s\S]*?)\n```/,- )![1],- );-}-const queryOutput = queryVerdict("SCAN-SQL-001", "not_actionable");-queryOutput.findings.push(- queryVerdict("SCAN-SQL-TEST-ONLY", "not_actionable").findings[0],-);-queryOutput.findings[1].triage_item_id = "triage-002";+const scannerContext = (+ parse(inputCases) as { vars: Record<string, string> }[]+).find((item) => item.vars.case_id === "scanner-query")!;+const scannerFinding = (id: string, verdict: string) => ({+ ...findingFor(id, verdict),+ source_type: "scanner_ticket",+});+const queryFindings = [+ scannerFinding("SCAN-SQL-001", "not_actionable"),+ {+ ...scannerFinding("SCAN-SQL-TEST-ONLY", "not_actionable"),+ triage_item_id: "triage-002",+ },+];assertPasses("a SQL text response is not confirmed SQL injection",- JSON.stringify(queryOutput),+ JSON.stringify(documentFor(queryFindings)),scannerContext,);-queryOutput.findings[0] = queryVerdict("SCAN-SQL-001", "confirmed").findings[0];+queryFindings[0] = scannerFinding("SCAN-SQL-001", "confirmed");assertFails("a missing SQL sink cannot earn a confirmed verdict",- JSON.stringify(queryOutput),+ JSON.stringify(documentFor(queryFindings)),scannerContext,/expected verdict not_actionable/,);diff --git a/evals/triage-finding/scripts/test-triage-provider.mts b/evals/triage-finding/scripts/test-triage-provider.mtsindex b7cd3a93c..07cb0ff20 100644--- a/evals/triage-finding/scripts/test-triage-provider.mts+++ b/evals/triage-finding/scripts/test-triage-provider.mts@@ -21,6 +21,48 @@ const fixtureRoot = fs.realpathSync(after(() => fs.rmSync(fixtureRoot, { recursive: true, force: true }));let builtMcp: string | undefined;+function promptfooEnvironment(root: string) {+ return {+ OPENAI_API_KEY: "synthetic-test-key",+ PROMPTFOO_CONFIG_DIR: path.join(root, "state"),+ PROMPTFOO_DISABLE_WAL_MODE: "true",+ PROMPTFOO_DISABLE_TELEMETRY: "1",+ PROMPTFOO_DISABLE_UPDATE: "1",+ };+}++function temporaryRoot(t: TestContext, prefix: string) {+ const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), prefix)));+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ return root;+}++function evalArgs(...args: string[]) {+ return ["eval", ...args, "--no-cache", "--no-share", "--no-progress-bar"];+}++function openDatabase(environment: { PROMPTFOO_CONFIG_DIR: string }) {+ return new DatabaseSync(+ path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),+ { enableForeignKeyConstraints: false },+ );+}++function storedReplayResult(+ environment: { PROMPTFOO_CONFIG_DIR: string },+ id: unknown,+) {+ const database = openDatabase(environment);+ const result = database+ .prepare(+ "SELECT metadata, test_case FROM eval_results WHERE eval_id = ? LIMIT 1",+ )+ .get(String(id));+ database.close();+ assert.ok(result && typeof result.test_case === "string");+ return { metadata: result.metadata, testCase: result.test_case };+}+function replayRunnerArgs() {if (!builtMcp) {const output = path.join(fixtureRoot, "mcp");@@ -168,10 +210,7 @@ test("preloaded runtime preserves native credentials and legacy hooks wrap once",{ skip: process.platform === "win32" },async (t) => {- const root = fs.realpathSync(- fs.mkdtempSync(path.join(os.tmpdir(), "triage-auth-")),- );- t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const root = temporaryRoot(t, "triage-auth-");const priorRuntime = process.env.TRIAGE_RUNTIME_ROOT;process.env.TRIAGE_RUNTIME_ROOT = root;t.after(() => {@@ -281,10 +320,7 @@ test("native CLI provider overrides without a working directory use the staged runtime",{ skip: process.platform === "win32", timeout: 120000 },async (t) => {- const root = fs.realpathSync(- fs.mkdtempSync(path.join(os.tmpdir(), "triage-cli-provider-")),- );- t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const root = temporaryRoot(t, "triage-cli-provider-");const capture = path.join(root, "capture.json");const fakeCodex = path.join(root, "custom-codex");fs.writeFileSync(@@ -321,23 +357,8 @@ console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_t}),);const result = await invoke(- [- "eval",- "-c",- configPath,- "--providers",- provider.id,- "--no-cache",- "--no-share",- "--no-progress-bar",- ],- {- OPENAI_API_KEY: "synthetic-test-key",- PROMPTFOO_CONFIG_DIR: path.join(root, "state"),- PROMPTFOO_DISABLE_WAL_MODE: "true",- PROMPTFOO_DISABLE_TELEMETRY: "1",- PROMPTFOO_DISABLE_UPDATE: "1",- },+ evalArgs("-c", configPath, "--providers", provider.id),+ promptfooEnvironment(root),["--experimental-strip-types", runner],);assert.equal(result.code, 0, result.output);@@ -369,10 +390,7 @@ function providerNode(...parts: string[]) {}function captureProvider(t: TestContext, mode: "launcher" | "replay") {- const root = fs.realpathSync(- fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-")),- );- t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const root = temporaryRoot(t, "triage-provider-");const capture = path.join(root, "captures.jsonl");const fakeCodex = path.join(root, "codex");const fail = path.join(root, "fail");@@ -406,13 +424,9 @@ if (fs.existsSync(${JSON.stringify(fail)})) {const customNode = providerNode("custom-node", "bin", "node");const selectedNode = providerNode("selected-node", "bin", "synthetic-node");const environment = {+ ...promptfooEnvironment(root),...proxies,CODEX_MCP_NODE_PATH: ambientNode,- OPENAI_API_KEY: "synthetic-test-key",- PROMPTFOO_CONFIG_DIR: path.join(root, "state"),- PROMPTFOO_DISABLE_WAL_MODE: "true",- PROMPTFOO_DISABLE_TELEMETRY: "1",- PROMPTFOO_DISABLE_UPDATE: "1",};const readRows = (): Capture[] =>fs@@ -513,21 +527,11 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' ");const simultaneous = await Promise.all([configPath, calibrationConfig, configPath].map((config, index) =>- invoke(- [- "eval",- "-c",- config,- "--no-cache",- "--no-share",- "--no-progress-bar",- ],- {- ...environment,- CODEX_MCP_NODE_PATH: index === 0 ? undefined : ambientNode,- PROMPTFOO_CONFIG_DIR: path.join(root, `concurrent-${index}`),- },- ),+ invoke(evalArgs("-c", config), {+ ...environment,+ CODEX_MCP_NODE_PATH: index === 0 ? undefined : ambientNode,+ PROMPTFOO_CONFIG_DIR: path.join(root, `concurrent-${index}`),+ }),),);for (const result of simultaneous)@@ -626,42 +630,21 @@ test();fs.writeFileSync(fail, "");const templatedInitial = await invoke(- [- "eval",- "-c",- templatedConfig,- "--max-concurrency",- "2",- "--no-cache",- "--no-share",- "--no-progress-bar",- ],+ evalArgs("-c", templatedConfig, "--max-concurrency", "2"),environment,);assert.notEqual(templatedInitial.code, 0, templatedInitial.output);assert.match(templatedInitial.output, /synthetic retryable failure/);fs.rmSync(fail);const templatedRetry = await invoke(- [- "eval",- "--retry-errors",- "--no-cache",- "--no-share",- "--no-progress-bar",- ],+ evalArgs("--retry-errors"),environment,);assert.equal(templatedRetry.code, 0, templatedRetry.output);- const templatedDatabase = new DatabaseSync(- path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),- { enableForeignKeyConstraints: false },- );+ const templatedDatabase = openDatabase(environment);templatedDatabase.prepare("DELETE FROM eval_results").run();templatedDatabase.close();- const templatedResume = await invoke(- ["eval", "--resume", "--no-cache", "--no-share", "--no-progress-bar"],- environment,- );+ const templatedResume = await invoke(evalArgs("--resume"), environment);assert.equal(templatedResume.code, 0, templatedResume.output);const templatedRows = readRows();assert.equal(templatedRows.length, 12);@@ -696,10 +679,7 @@ test("provider resolves each call's Node template without leaking sibling configuration",{ skip: process.platform === "win32", timeout: 120000 },async (t) => {- const root = fs.realpathSync(- fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-vars-")),- );- t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const root = temporaryRoot(t, "triage-provider-vars-");const nodes = ["first & node/node","second/node",@@ -723,15 +703,11 @@ console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,cached_i{ mode: 0o755 },);const environment = {+ ...promptfooEnvironment(root),TRIAGE_RUNTIME_ROOT: root,- PROMPTFOO_CONFIG_DIR: path.join(root, "state"),- PROMPTFOO_DISABLE_TELEMETRY: "1",- PROMPTFOO_DISABLE_UPDATE: "1",- PROMPTFOO_DISABLE_WAL_MODE: "true",PROMPTFOO_DISABLE_TEMPLATING: "",NODE_BASENAME: "node",CODEX_MCP_NODE_PATH: "",- OPENAI_API_KEY: "synthetic-test-key",};const original = new Map(Object.keys(environment).map((key) => [key, process.env[key]]),@@ -1086,10 +1062,7 @@ test("saved native-provider configs retain legacy paths through retry, resume, and viewer replay",{ skip: process.platform === "win32", timeout: 120000 },async (t) => {- const root = fs.realpathSync(- fs.mkdtempSync(path.join(os.tmpdir(), "legacy-provider-")),- );- t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const root = temporaryRoot(t, "legacy-provider-");// Keep the saved source layout independent of the checkout and its path.const legacySourceRoot = path.join(root, "source.checkout");const legacyEvalRoot = path.join(@@ -1187,43 +1160,15 @@ if (fs.existsSync(${JSON.stringify(failure)})) {],}),);- const environment = {- OPENAI_API_KEY: "synthetic-test-key",- PROMPTFOO_CONFIG_DIR: path.join(root, "state"),- PROMPTFOO_DISABLE_WAL_MODE: "true",- PROMPTFOO_DISABLE_TELEMETRY: "1",- PROMPTFOO_DISABLE_UPDATE: "1",- };+ const environment = promptfooEnvironment(root);fs.writeFileSync(failure, "");- const initial = await invoke(- [- "eval",- "-c",- configPath,- "--no-cache",- "--no-share",- "--no-progress-bar",- ],- environment,- );+ const initial = await invoke(evalArgs("-c", configPath), environment);assert.notEqual(initial.code, 0, initial.output);assert.match(initial.output, /synthetic legacy retryable failure/);fs.rmSync(failure);- const retry = await invoke(- [- "eval",- "--retry-errors",- "--no-cache",- "--no-share",- "--no-progress-bar",- ],- environment,- );+ const retry = await invoke(evalArgs("--retry-errors"), environment);assert.equal(retry.code, 0, retry.output);- const database = new DatabaseSync(- path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),- { enableForeignKeyConstraints: false },- );+ const database = openDatabase(environment);const stored = database.prepare("SELECT id, config FROM evals ORDER BY created_at DESC LIMIT 1").get();@@ -1237,25 +1182,12 @@ if (fs.existsSync(${JSON.stringify(failure)})) {assert.equal(saved.extensions?.length || 0, 0);database.exec("DELETE FROM eval_results");database.close();- const resumed = await invoke(- ["eval", "--resume", "--no-cache", "--no-share", "--no-progress-bar"],- environment,- );+ const resumed = await invoke(evalArgs("--resume"), environment);assert.equal(resumed.code, 0, resumed.output);- const resumedDatabase = new DatabaseSync(- path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),- { enableForeignKeyConstraints: false },- );- const resumedResult = resumedDatabase- .prepare(- "SELECT metadata, test_case FROM eval_results WHERE eval_id = ? LIMIT 1",- )- .get(String(stored.id));- resumedDatabase.close();- assert.ok(resumedResult && typeof resumedResult.test_case === "string");+ const resumedResult = storedReplayResult(environment, stored.id);const replayVariables =JSON.parse((resumedResult.metadata as string) || "{}").inputVars ||- JSON.parse(resumedResult.test_case).vars;+ JSON.parse(resumedResult.testCase).vars;await replayInViewer(t, environment, {evaluationId: stored.id,prompt:@@ -1301,14 +1233,8 @@ for (const [suite, legacy] of [`${suite}${legacy ? " legacy hook" : ""} provider survives persisted retry, resume, and viewer replay`,{ skip: process.platform === "win32", timeout: 120000 },async (t) => {- const root = fs.realpathSync(- fs.mkdtempSync(path.join(os.tmpdir(), "sast-provider-replay-")),- );- t.after(() => fs.rmSync(root, { recursive: true, force: true }));- const targetRoot = fs.realpathSync(- fs.mkdtempSync(path.join(os.tmpdir(), "sast-custom-target-")),- );- t.after(() => fs.rmSync(targetRoot, { recursive: true, force: true }));+ const root = temporaryRoot(t, "sast-provider-replay-");+ const targetRoot = temporaryRoot(t, "sast-custom-target-");const targetRepository = path.join(targetRoot, "selected-case");const siblingRepository = path.join(targetRoot, "other-case");fs.mkdirSync(targetRepository);@@ -1336,12 +1262,8 @@ if (fs.existsSync(${JSON.stringify(failure)})) {{ mode: 0o755 },);const environment = {- OPENAI_API_KEY: "synthetic-test-key",+ ...promptfooEnvironment(root),CODEX_MCP_NODE_PATH: process.execPath,- PROMPTFOO_CONFIG_DIR: path.join(root, "state"),- PROMPTFOO_DISABLE_WAL_MODE: "true",- PROMPTFOO_DISABLE_TELEMETRY: "1",- PROMPTFOO_DISABLE_UPDATE: "1",};const template = parse(fs.readFileSync(@@ -1408,23 +1330,10 @@ if (fs.existsSync(${JSON.stringify(failure)})) {}),);fs.writeFileSync(failure, "");- const initial = await invoke(- [- "eval",- "-c",- configPath,- "--no-cache",- "--no-share",- "--no-progress-bar",- ],- environment,- );+ const initial = await invoke(evalArgs("-c", configPath), environment);assert.notEqual(initial.code, 0, initial.output);assert.match(initial.output, /synthetic retryable failure/);- const initialDatabase = new DatabaseSync(- path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),- { enableForeignKeyConstraints: false },- );+ const initialDatabase = openDatabase(environment);const initialEvaluation = initialDatabase.prepare("SELECT config FROM evals ORDER BY created_at DESC LIMIT 1").get();@@ -1468,21 +1377,9 @@ if (fs.existsSync(${JSON.stringify(failure)})) {);}fs.rmSync(failure);- const retry = await invoke(- [- "eval",- "--retry-errors",- "--no-cache",- "--no-share",- "--no-progress-bar",- ],- environment,- );+ const retry = await invoke(evalArgs("--retry-errors"), environment);assert.equal(retry.code, 0, retry.output);- const database = new DatabaseSync(- path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),- { enableForeignKeyConstraints: false },- );+ const database = openDatabase(environment);const stored = database.prepare("SELECT id, config FROM evals ORDER BY created_at DESC LIMIT 1",@@ -1504,27 +1401,14 @@ if (fs.existsSync(${JSON.stringify(failure)})) {);database.prepare("DELETE FROM eval_results").run();database.close();- const resumed = await invoke(- ["eval", "--resume", "--no-cache", "--no-share", "--no-progress-bar"],- environment,- );+ const resumed = await invoke(evalArgs("--resume"), environment);assert.equal(resumed.code, 0, resumed.output);- const resumedDatabase = new DatabaseSync(- path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),- { enableForeignKeyConstraints: false },- );- const resumedResult = resumedDatabase- .prepare(- "SELECT metadata, test_case FROM eval_results WHERE eval_id = ? LIMIT 1",- )- .get(String(stored.id));- resumedDatabase.close();- assert.ok(resumedResult && typeof resumedResult.test_case === "string");+ const resumedResult = storedReplayResult(environment, stored.id);// Match the viewer: prefer saved inputVars, then the resolved test-case vars.const replayVariables =JSON.parse((resumedResult.metadata as string | null) ?? "{}")- .inputVars || JSON.parse(resumedResult.test_case).vars;+ .inputVars || JSON.parse(resumedResult.testCase).vars;assert.equal(typeof replayVariables.target_repo, "string");const viewerOutput = await replayInViewer(t, environment, {diff --git a/evals/triage-finding/tests/github-rest-intake.yaml b/evals/triage-finding/tests/github-rest-intake.yamlindex 27b9c7996..dad7fbe44 100644--- a/evals/triage-finding/tests/github-rest-intake.yaml+++ b/evals/triage-finding/tests/github-rest-intake.yaml@@ -13,9 +13,9 @@This is an automated eval. A GitHub repository was supplied, but noGitHub finding source was specified. Follow the GitHub REST intakebehavior and do not triage findings yet.- options:+ options: &intakeOnlydisableDefaultAsserts: true- assert:+ assert: &githubIntake- type: javascriptvalue: file://./assertions/github-rest-intake.mts@@ -36,11 +36,8 @@the current Codex project attachment indicates GitHub repositorypromptfoo/promptfoo. Infer that repository, ask only for the GitHubfinding source, and do not triage findings yet.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/github-rest-intake.mts+ options: *intakeOnly+ assert: *githubIntake- description: "GitHub Dependabot malware uses REST malware classification"metadata:@@ -53,15 +50,12 @@finding_input: "Source requested: Dependabot malware"invocation_preamble: "Use $codex-security:triage-finding for Dependabot malware in GitHub repo promptfoo/promptfoo."expected_github_rest_behavior: dependabot_malware- eval_instructions: |-+ eval_instructions: &restIntakeInstructions |-This is an automated eval. Describe the GitHub REST intake that should beused for this source. Do not query live GitHub, inspect code, or emit thetriage JSON result.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/github-rest-intake.mts+ options: *intakeOnly+ assert: *githubIntake- description: "GitHub code scanning uses REST alerts and instances"metadata:@@ -80,11 +74,8 @@path and a parameters object with the required query parameters. Use theliteral {alert_number} placeholder in the instances path. Do not query liveGitHub, inspect code, or emit the triage JSON result.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/github-rest-intake.mts+ options: *intakeOnly+ assert: *githubIntake- description: "GitHub advisories include private report triage state"metadata:@@ -97,15 +88,9 @@finding_input: "Source requested: security advisories and private vulnerability reports"invocation_preamble: "Use $codex-security:triage-finding for security advisories and private vulnerability reports in GitHub repo promptfoo/promptfoo."expected_github_rest_behavior: advisories_private_reports- eval_instructions: |-- This is an automated eval. Describe the GitHub REST intake that should be- used for this source. Do not query live GitHub, inspect code, or emit the- triage JSON result.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/github-rest-intake.mts+ eval_instructions: *restIntakeInstructions+ options: *intakeOnly+ assert: *githubIntake- description: "GitHub intake selects REST without an explicit Connector request"metadata:@@ -122,11 +107,8 @@This is an automated eval of the transport choice. Do not query liveGitHub, inspect code, or emit a triage result. Return one JSON object withonly transport, chosen from "rest", "github_connector_read_only", or "other".- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/github-rest-intake.mts+ options: *intakeOnly+ assert: *githubIntake- description: "Explicit GitHub Connector request preserves the selected transport"metadata:@@ -149,11 +131,8 @@"explain_and_request_rest_approval".- scope: an object with account ("any" or "user_specified_or_approved")and repository (the owner/repo identifier for the proposed REST scope).- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/github-rest-intake.mts+ options: *intakeOnly+ assert: *githubIntake- description: "Explicit GitHub Issue is supported outside all-source intake"metadata:@@ -170,8 +149,5 @@This is an automated eval. Describe how GitHub Issues should be handledby GitHub REST intake. Do not query live GitHub, inspect code, or emit thetriage JSON result.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/github-rest-intake.mts+ options: *intakeOnly+ assert: *githubIntakediff --git a/evals/triage-finding/tests/ticket-intake.yaml b/evals/triage-finding/tests/ticket-intake.yamlindex ad18f4ee4..7c8f44446 100644--- a/evals/triage-finding/tests/ticket-intake.yaml+++ b/evals/triage-finding/tests/ticket-intake.yaml@@ -9,13 +9,13 @@finding_input: "Linear issue SEC-293. Connector observation: the Linear connector is unavailable because it is not connected."invocation_preamble: "Use $codex-security:triage-finding to import Linear issue SEC-293."expected_ticket_failure: unavailable- eval_instructions: |-+ eval_instructions: &failedReadInstructions |-This is an automated eval. Treat the connector observation as the resultof the requested read. Respond exactly as the skill requires after thisintake failure. Do not inspect code or fabricate the issue contents.- options:+ options: &intakeOnlydisableDefaultAsserts: true- assert:+ assert: &ticketIntake- type: javascriptvalue: file://./assertions/ticket-intake.mts@@ -30,15 +30,9 @@finding_input: "Linear issue SEC-293. Connector observation: access was denied because the authenticated user lacks permission."invocation_preamble: "Use $codex-security:triage-finding to import Linear issue SEC-293."expected_ticket_failure: permission- eval_instructions: |-- This is an automated eval. Treat the connector observation as the result- of the requested read. Respond exactly as the skill requires after this- intake failure. Do not inspect code or fabricate the issue contents.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/ticket-intake.mts+ eval_instructions: *failedReadInstructions+ options: *intakeOnly+ assert: *ticketIntake- description: "Missing Linear issue stops with identifier guidance"metadata:@@ -51,15 +45,9 @@finding_input: "Linear issue SEC-999999. Connector observation: the issue was not found or is inaccessible."invocation_preamble: "Use $codex-security:triage-finding to import Linear issue SEC-999999."expected_ticket_failure: not_found- eval_instructions: |-- This is an automated eval. Treat the connector observation as the result- of the requested read. Respond exactly as the skill requires after this- intake failure. Do not inspect code or fabricate the issue contents.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/ticket-intake.mts+ eval_instructions: *failedReadInstructions+ options: *intakeOnly+ assert: *ticketIntake- description: "Repeated transient Linear failure stops after one retry"metadata:@@ -77,11 +65,8 @@of the requested read and its one retry. Respond exactly as the skillrequires after retry exhaustion. Do not inspect code or fabricate theissue contents.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/ticket-intake.mts+ options: *intakeOnly+ assert: *ticketIntake- description: "Linear parent asks before importing direct sub-issues"metadata:@@ -98,11 +83,8 @@This is an automated eval. Treat the parent fetch and child listing ascompleted connector observations. Follow the skill before importing anychild issue. Do not inspect code or fabricate child issue contents.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/ticket-intake.mts+ options: *intakeOnly+ assert: *ticketIntake- description: "Linear child-only selection asks before expanding to grandchildren"metadata:@@ -119,11 +101,8 @@This is an automated eval. Treat the connector observations as complete.Follow the skill before importing the next depth. Do not inspect code orfabricate the nested issue contents.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/ticket-intake.mts+ options: *intakeOnly+ assert: *ticketIntake- description: "Linear all-descendants selection proceeds across depths"metadata:@@ -174,8 +153,5 @@This is an automated eval. The parent may or may not be an independentfinding. Follow the skill before normalizing the parent. Do not inspectcode or emit triage JSON.- options:- disableDefaultAsserts: true- assert:- - type: javascript- value: file://./assertions/ticket-intake.mts+ options: *intakeOnly+ assert: *ticketIntake