OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
import assert from "node:assert/strict";
import { execFileSync, spawn } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import { createServer } from "node:net";
import path from "node:path";
import test, { after, type TestContext } from "node:test";
import { DatabaseSync } from "node:sqlite";
import { parse } from "yaml";
import { beforeAll } from "./triage-provider.mts";
const evalRoot = path.resolve(import.meta.dirname, "..");
const runner = path.join(import.meta.dirname, "run-promptfoo.mts");
const fixtureRunner = path.join(
import.meta.dirname,
"run-provider-fixture.mts",
);
const fixtureRoot = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-bundle-")),
);
after(() => fs.rmSync(fixtureRoot, { recursive: true, force: true }));
let builtMcp: string | undefined;
function promptfooEnvironment(root: string) {
return {
OPENAI_API_KEY: "synthetic-test-key",
PROMPTFOO_CONFIG_DIR: path.join(root, "state"),
PROMPTFOO_DISABLE_WAL_MODE: "true",
PROMPTFOO_DISABLE_TELEMETRY: "1",
PROMPTFOO_DISABLE_UPDATE: "1",
};
}
function temporaryRoot(t: TestContext, prefix: string) {
const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), prefix)));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
return root;
}
function evalArgs(...args: string[]) {
return ["eval", ...args, "--no-cache", "--no-share", "--no-progress-bar"];
}
function openDatabase(environment: { PROMPTFOO_CONFIG_DIR: string }) {
return new DatabaseSync(
path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),
{ enableForeignKeyConstraints: false },
);
}
function storedReplayResult(
environment: { PROMPTFOO_CONFIG_DIR: string },
id: unknown,
) {
const database = openDatabase(environment);
const result = database
.prepare(
"SELECT metadata, test_case FROM eval_results WHERE eval_id = ? LIMIT 1",
)
.get(String(id));
database.close();
assert.ok(result && typeof result.test_case === "string");
return { metadata: result.metadata, testCase: result.test_case };
}
function replayRunnerArgs() {
if (!builtMcp) {
const output = path.join(fixtureRoot, "mcp");
const mcpApp = path.join(evalRoot, "../../plugins/codex-security/mcp-app");
execFileSync(
process.execPath,
[
path.join(mcpApp, "scripts/build_mcp_app.mjs"),
"--output",
output,
"--native",
"host",
],
{ cwd: mcpApp },
);
builtMcp = output;
}
// Every child copies the same build into its own throwaway skill runtime.
return ["--experimental-strip-types", fixtureRunner, builtMcp];
}
const probeFixturePolicy = `
const launcher = path.join(cwd, 'plugins/codex-security/scripts/launch_codex_security_mcp');
const fixture = path.join(cwd, 'evals/triage-finding/fixtures/repo');
fs.writeFileSync(path.join(fixture, 'SECURITY.md'), '# Synthetic policy\\n');
const policy = require('node:child_process').execFileSync(launcher, ['--helper', 'resolve-security-md', '--repo', fixture, '--scope', 'src/server.js', '--out', '-'], {encoding:'utf8'});
`;
async function replayInViewer(
t: TestContext,
environment: NodeJS.ProcessEnv,
request: {
evaluationId: unknown;
prompt: string;
variables: Record<string, unknown>;
},
) {
const listener = createServer();
await new Promise<void>((resolve) =>
listener.listen(0, "127.0.0.1", resolve),
);
const address = listener.address();
assert.ok(address && typeof address !== "string");
const port = address.port;
await new Promise<void>((resolve, reject) =>
listener.close((error) => (error ? reject(error) : resolve())),
);
const viewer = spawn(
process.execPath,
[...replayRunnerArgs(), "view", "--port", String(port), "--no"],
{
cwd: evalRoot,
env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },
stdio: ["ignore", "pipe", "pipe"],
},
);
let viewerOutput = "";
for (const stream of [viewer.stdout, viewer.stderr])
stream.on("data", (chunk) => {
viewerOutput += chunk;
});
const closed = new Promise<number | null>((resolve, reject) => {
viewer.once("error", reject);
viewer.once("close", resolve);
});
t.after(async () => {
if (viewer.exitCode === null) viewer.kill("SIGTERM");
await closed;
});
const url = `http://127.0.0.1:${port}`;
while (true) {
try {
const response = await fetch(`${url}/api/eval`);
if (response.ok) break;
} catch {
/* The actual viewer is still building its staged runtime. */
}
assert.equal(viewer.exitCode, null, viewerOutput);
await new Promise((resolve) => setTimeout(resolve, 50));
}
const replay = await fetch(`${url}/api/eval/replay`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ testIndex: 0, ...request }),
});
assert.equal(replay.status, 200, viewerOutput);
const replayResult = (await replay.json()) as { output: unknown };
assert.equal(replayResult.output, "ok", JSON.stringify(replayResult));
viewer.kill("SIGTERM");
await closed;
return viewerOutput;
}
function providerConfig(name: string) {
return parse(
fs.readFileSync(path.join(evalRoot, "providers", `${name}.yaml`), "utf8"),
);
}
const extensions = [
`file://${path.join(import.meta.dirname, "triage-provider.mts")}:beforeAll`,
];
interface Capture {
cwd: string;
policy: string;
proxies: Record<string, string>;
nodePath: string;
nodeTarget: string;
directories: string[];
overrides: string[];
}
const proxies = {
HTTP_PROXY: "http://http.example.test:8080",
HTTPS_PROXY: "http://https.example.test:8080",
ALL_PROXY: "http://all.example.test:8080",
NO_PROXY: "localhost,.example.test",
};
function invoke(
args: string[],
environment: NodeJS.ProcessEnv,
runnerArgs = replayRunnerArgs(),
) {
return new Promise<{ code: number | null; output: string }>(
(resolve, reject) => {
const child = spawn(process.execPath, [...runnerArgs, ...args], {
cwd: evalRoot,
env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },
stdio: ["ignore", "pipe", "pipe"],
});
let output = "";
child.stdout.on("data", (chunk) => {
output += chunk;
});
child.stderr.on("data", (chunk) => {
output += chunk;
});
child.once("error", reject);
child.once("close", (code) => resolve({ code, output }));
},
);
}
test(
"preloaded runtime preserves native credentials and legacy hooks wrap once",
{ skip: process.platform === "win32" },
async (t) => {
const root = temporaryRoot(t, "triage-auth-");
const priorRuntime = process.env.TRIAGE_RUNTIME_ROOT;
process.env.TRIAGE_RUNTIME_ROOT = root;
t.after(() => {
if (priorRuntime === undefined) delete process.env.TRIAGE_RUNTIME_ROOT;
else process.env.TRIAGE_RUNTIME_ROOT = priorRuntime;
});
const fakeCodex = path.join(root, "codex");
fs.writeFileSync(
fakeCodex,
`#!${process.execPath}
console.log(JSON.stringify({type:'thread.started',thread_id:'synthetic-auth'}));
console.log(JSON.stringify({type:'item.completed',item:{id:'message',type:'agent_message',text:JSON.stringify({apiKey:process.env.CODEX_API_KEY,cwd:process.argv[process.argv.indexOf('--cd')+1],directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])})}}));
console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_tokens:1}}));
`,
{ mode: 0o755 },
);
await import("./initialize-provider.mts");
const { loadApiProvider } = await import("promptfoo");
const cases = [
{
env: { OPENAI_API_KEY: "synthetic-global" },
providerEnv: { CODEX_API_KEY: "synthetic-provider" },
expected: "synthetic-provider",
},
{
env: { CODEX_API_KEY: "synthetic-global" },
providerEnv: { OPENAI_API_KEY: "synthetic-provider" },
expected: "synthetic-provider",
},
{
env: { OPENAI_API_KEY: "synthetic-global" },
providerEnv: {
OPENAI_API_KEY: "synthetic-openai",
CODEX_API_KEY: "synthetic-codex",
},
expected: "synthetic-openai",
},
{
env: { OPENAI_API_KEY: "synthetic-global" },
providerEnv: { CODEX_API_KEY: "synthetic-provider" },
apiKey: "synthetic-explicit",
expected: "synthetic-explicit",
},
{
env: { OPENAI_API_KEY: "synthetic-global" },
providerEnv: { CODEX_API_KEY: "" },
expected: "synthetic-global",
},
];
for (const entry of cases) {
const load = (runtime = false) =>
loadApiProvider("openai:codex-sdk:gpt-5.5", {
basePath: root,
env: entry.env,
options: {
env: entry.providerEnv,
config: {
cli_config: runtime
? { default_permissions: "triage_runtime_only" }
: undefined,
working_dir: root,
codex_path_override: fakeCodex,
skip_git_repo_check: true,
model: "gpt-5.5",
maxRetries: 0,
apiKey: entry.apiKey,
cli_env: { CODEX_MCP_NODE_PATH: process.execPath },
},
},
});
const native = await load();
const adapted = await load(true);
try {
const preloaded = await adapted.callApi("synthetic");
const context = { suite: { providers: [adapted] } };
beforeAll(context);
beforeAll(context);
const original = await native.callApi("synthetic");
const result = await adapted.callApi("synthetic");
assert.equal(preloaded.error, undefined);
assert.deepEqual(
JSON.parse(String(preloaded.output)),
JSON.parse(String(result.output)),
);
assert.equal(original.error, undefined);
assert.equal(result.error, undefined);
const originalOutput = JSON.parse(String(original.output));
const output = JSON.parse(String(result.output));
assert.equal(originalOutput.apiKey, entry.expected);
assert.equal(output.apiKey, originalOutput.apiKey);
assert.equal(output.cwd, root);
assert.equal(adapted.id(), native.id());
assert.deepEqual(output.directories, [
path.dirname(fs.realpathSync(process.execPath)),
path.dirname(process.execPath),
path.dirname(fakeCodex),
]);
} finally {
await native.cleanup?.();
await adapted.cleanup?.();
}
}
},
);
test(
"native CLI provider overrides without a working directory use the staged runtime",
{ skip: process.platform === "win32", timeout: 120000 },
async (t) => {
const root = temporaryRoot(t, "triage-cli-provider-");
const capture = path.join(root, "capture.json");
const fakeCodex = path.join(root, "custom-codex");
fs.writeFileSync(
fakeCodex,
`#!${process.execPath}
const fs = require('node:fs');
const path = require('node:path');
const cwd = process.argv[process.argv.indexOf('--cd') + 1];
fs.readFileSync(path.join(cwd, 'plugins/codex-security/skills/triage-finding/SKILL.md'));
${probeFixturePolicy}
fs.writeFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy,node:process.env.CODEX_MCP_NODE_PATH,directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])}));
console.log(JSON.stringify({type:'thread.started',thread_id:'synthetic-cli-provider'}));
console.log(JSON.stringify({type:'item.completed',item:{id:'message',type:'agent_message',text:'ok'}}));
console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_tokens:1}}));
`,
{ mode: 0o755 },
);
const provider = {
id: "openai:codex-sdk:gpt-5.5",
config: {
codex_path_override: fakeCodex,
skip_git_repo_check: true,
approval_policy: "never",
cli_env: { CODEX_MCP_NODE_PATH: process.execPath },
},
};
const configPath = path.join(root, "config.json");
fs.writeFileSync(
configPath,
JSON.stringify({
providers: [provider],
prompts: ["hello"],
tests: [{ assert: [{ type: "equals", value: "ok" }] }],
}),
);
const result = await invoke(
evalArgs("-c", configPath, "--providers", provider.id),
promptfooEnvironment(root),
["--experimental-strip-types", runner],
);
assert.equal(result.code, 0, result.output);
const row = JSON.parse(fs.readFileSync(capture, "utf8"));
assert.notEqual(row.cwd, evalRoot);
assert.match(row.policy, /Synthetic policy/);
assert.equal(fs.existsSync(row.cwd), false);
assert.equal(row.node, process.execPath);
assert.deepEqual(row.directories, [
path.dirname(fs.realpathSync(process.execPath)),
path.dirname(process.execPath),
path.dirname(fakeCodex),
]);
},
);
// Reuse the immutable Node binaries; captures and Promptfoo databases belong to each test.
function providerNode(...parts: string[]) {
const executable = path.join(fixtureRoot, ...parts);
if (!fs.existsSync(executable)) {
fs.mkdirSync(path.dirname(executable), { recursive: true });
fs.copyFileSync(
process.execPath,
executable,
fs.constants.COPYFILE_FICLONE,
);
}
return executable;
}
function captureProvider(t: TestContext, mode: "launcher" | "replay") {
const root = temporaryRoot(t, "triage-provider-");
const capture = path.join(root, "captures.jsonl");
const fakeCodex = path.join(root, "codex");
const fail = path.join(root, "fail");
fs.writeFileSync(
fakeCodex,
`#!${process.execPath}
const fs = require('node:fs');
const path = require('node:path');
const cwd = process.argv[process.argv.indexOf('--cd') + 1];
fs.readFileSync(path.join(cwd, 'plugins/codex-security/skills/triage-finding/SKILL.md'));
// Each replay has a fresh cwd. One concurrent caller claims its helper probe.
let capturedPolicy = '';
if (${mode === "launcher"} || (!fs.existsSync(${JSON.stringify(fail)}) && fs.mkdirSync(path.join(cwd, '.policy-probe'), {recursive: true}))) {
${probeFixturePolicy}
capturedPolicy = policy;
}
const directories = process.argv.flatMap((arg, index) => arg === '--add-dir' ? [process.argv[index + 1]] : []);
const overrides = process.argv.flatMap((arg, index) => arg === '--config' ? [process.argv[index + 1]] : []);
fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy:capturedPolicy,directories,overrides,nodePath:process.env.CODEX_MCP_NODE_PATH,nodeTarget:fs.realpathSync.native(process.env.CODEX_MCP_NODE_PATH), proxies: Object.fromEntries(${JSON.stringify(Object.keys(proxies))}.map(key => [key,process.env[key]]))}) + '\\n');
console.log(JSON.stringify({type:'thread.started', thread_id:'synthetic-thread'}));
if (fs.existsSync(${JSON.stringify(fail)})) {
console.log(JSON.stringify({type:'turn.failed',error:{message:'synthetic retryable failure'}}));
} else {
console.log(JSON.stringify({type:'item.completed', item:{id:'message',type:'agent_message',text:'ok'}}));
console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,cached_input_tokens:0,output_tokens:1}}));
}
`,
{ mode: 0o755 },
);
const ambientNode = providerNode("ambient-node", "bin", "node");
const customNode = providerNode("custom-node", "bin", "node");
const selectedNode = providerNode("selected-node", "bin", "synthetic-node");
const environment = {
...promptfooEnvironment(root),
...proxies,
CODEX_MCP_NODE_PATH: ambientNode,
};
const readRows = (): Capture[] =>
fs
.readFileSync(capture, "utf8")
.trim()
.split("\n")
.map((line) => JSON.parse(line) as Capture);
return {
root,
fakeCodex,
fail,
ambientNode,
customNode,
selectedNode,
environment,
readRows,
};
}
// Exercise the pinned provider and Codex SDK subprocess without a model request.
test(
"provider preserves proxies and selects each Node launcher",
{ skip: process.platform === "win32", timeout: 120000 },
async (t) => {
const {
root,
fakeCodex,
ambientNode,
customNode,
selectedNode,
environment,
readRows,
} = captureProvider(t, "launcher");
const configPath = path.join(root, "config.json");
const provider = providerConfig("triage");
provider.config.codex_path_override = fakeCodex;
fs.writeFileSync(
configPath,
JSON.stringify({
providers: [provider],
prompts: ["hello"],
tests: [{ assert: [{ type: "equals", value: "ok" }] }],
}),
);
const calibration = providerConfig("calibration");
calibration.id = provider.id;
calibration.config.codex_path_override = fakeCodex;
calibration.config.cli_env.CODEX_MCP_NODE_PATH = customNode;
const nodeAlias = path.join(root, "node-alias");
const nodeDispatcher = path.join(root, "node-runtime", "dispatcher");
fs.mkdirSync(path.dirname(nodeDispatcher));
fs.writeFileSync(
nodeDispatcher,
`#!/bin/sh
case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "$@" ;; *) echo 'synthetic dispatcher requires node-alias' >&2; exit 42 ;; esac
`,
{ mode: 0o755 },
);
fs.symlinkSync(nodeDispatcher, nodeAlias);
const nodeChoices = [
customNode,
path.relative(path.join(os.tmpdir(), "runtime-placeholder"), customNode),
"node",
nodeAlias,
path.join(root, "missing-node"),
"synthetic-node",
];
const calibrationConfig = path.join(root, "calibration.json");
fs.mkdirSync(path.join(root, "case"));
fs.writeFileSync(
calibrationConfig,
JSON.stringify({
defaultTest: {
options: {
transformVars: `file://${path.join(import.meta.dirname, "runtime-vars.mts")}`,
},
},
providers: nodeChoices.map((nodePath, index) => ({
...calibration,
label: `custom-runtime-${index}`,
config: {
...calibration.config,
cli_env: {
...calibration.config.cli_env,
CODEX_MCP_NODE_PATH: nodePath,
PATH: `${path.dirname(selectedNode)}${path.delimiter}${path.dirname(customNode)}${path.delimiter}${process.env.PATH}`,
},
},
})),
prompts: ["hello"],
tests: [
{
vars: { calibration_repo_root: root, calibration_repo: "case" },
assert: [{ type: "equals", value: "ok" }],
},
],
}),
);
const simultaneous = await Promise.all(
[configPath, calibrationConfig, configPath].map((config, index) =>
invoke(evalArgs("-c", config), {
...environment,
CODEX_MCP_NODE_PATH: index === 0 ? undefined : ambientNode,
PROMPTFOO_CONFIG_DIR: path.join(root, `concurrent-${index}`),
}),
),
);
for (const result of simultaneous)
assert.equal(result.code, 0, result.output);
const rows = readRows();
assert.equal(rows.length, 8);
assert.equal(new Set(rows.map((row) => row.cwd)).size, 3);
for (const row of rows) {
assert.deepEqual(row.proxies, proxies);
assert.match(row.policy, /Synthetic policy/);
assert.equal(fs.existsSync(row.cwd), false);
assert.match(
row.overrides.join("\n"),
/permissions\.triage_runtime_only\.filesystem\.:workspace_roots="read"/,
);
assert.deepEqual(row.directories, [
...([customNode, selectedNode, nodeDispatcher].includes(row.nodeTarget)
? [path.join(root, "case")]
: []),
path.dirname(row.nodeTarget),
path.dirname(row.nodePath),
path.dirname(fakeCodex),
]);
}
assert.equal(rows.filter((row) => row.nodeTarget === customNode).length, 4);
assert.equal(rows.filter((row) => row.nodePath === nodeAlias).length, 1);
assert.equal(rows.filter((row) => row.nodePath === selectedNode).length, 1);
assert.equal(rows.filter((row) => row.nodePath === ambientNode).length, 1);
assert.equal(
rows.filter((row) => row.nodePath === fs.realpathSync(process.execPath))
.length,
1,
);
},
);
test(
"provider rebinds saved templated settings for retry and resume",
{ skip: process.platform === "win32", timeout: 120000 },
async (t) => {
const {
root,
fakeCodex,
fail,
customNode,
selectedNode,
environment,
readRows,
} = captureProvider(t, "replay");
const provider = providerConfig("triage");
const templatedConfig = path.join(root, "templated.json");
fs.writeFileSync(
templatedConfig,
JSON.stringify({
providers: [
{
...provider,
config: {
...provider.config,
codex_path_override: fakeCodex,
cli_env: {
...provider.config.cli_env,
CODEX_MCP_NODE_PATH: "{{node_path}}",
},
additional_directories: ["{{permission_root}}"],
},
},
],
prompts: [
"hello",
{
id: "hello with an override",
label: "per-case runtime override",
config: {
cli_env: {
...provider.config.cli_env,
CODEX_MCP_NODE_PATH: "{{override_node_path}}",
},
additional_directories: ["{{override_permission_root}}"],
},
},
],
tests: [customNode, selectedNode].map((nodePath) => ({
vars: {
node_path: nodePath,
permission_root: path.dirname(nodePath),
override_node_path:
nodePath === customNode ? selectedNode : customNode,
override_permission_root: path.dirname(
nodePath === customNode ? selectedNode : customNode,
),
},
assert: [{ type: "equals", value: "ok" }],
})),
}),
);
fs.writeFileSync(fail, "");
const templatedInitial = await invoke(
evalArgs("-c", templatedConfig, "--max-concurrency", "2"),
environment,
);
assert.notEqual(templatedInitial.code, 0, templatedInitial.output);
assert.match(templatedInitial.output, /synthetic retryable failure/);
fs.rmSync(fail);
const templatedRetry = await invoke(
evalArgs("--retry-errors"),
environment,
);
assert.equal(templatedRetry.code, 0, templatedRetry.output);
const templatedDatabase = openDatabase(environment);
templatedDatabase.prepare("DELETE FROM eval_results").run();
templatedDatabase.close();
const templatedResume = await invoke(evalArgs("--resume"), environment);
assert.equal(templatedResume.code, 0, templatedResume.output);
const templatedRows = readRows();
assert.equal(templatedRows.length, 12);
assert.equal(new Set(templatedRows.map((row) => row.cwd)).size, 3);
const replayProbes = templatedRows.filter((row) => row.policy);
assert.equal(replayProbes.length, 2);
assert.equal(new Set(replayProbes.map((row) => row.cwd)).size, 2);
for (const row of replayProbes)
assert.match(row.policy, /Synthetic policy/);
assert.equal(
templatedRows.filter((row) => row.nodePath === customNode).length,
6,
);
assert.equal(
templatedRows.filter((row) => row.nodePath === selectedNode).length,
6,
);
for (const row of templatedRows) {
assert.deepEqual(row.proxies, proxies);
assert.equal(fs.existsSync(row.cwd), false);
assert.deepEqual(row.directories, [
path.dirname(row.nodePath),
path.dirname(row.nodePath),
path.dirname(row.nodePath),
path.dirname(fakeCodex),
]);
}
},
);
test(
"provider resolves each call's Node template without leaking sibling configuration",
{ skip: process.platform === "win32", timeout: 120000 },
async (t) => {
const root = temporaryRoot(t, "triage-provider-vars-");
const nodes = [
"first & node/node",
"second/node",
"literal/{{custom_node}}",
].map((name) => path.join(root, name)) as [string, string, string];
for (const node of nodes) {
fs.mkdirSync(path.dirname(node), { recursive: true });
fs.writeFileSync(node, "#!/bin/sh\nexit 0\n", { mode: 0o755 });
}
const extra = path.join(root, "extra");
fs.mkdirSync(extra);
const fakeCodex = path.join(root, "codex");
fs.writeFileSync(
fakeCodex,
`#!${process.execPath}
const details = {executable:process.env.CODEX_TEST_EXECUTABLE || require('node:fs').realpathSync(process.argv[1]), path:process.env.PATH, node:process.env.CODEX_MCP_NODE_PATH, marker:process.env.EXTRA_MARKER, directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])};
console.log(JSON.stringify({type:'thread.started',thread_id:'synthetic-thread'}));
console.log(JSON.stringify({type:'item.completed',item:{id:'message',type:'agent_message',text:JSON.stringify(details)}}));
console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,cached_input_tokens:0,output_tokens:1}}));
`,
{ mode: 0o755 },
);
const environment = {
...promptfooEnvironment(root),
TRIAGE_RUNTIME_ROOT: root,
PROMPTFOO_DISABLE_TEMPLATING: "",
NODE_BASENAME: "node",
CODEX_MCP_NODE_PATH: "",
};
const original = new Map(
Object.keys(environment).map((key) => [key, process.env[key]]),
);
t.after(() => {
for (const [key, value] of original) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
});
Object.assign(process.env, environment);
const { loadApiProvider } = await import("promptfoo");
const load = (
template: string | undefined,
command = fakeCodex,
cliEnvironment: Record<string, string> = {},
) =>
loadApiProvider("openai:codex-sdk:gpt-5.5", {
basePath: root,
options: {
config: {
working_dir: root,
codex_path_override: command,
skip_git_repo_check: true,
model: "gpt-5.5",
maxRetries: 0,
cli_env: {
...cliEnvironment,
EXTRA_MARKER: "{{marker}}",
...(template === undefined
? {}
: { CODEX_MCP_NODE_PATH: template }),
},
},
},
}).then((provider) => {
beforeAll({ suite: { providers: [provider] } });
return provider;
});
for (const template of [
"{{custom_node}}",
'{{custom_node | replace("second", "second")}}',
"{{custom_node_json | load}}",
nodes[0],
]) {
const provider = await load(template);
const overrides = template === nodes[0];
const invoke = async (node: string) => {
const context = {
vars: {
custom_node: node,
custom_node_json: JSON.stringify(node),
marker: node,
},
prompt: { raw: "synthetic", label: "synthetic" },
...(overrides
? {
prompt: {
raw: "synthetic",
label: "synthetic",
config: {
cli_env: {
CODEX_MCP_NODE_PATH: "{{custom_node}}",
EXTRA_MARKER: "{{marker}}",
},
additional_directories: [extra],
},
},
}
: {}),
};
const saved = structuredClone(context);
const result = await provider.callApi(
"Return synthetic launch details",
context as Parameters<typeof provider.callApi>[1],
);
assert.equal(result.error, undefined);
const captured = JSON.parse(String(result.output));
assert.equal(captured.node, node);
assert.deepEqual(captured.directories, [
...(overrides ? [extra] : []),
path.dirname(node),
path.dirname(node),
path.dirname(fakeCodex),
]);
if (overrides) assert.equal(captured.marker, node);
assert.deepEqual(context, saved);
};
try {
for (const node of nodes.slice(0, 2)) await invoke(node);
await Promise.all(nodes.slice(0, 2).map(invoke));
} finally {
await provider.cleanup?.();
}
}
for (const sibling of [false, true]) {
const nodeRoot = path.join(
root,
sibling ? "node-sibling" : "node-missing",
);
const target = path.join(nodeRoot, "target", "node");
const alias = path.join(nodeRoot, "alias");
fs.mkdirSync(path.join(nodeRoot, "target", "child"), { recursive: true });
fs.copyFileSync(nodes[0], target);
fs.symlinkSync(path.join(nodeRoot, "target", "child"), alias);
if (sibling) fs.copyFileSync(nodes[1], path.join(nodeRoot, "node"));
for (const [command, expected, searchPath] of [
[`${alias}/../node`, target, process.env.PATH],
[`${path.relative(root, alias)}/../node`, target, process.env.PATH],
["node", target, `${alias}/..`],
["node", target, `${path.relative(root, alias)}/..`],
[
path.join(nodeRoot, "target") + "/child/../node",
target,
process.env.PATH,
],
[path.join(nodeRoot, "missing"), nodes[0], path.dirname(nodes[0])],
] as const) {
const provider = await load(command, fakeCodex, { PATH: searchPath! });
try {
const result = await provider.callApi("synthetic");
assert.equal(result.error, undefined);
const captured = JSON.parse(String(result.output));
assert.equal(
fs.realpathSync.native(captured.node),
expected,
command,
);
assert.deepEqual(captured.directories, [
path.dirname(expected),
path.dirname(captured.node),
path.dirname(fakeCodex),
]);
} finally {
await provider.cleanup?.();
}
}
}
const alias = path.join(root, "codex-alias");
fs.symlinkSync(fakeCodex, alias);
const selectedPath = `${root}${path.delimiter}${process.env.PATH}`;
const builtinCodex = path.join(root, "test");
fs.copyFileSync(fakeCodex, builtinCodex);
const literalCodex = path.join(root, String.raw`codex\probe`);
fs.copyFileSync(fakeCodex, literalCodex);
const targetRoot = path.join(root, "path-target");
fs.mkdirSync(path.join(targetRoot, "child"), { recursive: true });
const directoryAlias = path.join(root, "path-alias");
fs.symlinkSync(path.join(targetRoot, "child"), directoryAlias);
const targetCodex = path.join(targetRoot, "codex");
const shellQuote = (value: string) =>
"'" + value.replaceAll("'", "'\\''") + "'";
fs.writeFileSync(
targetCodex,
`#!/bin/sh\nCODEX_TEST_EXECUTABLE=${shellQuote(targetCodex)} exec ${shellQuote(process.execPath)} ${shellQuote(fakeCodex)} "$@"\n`,
{ mode: 0o755 },
);
const relativePath = path.relative(process.cwd(), root);
for (const [command, executable, searchPath] of [
[fakeCodex, fakeCodex, selectedPath],
[path.relative(process.cwd(), fakeCodex), fakeCodex, selectedPath],
[path.basename(fakeCodex), fakeCodex, selectedPath],
[alias, fakeCodex, selectedPath],
["test", builtinCodex, selectedPath],
[path.basename(literalCodex), literalCodex, selectedPath],
["codex", targetCodex, `${directoryAlias}/..`],
[
"codex",
targetCodex,
`${path.relative(process.cwd(), directoryAlias)}/..`,
],
[`${directoryAlias}/../codex`, targetCodex, selectedPath],
["test", builtinCodex, relativePath],
["test", builtinCodex, `${path.delimiter}${relativePath}`],
] as const) {
const provider = await load(nodes[0], command, { PATH: searchPath });
try {
const result = await provider.callApi("synthetic");
assert.equal(result.error, undefined);
const captured = JSON.parse(String(result.output));
assert.equal(captured.executable, executable);
assert.equal(captured.path, searchPath);
assert.deepEqual(captured.directories, [
path.dirname(nodes[0]),
path.dirname(nodes[0]),
path.dirname(executable),
]);
} finally {
await provider.cleanup?.();
}
}
const otherCodex = path.join(root, "other", "codex");
fs.mkdirSync(path.dirname(otherCodex));
fs.copyFileSync(fakeCodex, otherCodex);
const selectedCodex = await load(nodes[0], "{{selected_codex}}");
try {
await Promise.all(
[fakeCodex, otherCodex, fakeCodex].map(async (command) => {
const context = {
vars: { selected_codex: command },
prompt: { raw: "synthetic", label: "synthetic" },
};
const result = await selectedCodex.callApi("synthetic", context);
assert.equal(result.error, undefined);
const captured = JSON.parse(String(result.output));
assert.equal(captured.executable, command);
assert.deepEqual(captured.directories, [
path.dirname(nodes[0]),
path.dirname(nodes[0]),
path.dirname(command),
]);
assert.deepEqual(context, {
vars: { selected_codex: command },
prompt: { raw: "synthetic", label: "synthetic" },
});
}),
);
} finally {
await selectedCodex.cleanup?.();
}
const packages = ["first", "second", "vendored"].map((name) => {
const modules = path.join(root, name, "node_modules", "@openai");
const packageRoot = path.join(modules, "codex");
const nativePackage =
name === "vendored"
? packageRoot
: path.join(modules, `codex-${process.platform}-${process.arch}`);
const nativeRoot = path.join(nativePackage, "vendor", "synthetic-target");
const native = path.join(nativeRoot, "bin", "codex");
const tools = path.join(nativeRoot, "codex-path");
const launcher = path.join(packageRoot, "bin", "codex.js");
fs.mkdirSync(path.dirname(launcher), { recursive: true });
fs.mkdirSync(path.dirname(native), { recursive: true });
fs.mkdirSync(tools);
fs.writeFileSync(
path.join(packageRoot, "package.json"),
JSON.stringify({
name: "@openai/codex",
bin: { codex: "bin/codex.js" },
}),
);
if (name !== "vendored")
fs.writeFileSync(path.join(nativePackage, "package.json"), "{}");
fs.copyFileSync(fakeCodex, native);
fs.writeFileSync(
launcher,
`#!${process.execPath}
const child = require('node:child_process').spawnSync(${JSON.stringify(native)}, process.argv.slice(2), {stdio:'inherit',env:process.env});
process.exit(child.status ?? 1);
`,
{ mode: 0o755 },
);
return {
launcher,
native,
tools,
configured: name === "vendored" ? [path.dirname(native), tools] : [],
};
});
const packaged = await load(nodes[0], "{{selected_codex}}");
try {
await Promise.all(
packages.map(async ({ launcher, native, tools, configured }) => {
const result = await packaged.callApi("synthetic", {
vars: { selected_codex: launcher },
prompt: {
raw: "synthetic",
label: "synthetic",
config: { additional_directories: configured },
},
});
assert.equal(result.error, undefined);
const captured = JSON.parse(String(result.output));
assert.equal(captured.executable, native);
assert.equal(captured.path, process.env.PATH);
assert.deepEqual(captured.directories, [
...configured,
path.dirname(nodes[0]),
path.dirname(nodes[0]),
path.dirname(launcher),
...(configured.length ? [] : [path.dirname(native), tools]),
]);
}),
);
} finally {
await packaged.cleanup?.();
}
const mixed = await load("{{prefix}}/{{env.NODE_BASENAME}}");
try {
const result = await mixed.callApi("synthetic", {
vars: { prefix: path.dirname(nodes[1]) },
prompt: { raw: "synthetic", label: "synthetic" },
});
assert.equal(JSON.parse(String(result.output)).node, nodes[1]);
} finally {
await mixed.cleanup?.();
}
for (const [ambient, template, value, expected] of [
[undefined, undefined, "", process.execPath],
["", undefined, "", process.execPath],
[nodes[0], "", "", process.execPath],
[nodes[0], "{{custom_node}}", "", process.execPath],
[undefined, "{{missing_node}}", "", process.execPath],
[nodes[0], undefined, "", nodes[0]],
[nodes[0], "{{custom_node}}", nodes[1]!, nodes[1]],
] as const) {
if (ambient === undefined) delete process.env.CODEX_MCP_NODE_PATH;
else process.env.CODEX_MCP_NODE_PATH = ambient;
const provider = await load(template);
try {
const result = await provider.callApi("synthetic", {
vars: { custom_node: value },
prompt: { raw: "synthetic", label: "synthetic" },
});
assert.equal(result.error, undefined);
const captured = JSON.parse(String(result.output));
const selected = fs.realpathSync(expected!);
assert.equal(captured.node, selected);
assert.deepEqual(captured.directories, [
path.dirname(selected),
path.dirname(captured.node),
path.dirname(fakeCodex),
]);
} finally {
await provider.cleanup?.();
}
}
const withoutContext = await load(nodes[0]);
try {
const result = await withoutContext.callApi("synthetic");
const captured = JSON.parse(String(result.output));
assert.equal(captured.node, nodes[0]);
assert.equal(captured.marker, "{{marker}}");
} finally {
await withoutContext.cleanup?.();
}
process.env.PROMPTFOO_DISABLE_TEMPLATING = "true";
const disabled = await load(nodes[2]!);
try {
const result = await disabled.callApi("synthetic", {
vars: { custom_node: "other" },
prompt: { raw: "synthetic", label: "synthetic" },
});
assert.equal(JSON.parse(String(result.output)).node, nodes[2]);
} finally {
await disabled.cleanup?.();
}
},
);
test(
"saved native-provider configs retain legacy paths through retry, resume, and viewer replay",
{ skip: process.platform === "win32", timeout: 120000 },
async (t) => {
const root = temporaryRoot(t, "legacy-provider-");
// Keep the saved source layout independent of the checkout and its path.
const legacySourceRoot = path.join(root, "source.checkout");
const legacyEvalRoot = path.join(
legacySourceRoot,
"evals",
"triage-finding",
);
const legacyTransform = path.join(
legacyEvalRoot,
"scripts",
"runtime-vars.mts",
);
fs.mkdirSync(path.dirname(legacyTransform), { recursive: true });
fs.copyFileSync(
path.join(import.meta.dirname, "runtime-vars.mts"),
legacyTransform,
);
const skill = path.join(
legacySourceRoot,
"plugins",
"codex-security",
"skills",
"triage-finding",
"SKILL.md",
);
fs.mkdirSync(path.dirname(skill), { recursive: true });
fs.writeFileSync(skill, "Synthetic legacy skill fixture.\n");
const fixture = path.join(
legacyEvalRoot,
"fixtures",
"repo",
"src",
"server.js",
);
fs.mkdirSync(path.dirname(fixture), { recursive: true });
fs.writeFileSync(fixture, "// Synthetic repository fixture.\n");
const capture = path.join(root, "captures.jsonl");
const failure = path.join(root, "failure");
const fakeCodex = path.join(root, "codex");
fs.writeFileSync(
fakeCodex,
`#!${process.execPath}
const fs = require('node:fs');
const path = require('node:path');
const cwd = process.argv[process.argv.indexOf('--cd') + 1];
const prompt = fs.readFileSync(0, 'utf8');
const target = prompt.match(/^TARGET=(.+)$/m)[1];
const skill = fs.readFileSync(path.join(cwd, 'skills/triage-finding/SKILL.md'), 'utf8');
fs.readFileSync(path.join(target, 'src/server.js'), 'utf8');
fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,target,skillLoaded:skill.length>0,nodePath:process.env.CODEX_MCP_NODE_PATH,directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])})+'\\n');
console.log(JSON.stringify({type:'thread.started',thread_id:'synthetic-legacy'}));
if (fs.existsSync(${JSON.stringify(failure)})) {
console.log(JSON.stringify({type:'turn.failed',error:{message:'synthetic legacy retryable failure'}}));
} else {
console.log(JSON.stringify({type:'item.completed',item:{id:'message',type:'agent_message',text:'ok'}}));
console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_tokens:1}}));
}
`,
{ mode: 0o755 },
);
const configPath = path.join(root, "config.json");
// These are the native-provider placeholders saved by the previous configs.
// No startup extension is available when Promptfoo reloads those records.
const provider = {
id: "openai:codex-sdk:gpt-5.5",
config: {
working_dir: "{{triage_runtime_root}}",
codex_path_override: fakeCodex,
skip_git_repo_check: true,
approval_policy: "never",
additional_directories: [
"{{triage_fixture_root}}",
"{{triage_node_root}}",
],
cli_env: { CODEX_MCP_NODE_PATH: "{{triage_node_path}}" },
},
};
fs.writeFileSync(
configPath,
JSON.stringify({
providers: [provider],
prompts: [
"Read and follow skills/triage-finding/SKILL.md.\nTARGET={{target_repo}}",
],
defaultTest: {
options: {
transformVars: `file://${legacyTransform}`,
},
},
tests: [
{
vars: { target_repo: "evals/triage-finding/fixtures/repo" },
assert: [{ type: "equals", value: "ok" }],
},
],
}),
);
const environment = promptfooEnvironment(root);
fs.writeFileSync(failure, "");
const initial = await invoke(evalArgs("-c", configPath), environment);
assert.notEqual(initial.code, 0, initial.output);
assert.match(initial.output, /synthetic legacy retryable failure/);
fs.rmSync(failure);
const retry = await invoke(evalArgs("--retry-errors"), environment);
assert.equal(retry.code, 0, retry.output);
const database = openDatabase(environment);
const stored = database
.prepare("SELECT id, config FROM evals ORDER BY created_at DESC LIMIT 1")
.get();
assert.ok(stored && typeof stored.config === "string");
const saved = JSON.parse(stored.config);
assert.equal(saved.providers[0].id, provider.id);
assert.equal(
saved.providers[0].config.working_dir,
"{{triage_runtime_root}}",
);
assert.equal(saved.extensions?.length || 0, 0);
database.exec("DELETE FROM eval_results");
database.close();
const resumed = await invoke(evalArgs("--resume"), environment);
assert.equal(resumed.code, 0, resumed.output);
const resumedResult = storedReplayResult(environment, stored.id);
const replayVariables =
JSON.parse((resumedResult.metadata as string) || "{}").inputVars ||
JSON.parse(resumedResult.testCase).vars;
await replayInViewer(t, environment, {
evaluationId: stored.id,
prompt:
"Read and follow skills/triage-finding/SKILL.md.\nTARGET=" +
replayVariables.target_repo,
variables: replayVariables,
});
const rows = fs
.readFileSync(capture, "utf8")
.trim()
.split("\n")
.map((line) => JSON.parse(line));
assert.equal(rows.length, 4);
assert.equal(new Set(rows.map((row) => row.cwd)).size, 1);
const nodePath = fs.realpathSync(process.execPath);
for (const row of rows) {
const runtimeRoot = legacySourceRoot;
const fixtureRoot = path.join(
runtimeRoot,
"evals",
"triage-finding",
"fixtures",
);
assert.equal(
row.cwd,
path.join(runtimeRoot, "plugins", "codex-security"),
);
assert.equal(row.target, path.join(fixtureRoot, "repo"));
assert.equal(row.nodePath, nodePath);
assert.deepEqual(row.directories, [fixtureRoot, path.dirname(nodePath)]);
assert.equal(row.skillLoaded, true);
assert.equal(fs.existsSync(row.cwd), true);
}
},
);
for (const [suite, legacy] of [
["SAST", false],
["calibration", false],
["SAST", true],
] as const) {
test(
`${suite}${legacy ? " legacy hook" : ""} provider survives persisted retry, resume, and viewer replay`,
{ skip: process.platform === "win32", timeout: 120000 },
async (t) => {
const root = temporaryRoot(t, "sast-provider-replay-");
const targetRoot = temporaryRoot(t, "sast-custom-target-");
const targetRepository = path.join(targetRoot, "selected-case");
const siblingRepository = path.join(targetRoot, "other-case");
fs.mkdirSync(targetRepository);
fs.mkdirSync(siblingRepository);
const capture = path.join(root, "captures.jsonl");
const failure = path.join(root, "failure");
const fakeCodex = path.join(root, "codex");
fs.writeFileSync(
fakeCodex,
`#!${process.execPath}
const fs = require('node:fs');
const cwd = process.argv[process.argv.indexOf('--cd') + 1];
const model = process.argv[process.argv.indexOf('--model') + 1];
const overrides = process.argv.flatMap((arg, index) => arg === '--config' ? [process.argv[index + 1]] : []);
const effort = overrides.find(value => value.startsWith('model_reasoning_effort='));
fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({ cwd, model, effort, marker: process.env.SAVED_PROVIDER_MARKER, directories: process.argv.flatMap((arg, index) => arg === '--add-dir' ? [process.argv[index + 1]] : []) }) + '\\n');
console.log(JSON.stringify({ type: 'thread.started', thread_id: 'synthetic-sast-replay' }));
if (fs.existsSync(${JSON.stringify(failure)})) {
console.log(JSON.stringify({ type: 'turn.failed', error: { message: 'synthetic retryable failure' } }));
} else {
console.log(JSON.stringify({ type: 'item.completed', item: { id: 'message', type: 'agent_message', text: 'ok' } }));
console.log(JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } }));
}
`,
{ mode: 0o755 },
);
const environment = {
...promptfooEnvironment(root),
CODEX_MCP_NODE_PATH: process.execPath,
};
const template = parse(
fs.readFileSync(
path.join(
evalRoot,
suite === "SAST"
? "promptfooconfig.sastbench.yaml"
: "promptfooconfig.calibration.yaml",
),
"utf8",
),
);
const provider = providerConfig(
suite === "SAST" ? "sastbench" : "calibration",
);
provider.config.codex_path_override = fakeCodex;
provider.config.model = "gpt-5.5";
provider.config.model_reasoning_effort = "high";
provider.config.cli_env = {
...provider.config.cli_env,
SAVED_PROVIDER_MARKER: "original-provider",
};
const providerFile = path.join(root, "provider.json");
fs.writeFileSync(providerFile, JSON.stringify(provider));
// Resolve suite hooks relative to their original configuration directory.
const configPath = path.join(
evalRoot,
suite === "SAST" ? "sastbench" : ".",
`.provider-replay-${process.pid}.json`,
);
t.after(() => fs.rmSync(configPath, { force: true }));
fs.writeFileSync(
configPath,
JSON.stringify({
extensions: legacy
? [...extensions, ...(template.extensions ?? [])]
: template.extensions,
providers: [legacy ? provider : { $ref: providerFile }],
prompts: ["hello"],
defaultTest:
suite === "calibration"
? {
options: {
transformVars: `file://${path.join(import.meta.dirname, "runtime-vars.mts")}`,
},
}
: undefined,
tests: [
{
metadata:
suite === "SAST"
? { ground_truth: "true_positive" }
: undefined,
vars:
suite === "SAST"
? { target_repo: targetRepository }
: {
calibration_repo: "calibration-0123456789abcdef",
calibration_repo_root: "",
},
assert: [{ type: "equals", value: "ok" }],
},
],
}),
);
fs.writeFileSync(failure, "");
const initial = await invoke(evalArgs("-c", configPath), environment);
assert.notEqual(initial.code, 0, initial.output);
assert.match(initial.output, /synthetic retryable failure/);
const initialDatabase = openDatabase(environment);
const initialEvaluation = initialDatabase
.prepare("SELECT config FROM evals ORDER BY created_at DESC LIMIT 1")
.get();
assert.ok(
initialEvaluation && typeof initialEvaluation.config === "string",
);
const savedProvider = JSON.parse(initialEvaluation.config).providers[0];
assert.equal(savedProvider.id, provider.id);
assert.equal(savedProvider.$ref, undefined);
assert.equal(savedProvider.config.model, "gpt-5.5");
assert.equal(savedProvider.config.model_reasoning_effort, "high");
assert.equal(
savedProvider.config.cli_env.SAVED_PROVIDER_MARKER,
"original-provider",
);
if (suite === "SAST") {
const initialResult = initialDatabase
.prepare("SELECT metadata FROM eval_results LIMIT 1")
.get();
assert.ok(
JSON.parse((initialResult?.metadata as string) || "{}").sastbench,
initial.output,
);
}
initialDatabase.close();
if (!legacy) {
fs.writeFileSync(
providerFile,
JSON.stringify({
...provider,
config: {
...provider.config,
model: "gpt-5.4",
model_reasoning_effort: "minimal",
cli_env: {
...provider.config.cli_env,
SAVED_PROVIDER_MARKER: "changed-provider",
},
},
}),
);
}
fs.rmSync(failure);
const retry = await invoke(evalArgs("--retry-errors"), environment);
assert.equal(retry.code, 0, retry.output);
const database = openDatabase(environment);
const stored = database
.prepare(
"SELECT id, config FROM evals ORDER BY created_at DESC LIMIT 1",
)
.get();
assert.ok(stored && typeof stored.config === "string");
assert.deepEqual(JSON.parse(stored.config).providers[0], savedProvider);
assert.deepEqual(
JSON.parse(stored.config).extensions,
(legacy
? [...extensions, ...(template.extensions ?? [])]
: (template.extensions ?? [])
).map((extension: string) =>
extension.replace(
"{{env.TRIAGE_PROVIDER_PATH}}",
path.join(import.meta.dirname, "triage-provider.mts"),
),
),
);
database.prepare("DELETE FROM eval_results").run();
database.close();
const resumed = await invoke(evalArgs("--resume"), environment);
assert.equal(resumed.code, 0, resumed.output);
const resumedResult = storedReplayResult(environment, stored.id);
// Match the viewer: prefer saved inputVars, then the resolved test-case vars.
const replayVariables =
JSON.parse((resumedResult.metadata as string | null) ?? "{}")
.inputVars || JSON.parse(resumedResult.testCase).vars;
assert.equal(typeof replayVariables.target_repo, "string");
const viewerOutput = await replayInViewer(t, environment, {
evaluationId: stored.id,
prompt: "hello",
variables: replayVariables,
});
assert.doesNotMatch(viewerOutput, /afterEach extension hook failed/);
const rows = fs
.readFileSync(capture, "utf8")
.trim()
.split("\n")
.map((line) => JSON.parse(line));
assert.equal(rows.length, 4);
assert.equal(new Set(rows.map((row) => row.cwd)).size, 4);
for (const row of rows) {
assert.equal(fs.existsSync(row.cwd), false);
assert.equal(row.model, "gpt-5.5");
assert.equal(row.effort, 'model_reasoning_effort="high"');
assert.equal(row.marker, "original-provider");
assert.deepEqual(row.directories, [
...(suite === "SAST"
? [
path.join(evalRoot, "artifacts", "sastbench-git-cache"),
targetRepository,
]
: [
path.join(
evalRoot,
"artifacts",
"calibration-repos",
"calibration-0123456789abcdef",
),
]),
path.dirname(fs.realpathSync(process.execPath)),
path.dirname(process.execPath),
path.dirname(fakeCodex),
]);
assert.ok(
row.directories.every(
(directory: string) =>
siblingRepository !== directory &&
!siblingRepository.startsWith(directory + path.sep),
),
);
}
},
);
}