Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(tooling): reduce repeated Action and eval setup (#2040)

Michael D'Angelob3e9d40f5a3d
diff --git a/.github/scripts/invoice-desk-target.test.mjs b/.github/scripts/invoice-desk-target.test.mjs
index 7a4be077a..8cce9de3d 100644
--- a/.github/scripts/invoice-desk-target.test.mjs
+++ b/.github/scripts/invoice-desk-target.test.mjs
@@ -27,55 +27,61 @@ function pullRequest(overrides = {}) {
};
}
-test("selects the requested PR head rather than the trusted workflow commit", async () => {
- assert.deepEqual(
- await resolveScanTargets(context, async (number) => {
- assert.equal(number, "7");
- return pullRequest();
- }),
- [{ pr: 7, sha: sourceSha }],
- );
-});
-
-test("includes fork and draft PRs without depending on a PR workflow run", async () => {
- assert.deepEqual(
- await resolveScanTargets(context, async () =>
- pullRequest({
- draft: true,
- head: { sha: sourceSha, repo: { full_name: "contributor/invoices" } },
- }),
- ),
- [{ pr: 7, sha: sourceSha }],
- );
+test("initial selection and approval recheck require the selected PR and revision to remain eligible", async () => {
+ const target = { pr: 7, sha: sourceSha };
+ // The approval step must use its matrix target even if dispatch inputs differ.
+ const recheck = {
+ ...context,
+ target: { ...target },
+ event: { inputs: { pr_number: "9", source_sha: mainSha } },
+ };
+ for (const request of [context, recheck]) {
+ for (const [overrides, expected] of [
+ [{}, [target]],
+ [
+ {
+ draft: true,
+ head: { sha: sourceSha, repo: { full_name: "contributor/invoices" } },
+ },
+ [target],
+ ],
+ [{ state: "closed" }, []],
+ [{ base: { ref: "release", repo: { full_name: repository } } }, []],
+ [{ base: { ref: "main", repo: { full_name: "other/invoices" } } }, []],
+ [{ head: { sha: mainSha } }, []],
+ [{ head: { sha: "3".repeat(40) } }, []],
+ ]) {
+ const actual = await resolveScanTargets(request, async (number) => {
+ assert.equal(number, "7");
+ return pullRequest(overrides);
+ });
+ assert.deepEqual(actual, expected);
+ }
+ }
});
-test("rechecking after approval excludes closed, retargeted, and superseded PRs", async () => {
- assert.deepEqual(
- await resolveScanTargets(context, async () => pullRequest()),
- [{ pr: 7, sha: sourceSha }],
- );
- for (const changed of [
- { state: "closed" },
- { base: { ref: "release", repo: { full_name: repository } } },
- { base: { ref: "main", repo: { full_name: "other/invoices" } } },
- { head: { sha: "3".repeat(40) } },
+test("manual and scheduled baselines use the trusted workflow commit before and after approval", async () => {
+ const target = { pr: 0, sha: mainSha };
+ const scheduled = {
+ ...context,
+ eventName: "schedule",
+ event: { schedule: "17 3 * * 1" },
+ };
+ for (const request of [
+ { ...context, event: { inputs: {} } },
+ scheduled,
+ { ...context, target: { ...target } },
+ { ...scheduled, target: { ...target } },
]) {
assert.deepEqual(
- await resolveScanTargets(context, async () => pullRequest(changed)),
- [],
+ await resolveScanTargets(request, () =>
+ assert.fail("unexpected API call"),
+ ),
+ [target],
);
}
});
-test("manual baseline scans use the trusted workflow commit", async () => {
- assert.deepEqual(
- await resolveScanTargets({ ...context, event: { inputs: {} } }, () =>
- assert.fail("unexpected API call"),
- ),
- [{ pr: 0, sha: mainSha }],
- );
-});
-
test("incomplete or invalid PR inputs fail instead of falling back to a baseline", async () => {
for (const inputs of [
{ pr_number: "7" },
@@ -117,48 +123,6 @@ test("PR events and completed PR workflows do not queue inference or query GitHu
}
});
-test("scheduled baseline scans use the trusted workflow commit", async () => {
- assert.deepEqual(
- await resolveScanTargets(
- { ...context, eventName: "schedule", event: { schedule: "17 3 * * 1" } },
- () => assert.fail("unexpected API call"),
- ),
- [{ pr: 0, sha: mainSha }],
- );
-});
-
-test("approval recheck stays bound to its matrix PR and source revision", async () => {
- const run = { ...context, target: { pr: 7, sha: sourceSha } };
- for (const changed of [
- { state: "closed" },
- { base: { ref: "release", repo: { full_name: repository } } },
- { head: { sha: mainSha } },
- ]) {
- assert.deepEqual(
- await resolveScanTargets(run, async (number) => {
- assert.equal(number, "7");
- return pullRequest(changed);
- }),
- [],
- );
- }
- assert.deepEqual(await resolveScanTargets(run, async () => pullRequest()), [
- { pr: 7, sha: sourceSha },
- ]);
-});
-
-test("baseline approval recheck retains the original trusted workflow commit", async () => {
- for (const eventName of ["workflow_dispatch", "schedule"]) {
- assert.deepEqual(
- await resolveScanTargets(
- { ...context, eventName, target: { pr: 0, sha: mainSha } },
- () => assert.fail("unexpected API call"),
- ),
- [{ pr: 0, sha: mainSha }],
- );
- }
-});
-
test("resolver CLI emits the requested target without dispatching another workflow", (t) => {
const root = mkdtempSync(join(tmpdir(), "invoice-desk-target-"));
t.after(() => rmSync(root, { recursive: true, force: true }));
diff --git a/.github/scripts/test_check_plugin_source_compatibility.mts b/.github/scripts/test_check_plugin_source_compatibility.mts
index a99528dd3..158398027 100644
--- a/.github/scripts/test_check_plugin_source_compatibility.mts
+++ b/.github/scripts/test_check_plugin_source_compatibility.mts
@@ -92,10 +92,10 @@ test("checkout preserves source size with autocrlf", () => {
git(root, "add", "--", ".gitattributes", name);
unlinkSync(source);
git(root, "checkout-index", "--", name);
- const result = runChecker("--plugin-root", root);
- assert.equal(result.status, 0, result.stderr);
assert.deepEqual(readFileSync(source), content);
}
+ const result = runChecker("--plugin-root", root);
+ assert.equal(result.status, 0, result.stderr);
});
test("accepts Markdown structures, natural line endings, and inline markup", () => {
diff --git a/evals/secret-discovery/test_discovery.mts b/evals/secret-discovery/test_discovery.mts
index 6282e8df1..4f043966e 100644
--- a/evals/secret-discovery/test_discovery.mts
+++ b/evals/secret-discovery/test_discovery.mts
@@ -58,16 +58,18 @@ interface TestResult extends Result {
};
resolved_questions?: TestFinding[];
}
-function retainedResult(fixture: ReturnType<typeof createFixture>): TestResult {
+const fixture = createFixture();
+
+function retainedResult(source = fixture): TestResult {
return {
- findings: fixture.positives.map((expected) => ({
+ findings: source.positives.map((expected) => ({
title: "Embedded credential",
taxonomy: { category: "hardcoded-credentials", cwe: [expected.cwes[0]] },
locations: [
{ path: expected.path, startLine: expected.line, role: "root_control" },
],
codeEvidence: [
- sourceEvidence(fixture, expected.path, expected.line, expected.endLine),
+ sourceEvidence(source, expected.path, expected.line, expected.endLine),
],
})),
coverage: {
@@ -79,14 +81,8 @@ function retainedResult(fixture: ReturnType<typeof createFixture>): TestResult {
};
}
-function retainedFixture() {
- const fixture = createFixture();
- return { fixture, result: retainedResult(fixture) };
-}
-
test("counts every retained secret location, including unused and integration source", () => {
- const fixture = createFixture();
- const report = gradeResult(retainedResult(fixture), fixture);
+ const report = gradeResult(retainedResult(), fixture);
assert.equal(report.passed, true);
assert.equal(report.recall, 1);
assert.deepEqual(
@@ -103,7 +99,7 @@ test("counts every retained secret location, including unused and integration so
});
test("resolved, deferred, and intermediate findings do not satisfy final recall", () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
result.coverage.deferred = result.findings;
result.resolved_questions = result.findings;
result.findings = [];
@@ -145,7 +141,7 @@ for (const [name, mutate] of [
],
] as const) {
test(`rejects ${name}`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
mutate(result.findings[0]);
const report = gradeResult(result, fixture);
assert.equal(report.passed, false);
@@ -173,7 +169,7 @@ for (const category of [
"without-credentials",
]) {
test(`rejects negated category ${category}`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
result.findings[0].taxonomy.category = category;
const report = gradeResult(result, fixture);
assert.equal(report.passed, false);
@@ -182,10 +178,9 @@ for (const category of [
}
test("rejects unrelated categories containing credential-term substrings", () => {
- const fixture = createFixture();
for (const category of ["tokenization-error", "secretary-data"]) {
for (const cwe of ["CWE-798", "CWE-200"]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].taxonomy = { category, cwe: [cwe] };
const report = gradeResult(result, fixture);
assert.equal(report.passed, false, `${category}:${cwe}`);
@@ -195,7 +190,6 @@ test("rejects unrelated categories containing credential-term substrings", () =>
});
test("accepts plural and identifier-style credential categories", () => {
- const fixture = createFixture();
for (const category of [
"tokens-exposed",
"secrets_in_source",
@@ -203,14 +197,14 @@ test("accepts plural and identifier-style credential categories", () => {
"hardcodedCredentials",
"credentialExposure",
]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].taxonomy.category = category;
assert.equal(gradeResult(result, fixture).passed, true, category);
}
});
test("accepts credential exposure with non-secret context in the category", () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
for (const category of [
"credential exposure with non-secret context",
"credential exposure with no-secret context",
@@ -223,7 +217,7 @@ test("accepts credential exposure with non-secret context in the category", () =
});
test("accepts a secret's valid source range and sensitive-data category", () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
result.findings[0].locations[0].endLine = fixture.positives[0].line + 1;
result.findings[0].taxonomy.category = "sensitive-data-exposure";
assert.equal(gradeResult(result, fixture).passed, true);
@@ -235,14 +229,13 @@ for (const category of [
"api-key-exposure",
]) {
test(`accepts credential category ${category}`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
result.findings[0].taxonomy.category = category;
assert.equal(gradeResult(result, fixture).passed, true);
});
}
test("accepts equivalent CWE spelling without weakening generic disclosure categories", () => {
- const fixture = createFixture();
for (const [cwe, generic] of [
["cwe-798", false],
["CWE-0798", false],
@@ -250,7 +243,7 @@ test("accepts equivalent CWE spelling without weakening generic disclosure categ
[" cwe-00200 ", true],
["CWE-0540", true],
] as const) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].taxonomy.cwe = ["CWE-798", cwe];
assert.equal(gradeResult(result, fixture).passed, true, cwe);
if (generic) {
@@ -261,7 +254,6 @@ test("accepts equivalent CWE spelling without weakening generic disclosure categ
});
test("rejects unrelated or malformed CWE identifiers after normalization", () => {
- const fixture = createFixture();
for (const cwe of [
" cwe-0089 ",
"CWE-798-extra",
@@ -270,7 +262,7 @@ test("rejects unrelated or malformed CWE identifiers after normalization", () =>
"CWE-321",
]) {
for (const withValidCwe of [false, true]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].taxonomy.cwe = withValidCwe ? ["CWE-798", cwe] : [cwe];
const report = gradeResult(result, fixture);
assert.equal(report.passed, false, cwe);
@@ -281,12 +273,11 @@ test("rejects unrelated or malformed CWE identifiers after normalization", () =>
});
test("matches path casing only on Windows while preserving citation checks", () => {
- const fixture = createFixture();
const descriptor = Object.getOwnPropertyDescriptor(process, "platform");
try {
for (const platform of ["linux", "win32"]) {
Object.defineProperty(process, "platform", { value: platform });
- const result = retainedResult(fixture);
+ const result = retainedResult();
const finding = result.findings[0];
finding.locations.push(
{ path: "src/client.py", startLine: 7, role: "sink" },
@@ -315,7 +306,7 @@ test("matches path casing only on Windows while preserving citation checks", ()
for (const cwe of ["CWE-256", "CWE-259", "CWE-260"]) {
test(`accepts ${cwe} for the password in the database configuration`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
const databaseIndex = fixture.positives.findIndex(
(entry) => entry.id === "dotenv-url",
);
@@ -328,7 +319,7 @@ for (const cwe of ["CWE-256", "CWE-259", "CWE-260"]) {
for (const cwe of ["CWE-312", "CWE-313"]) {
test(`accepts ${cwe} for credentials stored in source and configuration files`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
for (const finding of result.findings) finding.taxonomy.cwe = [cwe];
assert.equal(gradeResult(result, fixture).passed, true);
@@ -340,7 +331,6 @@ for (const cwe of ["CWE-312", "CWE-313"]) {
}
test("keeps password and cryptographic-key CWEs specific to those exposures", () => {
- const fixture = createFixture();
for (const [index, expected] of fixture.positives.entries()) {
for (const [cwe, applicableId] of [
["CWE-256", "dotenv-url"],
@@ -349,7 +339,7 @@ test("keeps password and cryptographic-key CWEs specific to those exposures", ()
["CWE-321", "private-key"],
] as const) {
if (expected.id === applicableId) continue;
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[index].taxonomy.cwe = [cwe];
const report = gradeResult(result, fixture);
assert.equal(report.passed, false, `${expected.id}:${cwe}`);
@@ -360,7 +350,7 @@ test("keeps password and cryptographic-key CWEs specific to those exposures", ()
for (const cwe of ["CWE-200", "CWE-540"]) {
test(`accepts ${cwe} for specifically categorized credential exposure`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
for (const finding of result.findings) finding.taxonomy.cwe = [cwe];
assert.equal(gradeResult(result, fixture).passed, true);
@@ -373,7 +363,7 @@ for (const cwe of ["CWE-200", "CWE-540"]) {
for (const negativeIndex of [0, 1, 2, 3]) {
test(`rejects a finding against negative control ${negativeIndex}`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
result.findings.push({
taxonomy: { category: "hardcoded-credentials", cwe: ["CWE-798"] },
locations: [
@@ -399,13 +389,12 @@ for (const role of [
"expected_control",
]) {
test(`benign ${role} context does not become a false positive`, () => {
- const fixture = createFixture();
for (const path of [
fixture.negatives[0],
`./${fixture.negatives[0]}`,
`.\\${fixture.negatives[0].replaceAll("/", "\\")}`,
]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].locations.push({
path,
startLine: 1,
@@ -419,7 +408,6 @@ for (const role of [
});
test(`rejects invalid ${role} source citations`, () => {
- const fixture = createFixture();
for (const location of [
{ path: "src/absent.py", startLine: 999 },
{ path: "src/runtime_config.py", startLine: 0 },
@@ -428,7 +416,7 @@ for (const role of [
{ path: "src/runtime_config.py", startLine: 3, endLine: 1 },
{ path: "src/runtime_config.py", startLine: 1, endLine: 999 },
]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].locations.push({ ...location, role });
const report = gradeResult(result, fixture);
assert.equal(report.recall, 1);
@@ -445,7 +433,7 @@ for (const [name, location] of [
["negative control", { path: "src/runtime_config.py", startLine: 1 }],
] as const) {
test(`rejects an additional non-supporting location at ${name}`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
result.findings[0].locations.push({ ...location, role: "root_control" });
const report = gradeResult(result, fixture);
assert.equal(report.recall, 1);
@@ -456,7 +444,7 @@ for (const [name, location] of [
for (const role of ["root_control", "sink"]) {
test(`rejects a ${role} citation classified only by a different finding`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
const finding = result.findings[0];
finding.taxonomy.cwe = ["CWE-522"];
const key = fixture.positives.find(
@@ -481,7 +469,6 @@ for (const role of ["root_control", "sink"]) {
}
test("correct finding locations cannot hide missing or unrelated code evidence", () => {
- const fixture = createFixture();
for (const evidence of [
undefined,
[],
@@ -489,7 +476,7 @@ test("correct finding locations cannot hide missing or unrelated code evidence",
[sourceEvidence(fixture, "src/runtime_config.py", 3)],
[{ ...sourceEvidence(fixture, "src/client.py", 3), code: " \n" }],
]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].codeEvidence = evidence as CodeEvidence[];
const report = gradeResult(result, fixture);
assert.equal(report.recall, 1);
@@ -499,7 +486,6 @@ test("correct finding locations cannot hide missing or unrelated code evidence",
});
test("rejects fabricated source text at otherwise valid evidence locations", () => {
- const fixture = createFixture();
for (const code of [
"invented source",
sourceEvidence(fixture, "src/client.py", 1).code,
@@ -508,7 +494,7 @@ test("rejects fabricated source text at otherwise valid evidence locations", ()
"OTHER_TOKEN",
),
]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].codeEvidence[0].code = code;
const report = gradeResult(result, fixture);
assert.equal(report.recall, 1);
@@ -516,7 +502,7 @@ test("rejects fabricated source text at otherwise valid evidence locations", ()
assert.deepEqual(report.errors, ["missing or invalid code evidence"]);
}
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].codeEvidence.push({
...sourceEvidence(fixture, "README.md", 1),
code: "invented supporting context",
@@ -527,14 +513,13 @@ test("rejects fabricated source text at otherwise valid evidence locations", ()
});
test("rejects invented and out-of-bounds citations alongside valid evidence", () => {
- const fixture = createFixture();
for (const citation of [
{ path: "src/absent.py", startLine: 1 },
{ path: "src/client.py", startLine: 0 },
{ path: "src/client.py", startLine: 999 },
{ path: "src/client.py", startLine: 7, code: "first line\nsecond line" },
]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].codeEvidence.push({
...sourceEvidence(fixture, "src/client.py", 3),
...citation,
@@ -546,12 +531,11 @@ test("rejects invented and out-of-bounds citations alongside valid evidence", ()
});
test("accepts multiline exposure and consumer excerpts with supporting context", () => {
- const fixture = createFixture();
for (const [startLine, endLine] of [
[1, 3],
[4, 7],
] as const) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].codeEvidence = [
sourceEvidence(fixture, "src/client.py", startLine, endLine),
sourceEvidence(fixture, "src/runtime_config.py", 3),
@@ -564,7 +548,6 @@ test("accepts multiline exposure and consumer excerpts with supporting context",
});
test("consumer evidence and sinks must cover credential use, not function framing", () => {
- const fixture = createFixture();
for (const [id, line] of [
["active-source", 5],
["active-source", 6],
@@ -575,7 +558,7 @@ test("consumer evidence and sinks must cover credential use, not function framin
] as const) {
const index = fixture.positives.findIndex((expected) => expected.id === id);
const expected = fixture.positives[index];
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[index].codeEvidence = [
sourceEvidence(fixture, expected.path, line),
];
@@ -585,7 +568,7 @@ test("consumer evidence and sinks must cover credential use, not function framin
`${id}:${line}`,
);
- const invalidSink = retainedResult(fixture);
+ const invalidSink = retainedResult();
invalidSink.findings[index].locations.push({
path: expected.path,
startLine: line,
@@ -600,14 +583,13 @@ test("consumer evidence and sinks must cover credential use, not function framin
});
test("accepts actual credential-use lines and excerpts containing them", () => {
- const fixture = createFixture();
for (const [index, expected] of fixture.positives.entries()) {
if (expected.consumerLine === null) continue;
for (const [startLine, endLine] of [
[expected.consumerLine, expected.consumerLine],
[5, expected.lineCount],
] as const) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[index].codeEvidence = [
sourceEvidence(fixture, expected.path, startLine, endLine),
];
@@ -623,7 +605,7 @@ test("accepts actual credential-use lines and excerpts containing them", () => {
});
test("multiple expected exposures can share a finding", () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
const unused = result.findings.splice(1, 1)[0];
result.findings[0].locations.push(...unused.locations);
assert.deepEqual(gradeResult(result, fixture).errors, [
@@ -634,7 +616,7 @@ test("multiple expected exposures can share a finding", () => {
});
test("grouped exposures retain their applicable CWE classifications", () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
const indices = ["dotenv-url", "private-key"].map((id) =>
fixture.positives.findIndex((expected) => expected.id === id),
);
@@ -663,7 +645,7 @@ test("grouped exposures retain their applicable CWE classifications", () => {
for (const role of ["supporting", "expected_control"]) {
test(`${role} citations cannot satisfy an independently missed exposure`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
const unused = result.findings.splice(1, 1)[0];
result.findings[0].locations.push({
...unused.locations[0],
@@ -676,7 +658,7 @@ for (const role of ["supporting", "expected_control"]) {
}
test("accepts real credential-use sinks without treating them as separate exposures", () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
for (const [index, expected] of fixture.positives.entries()) {
if (expected.consumerLine === null) continue;
const sink = {
@@ -686,7 +668,7 @@ test("accepts real credential-use sinks without treating them as separate exposu
role: "sink",
};
result.findings[index].locations.push(sink);
- const onlyConsumer = retainedResult(fixture);
+ const onlyConsumer = retainedResult();
onlyConsumer.findings[index].locations = [sink];
assert.equal(gradeResult(onlyConsumer, fixture).cases[index].found, false);
}
@@ -697,7 +679,7 @@ test("accepts real credential-use sinks without treating them as separate exposu
{ path: "src/absent.py", startLine: 5 },
{ path: "src/client.py", startLine: 1 },
]) {
- const invalid = retainedResult(fixture);
+ const invalid = retainedResult();
invalid.findings[0].locations.push({ ...location, role: "sink" });
assert.equal(gradeResult(invalid, fixture).falsePositiveCount, 1);
}
@@ -708,9 +690,8 @@ for (const [name, separator] of [
["Windows", "\\"],
] as const) {
test(`accepts ${name} source and credential-use paths with relative prefixes`, () => {
- const fixture = createFixture();
for (const prefix of ["", "./", "././", ".\\", ".\\./", "./.\\"]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
for (const [index, expected] of fixture.positives.entries()) {
const path = `${prefix}${expected.path.replaceAll("/", separator)}`;
result.findings[index].locations[0].path = path;
@@ -730,7 +711,6 @@ for (const [name, separator] of [
}
test("path normalization does not make unrelated or unsafe source locations valid", () => {
- const fixture = createFixture();
for (const path of [
"./src/absent.py",
"./src/../src/client.py",
@@ -740,7 +720,7 @@ test("path normalization does not make unrelated or unsafe source locations vali
"\\src\\client.py",
"C:\\src\\client.py",
]) {
- const result = retainedResult(fixture);
+ const result = retainedResult();
result.findings[0].locations[0].path = path;
const report = gradeResult(result, fixture);
assert.equal(report.cases[0].found, false, path);
@@ -750,7 +730,7 @@ test("path normalization does not make unrelated or unsafe source locations vali
});
test("accepts the private-key body and rejects reversed source ranges", () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
const keyIndex = fixture.positives.findIndex(
(entry) => entry.id === "private-key",
);
@@ -763,7 +743,7 @@ test("accepts the private-key body and rejects reversed source ranges", () => {
for (const footerLocation of [true, false]) {
test(`rejects footer-only private-key evidence with ${footerLocation ? "footer" : "declaration"} location`, () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
const keyIndex = fixture.positives.findIndex(
(entry) => entry.id === "private-key",
);
@@ -784,7 +764,7 @@ for (const footerLocation of [true, false]) {
}
test("accepts public-key context alongside credential evidence", () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
result.findings[0].codeEvidence.push(
sourceEvidence(fixture, "src/signing.mjs", 2, 4),
sourceEvidence(fixture, "config/public.pem", 1, 3),
@@ -793,7 +773,7 @@ test("accepts public-key context alongside credential evidence", () => {
});
test("scope notes neither fail correct findings nor compensate for missing findings", () => {
- const { fixture, result } = retainedFixture();
+ const result = retainedResult();
Object.assign(result.coverage, {
completeness: "partial",
explicitExclusions: [
diff --git a/evals/secret-discovery/test_runtime.mts b/evals/secret-discovery/test_runtime.mts
index 302e3e715..bfb178497 100644
--- a/evals/secret-discovery/test_runtime.mts
+++ b/evals/secret-discovery/test_runtime.mts
@@ -289,11 +289,9 @@ async function nativeFixture(
await mkdir(dirname(executable), { recursive: true });
const tools = join(dirname(dirname(executable)), "codex-path");
await mkdir(tools);
- await writeFile(
- join(tools, "rg"),
- `#!${process.execPath}\nconsole.log("synthetic rg");\n`,
- { mode: 0o755 },
- );
+ await writeFile(join(tools, "rg"), "#!/bin/sh\necho synthetic rg\n", {
+ mode: 0o755,
+ });
const result = {
findings: prepared.fixture.positives.map((expected) => ({
taxonomy: { category: "hardcoded-credentials", cwe: [expected.cwes[0]] },
diff --git a/evals/triage-finding/package.json b/evals/triage-finding/package.json
index 648abad5c..9e706e116 100644
--- a/evals/triage-finding/package.json
+++ b/evals/triage-finding/package.json
@@ -14,9 +14,9 @@
"test:calibration-generate": "node --experimental-strip-types scripts/test-generate-calibration-tests.mts",
"test:calibration-hydrate": "node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts",
"test:dataset": "node --experimental-strip-types scripts/test-calibration-dataset.mts",
- "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-intake-assertions.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-promptfoo-runtime.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts && node --experimental-strip-types --test scripts/test-promptfoo-runner.mts scripts/test-triage-provider.mts",
+ "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-intake-assertions.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-promptfoo-runtime.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts && node --experimental-strip-types --test scripts/test-promptfoo-runner.mts scripts/test-triage-provider.mts",
"test:instructions": "node --experimental-strip-types scripts/test-app-surface-instructions.mts",
- "test:sastbench": "node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts",
+ "test:sastbench": "node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts",
"validate": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./promptfooconfig.yaml",
"validate:calibration": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./promptfooconfig.calibration.yaml",
"validate:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./sastbench/promptfooconfig.sastbench.yaml",
diff --git a/evals/triage-finding/sastbench/scripts/test-hydrate-sastbench-repos.mts b/evals/triage-finding/sastbench/scripts/test-hydrate-sastbench-repos.mts
deleted file mode 100644
index b09314472..000000000
--- a/evals/triage-finding/sastbench/scripts/test-hydrate-sastbench-repos.mts
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env node
-import assert from "node:assert/strict";
-import path from "node:path";
-
-import { buildHydrationPlan, parseArgs } from "./hydrate-sastbench-repos.mts";
-
-const repoUrl = "https://github.com/example/project";
-const firstCommit = "1".repeat(40);
-const secondCommit = "2".repeat(40);
-const baseRecord = {
- repo_name: "example/project",
- repo_url: repoUrl,
- commit_hash: firstCommit,
- ground_truth: "true_positive",
-};
-const records = [
- { ...baseRecord },
- { ...baseRecord, ground_truth: "false_positive" },
- { ...baseRecord, commit_hash: secondCommit },
-];
-const cacheRoot = path.resolve("/tmp/sastbench-git-cache");
-const targetRoot = path.resolve("/tmp/sastbench-targets");
-
-assert.equal(parseArgs(["--", "--dry-run"]).dryRun, true);
-
-const jobs = buildHydrationPlan(records, { cacheRoot, targetRoot });
-assert.equal(
- buildHydrationPlan(records, { cacheRoot, targetRoot })[0].cacheDir,
- jobs[0].cacheDir,
-);
-assert.equal(jobs.length, 2);
-assert.equal(jobs[0].caseCount, 2);
-assert.equal(jobs[1].caseCount, 1);
-assert.equal(
- jobs[0].cacheDir,
- path.join(cacheRoot, "repo-acb9cf7430784a93.git"),
-);
-for (const job of jobs) {
- assert.equal(
- path.relative(targetRoot, job.targetDir).startsWith(".."),
- false,
- );
- assert.equal(path.relative(cacheRoot, job.cacheDir).startsWith(".."), false);
- assert.equal(job.repoUrl, repoUrl);
-}
-
-assert.throws(
- () =>
- buildHydrationPlan(
- [
- {
- ...baseRecord,
- repo_name: "same/name",
- repo_url: "https://github.com/example/one",
- },
- {
- ...baseRecord,
- repo_name: "same/name",
- repo_url: "https://github.com/example/two",
- commit_hash: secondCommit,
- },
- ],
- { cacheRoot, targetRoot },
- ),
- /maps to multiple repository URLs/,
-);
-
-console.log("sastbench hydration planning tests passed");
diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts
index fb124b869..69e85e00e 100644
--- a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts
+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts
@@ -1,7 +1,7 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import path from "node:path";
-import { buildHydrationPlan } from "./hydrate-sastbench-repos.mts";
+import { buildHydrationPlan, parseArgs } from "./hydrate-sastbench-repos.mts";
import {
REPRESENTATIVE_SAMPLE_SPEC,
@@ -119,15 +119,36 @@ assert.equal(
);
const stateId = repositoryStateId(positive);
-assert.match(stateId, /^state-[a-f0-9]{16}$/);
assert.equal(repositoryStateId(negative), stateId);
-const jobs = buildHydrationPlan([positive, negative], { targetRoot });
-assert.equal(jobs.length, 1);
-assert.equal(jobs[0].caseCount, 2);
-assert.equal(jobs[0].repoUrl, positive.repo_url);
-assert.equal(jobs[0].commitHash, positive.commit_hash);
-assert.equal(jobs[0].targetDir, path.join(targetRoot, stateId));
+assert.equal(parseArgs(["--", "--dry-run"]).dryRun, true);
+const cacheRoot = path.resolve("/tmp/sastbench-git-cache");
+const otherRevision = { ...positive, commit_hash: "2".repeat(40) };
+const jobs = buildHydrationPlan([positive, negative, otherRevision], {
+ cacheRoot,
+ targetRoot,
+});
+assert.deepEqual(
+ jobs.map((job) => job.caseCount),
+ [2, 1],
+);
+for (const [index, record] of [positive, otherRevision].entries()) {
+ const job = jobs[index]!;
+ assert.equal(job.repoUrl, record.repo_url);
+ assert.equal(job.commitHash, record.commit_hash);
+ const target = repositoryStateId(record);
+ assert.match(target, /^state-[a-f0-9]{16}$/);
+ assert.equal(job.targetDir, path.join(targetRoot, target));
+ assert.equal(job.cacheDir, path.join(cacheRoot, "repo-acb9cf7430784a93.git"));
+}
+assert.throws(
+ () =>
+ buildHydrationPlan([
+ positive,
+ { ...otherRevision, repo_url: "https://github.com/example/other" },
+ ]),
+ /maps to multiple repository URLs/,
+);
const promptVars = buildPromptVars(positive, 0, targetRoot);
assert.equal(promptVars.case_id, "sastbench-000000");
diff --git a/evals/triage-finding/scripts/test-promptfoo-runner.mts b/evals/triage-finding/scripts/test-promptfoo-runner.mts
index aeb619421..39f19cb0d 100644
--- a/evals/triage-finding/scripts/test-promptfoo-runner.mts
+++ b/evals/triage-finding/scripts/test-promptfoo-runner.mts
@@ -1,8 +1,9 @@
import assert from "node:assert/strict";
-import { spawn } from "node:child_process";
+import { spawn, type ChildProcess } from "node:child_process";
+import { once } from "node:events";
import fs from "node:fs";
import path from "node:path";
-import test from "node:test";
+import test, { type TestContext } from "node:test";
import { pathToFileURL } from "node:url";
import { stageSkillRuntime } from "./run-promptfoo.mts";
@@ -88,6 +89,43 @@ test("initializes native providers only in the Promptfoo child", async () => {
}
});
+async function interruptRunner(
+ t: TestContext,
+ child: ChildProcess,
+ interrupt: () => void,
+ exitCode: number,
+) {
+ t.after(() => {
+ if (child.exitCode === null) child.kill("SIGKILL");
+ });
+ let stderr = "";
+ child.stderr!.on("data", (chunk) => {
+ stderr += chunk;
+ });
+ const exited = once(child, "exit");
+ const outputEnded = once(child.stdout!, "end");
+ let output = "";
+ const ready = new Promise<{ root: string }>((resolve) => {
+ child.stdout!.on("data", (chunk) => {
+ output += chunk;
+ if (output.includes("\n")) resolve(JSON.parse(output.split("\n")[0]));
+ });
+ });
+ const { root } = await Promise.race([
+ ready,
+ exited.then(() => {
+ throw new Error(`runner exited early: ${stderr}`);
+ }),
+ ]);
+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+ assert.ok(fs.existsSync(root));
+ interrupt();
+ assert.deepEqual(await exited, [exitCode, null], stderr);
+ assert.equal(fs.existsSync(root), false);
+ await outputEnded;
+ return output;
+}
+
test(
"Windows console Ctrl+C lets the child save progress before runtime cleanup",
{ timeout: 15000 },
@@ -137,112 +175,64 @@ test(
["--experimental-strip-types", "--eval", driver],
{ stdio: ["ignore", "pipe", "pipe", "ipc"] },
);
- t.after(() => {
- if (child.exitCode === null) child.kill("SIGKILL");
- });
- let stderr = "";
- child.stderr!.on("data", (chunk) => {
- stderr += chunk;
- });
- const exited = new Promise<number | null>((resolve, reject) => {
- child.once("error", reject);
- child.once("close", resolve);
- });
- let output = "";
- const ready = new Promise<{ root: string }>((resolve) => {
- child.stdout!.on("data", (chunk) => {
- output += chunk;
- if (output.includes("\n")) resolve(JSON.parse(output.split("\n")[0]));
- });
- });
- const { root } = await Promise.race([
- ready,
- exited.then(() => {
- throw new Error(`runner exited early: ${stderr}`);
- }),
- ]);
- t.after(() => fs.rmSync(root, { recursive: true, force: true }));
- assert.ok(fs.existsSync(root));
- child.send("console-interrupt");
- assert.equal(await exited, 130, stderr);
+ const output = await interruptRunner(
+ t,
+ child,
+ () => child.send("console-interrupt"),
+ 130,
+ );
assert.match(output, /saved:1:true/);
assert.doesNotMatch(output, /forced:/);
- assert.equal(fs.existsSync(root), false);
},
);
-for (const [signal, exitCode] of [
- ["SIGINT", 130],
- ["SIGTERM", 143],
+for (const [signal, exitCode, delivery] of [
+ ["SIGINT", 130, "pid"],
+ ["SIGINT", 130, "group"],
+ ["SIGTERM", 143, "pid"],
+ ["SIGTERM", 143, "group"],
] as const) {
- for (const delivery of ["pid", "group"]) {
- test(
- `runner forwards ${signal} once (${delivery}) and removes its runtime after the child exits`,
- { skip: process.platform === "win32", timeout: 15000 },
- async (t) => {
- const target = `
- let received = 0;
- process.on(${JSON.stringify(signal)}, () => {
- received++;
- if (received > 1) process.exit(99);
- setTimeout(() => { console.log("graceful:" + received); process.exit(0); }, 150);
- });
- console.log(JSON.stringify({ root: process.env.TRIAGE_RUNTIME_ROOT }));
- setInterval(() => {}, 1000);
- `;
- const driver = `
- const cp = require('node:child_process');
- const spawn = cp.spawn;
- cp.spawn = (command, args, options) => spawn(process.execPath, ['--eval', args[0].endsWith("build_mcp_app.mjs") ? "" : ${JSON.stringify(target)}], options);
- import(${JSON.stringify(new URL("./run-promptfoo.mts", import.meta.url).href)}).then(({ runPromptfoo }) => runPromptfoo([])).then(code => { process.exitCode = code; });
+ test(
+ `runner forwards ${signal} once (${delivery}) and removes its runtime after the child exits`,
+ { skip: process.platform === "win32", timeout: 15000 },
+ async (t) => {
+ const target = `
+ let received = 0;
+ process.on(${JSON.stringify(signal)}, () => {
+ received++;
+ if (received > 1) process.exit(99);
+ setTimeout(() => {
+ console.log("graceful:" + received + ":" + require("node:fs").existsSync(process.env.TRIAGE_RUNTIME_ROOT));
+ process.exit(0);
+ }, 150);
+ });
+ console.log(JSON.stringify({ root: process.env.TRIAGE_RUNTIME_ROOT }));
+ setInterval(() => {}, 1000);
`;
- const child = spawn(
- process.execPath,
- ["--experimental-strip-types", "--eval", driver],
- {
- stdio: ["ignore", "pipe", "pipe"],
- detached: true,
- },
- );
- t.after(() => {
- if (child.exitCode === null) child.kill("SIGKILL");
- });
- let stderr = "";
- child.stderr.on("data", (chunk) => {
- stderr += chunk;
- });
- const exited = new Promise<{
- code: number | null;
- signal: NodeJS.Signals | null;
- }>((resolve, reject) => {
- child.once("error", reject);
- child.once("exit", (code, signal) => resolve({ code, signal }));
- });
- let text = "";
- const ready = new Promise<{ root: string }>((resolve) => {
- child.stdout.on("data", (chunk) => {
- text += chunk;
- if (text.includes("\n")) resolve(JSON.parse(text.split("\n")[0]));
- });
- });
- const { root } = await Promise.race([
- ready,
- exited.then(() => {
- throw new Error(`runner exited early: ${stderr}`);
- }),
- ]);
- t.after(() => fs.rmSync(root, { recursive: true, force: true }));
- assert.ok(fs.existsSync(root));
- if (delivery === "group") process.kill(-child.pid!, signal);
- else child.kill(signal);
- assert.deepEqual(
- await exited,
- { code: exitCode, signal: null },
- stderr,
- );
- assert.match(text, /graceful:1/);
- assert.equal(fs.existsSync(root), false);
- },
- );
- }
+ const driver = `
+ const cp = require('node:child_process');
+ const spawn = cp.spawn;
+ cp.spawn = (command, args, options) => spawn(process.execPath, ['--eval', args[0].endsWith("build_mcp_app.mjs") ? "" : ${JSON.stringify(target)}], options);
+ import(${JSON.stringify(new URL("./run-promptfoo.mts", import.meta.url).href)}).then(({ runPromptfoo }) => runPromptfoo([])).then(code => { process.exitCode = code; });
+ `;
+ const child = spawn(
+ process.execPath,
+ ["--experimental-strip-types", "--eval", driver],
+ {
+ stdio: ["ignore", "pipe", "pipe"],
+ detached: true,
+ },
+ );
+ const output = await interruptRunner(
+ t,
+ child,
+ () => {
+ if (delivery === "group") process.kill(-child.pid!, signal);
+ else child.kill(signal);
+ },
+ exitCode,
+ );
+ assert.match(output, /graceful:1:true/);
+ },
+ );
}
diff --git a/evals/triage-finding/scripts/test-triage-provider.mts b/evals/triage-finding/scripts/test-triage-provider.mts
index 35520e993..b7cd3a93c 100644
--- a/evals/triage-finding/scripts/test-triage-provider.mts
+++ b/evals/triage-finding/scripts/test-triage-provider.mts
@@ -49,6 +49,71 @@ fs.writeFileSync(path.join(fixture, 'SECURITY.md'), '# Synthetic policy\\n');
const policy = require('node:child_process').execFileSync(launcher, ['--helper', 'resolve-security-md', '--repo', fixture, '--scope', 'src/server.js', '--out', '-'], {encoding:'utf8'});
`;
+async function replayInViewer(
+ t: TestContext,
+ environment: NodeJS.ProcessEnv,
+ request: {
+ evaluationId: unknown;
+ prompt: string;
+ variables: Record<string, unknown>;
+ },
+) {
+ const listener = createServer();
+ await new Promise<void>((resolve) =>
+ listener.listen(0, "127.0.0.1", resolve),
+ );
+ const address = listener.address();
+ assert.ok(address && typeof address !== "string");
+ const port = address.port;
+ await new Promise<void>((resolve, reject) =>
+ listener.close((error) => (error ? reject(error) : resolve())),
+ );
+ const viewer = spawn(
+ process.execPath,
+ [...replayRunnerArgs(), "view", "--port", String(port), "--no"],
+ {
+ cwd: evalRoot,
+ env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },
+ stdio: ["ignore", "pipe", "pipe"],
+ },
+ );
+ let viewerOutput = "";
+ for (const stream of [viewer.stdout, viewer.stderr])
+ stream.on("data", (chunk) => {
+ viewerOutput += chunk;
+ });
+ const closed = new Promise<number | null>((resolve, reject) => {
+ viewer.once("error", reject);
+ viewer.once("close", resolve);
+ });
+ t.after(async () => {
+ if (viewer.exitCode === null) viewer.kill("SIGTERM");
+ await closed;
+ });
+ const url = `http://127.0.0.1:${port}`;
+ while (true) {
+ try {
+ const response = await fetch(`${url}/api/eval`);
+ if (response.ok) break;
+ } catch {
+ /* The actual viewer is still building its staged runtime. */
+ }
+ assert.equal(viewer.exitCode, null, viewerOutput);
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ const replay = await fetch(`${url}/api/eval/replay`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ testIndex: 0, ...request }),
+ });
+ assert.equal(replay.status, 200, viewerOutput);
+ const replayResult = (await replay.json()) as { output: unknown };
+ assert.equal(replayResult.output, "ok", JSON.stringify(replayResult));
+ viewer.kill("SIGTERM");
+ await closed;
+ return viewerOutput;
+}
+
function providerConfig(name: string) {
return parse(
fs.readFileSync(path.join(evalRoot, "providers", `${name}.yaml`), "utf8"),
@@ -1191,63 +1256,13 @@ if (fs.existsSync(${JSON.stringify(failure)})) {
const replayVariables =
JSON.parse((resumedResult.metadata as string) || "{}").inputVars ||
JSON.parse(resumedResult.test_case).vars;
- const listener = createServer();
- await new Promise<void>((resolve) =>
- listener.listen(0, "127.0.0.1", resolve),
- );
- const address = listener.address();
- assert.ok(address && typeof address !== "string");
- const port = address.port;
- await new Promise<void>((resolve, reject) =>
- listener.close((error) => (error ? reject(error) : resolve())),
- );
- const viewer = spawn(
- process.execPath,
- [...replayRunnerArgs(), "view", "--port", String(port), "--no"],
- {
- cwd: evalRoot,
- env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },
- stdio: ["ignore", "pipe", "pipe"],
- },
- );
- let viewerOutput = "";
- for (const stream of [viewer.stdout, viewer.stderr])
- stream.on("data", (chunk) => {
- viewerOutput += chunk;
- });
- const closed = new Promise<number | null>((resolve, reject) => {
- viewer.once("error", reject);
- viewer.once("close", resolve);
- });
- t.after(async () => {
- if (viewer.exitCode === null) viewer.kill("SIGTERM");
- await closed;
+ await replayInViewer(t, environment, {
+ evaluationId: stored.id,
+ prompt:
+ "Read and follow skills/triage-finding/SKILL.md.\nTARGET=" +
+ replayVariables.target_repo,
+ variables: replayVariables,
});
- const url = `http://127.0.0.1:${port}`;
- while (true) {
- try {
- if ((await fetch(`${url}/api/eval`)).ok) break;
- } catch {}
- assert.equal(viewer.exitCode, null, viewerOutput);
- await new Promise((resolve) => setTimeout(resolve, 50));
- }
- const replay = await fetch(`${url}/api/eval/replay`, {
- method: "POST",
- headers: { "content-type": "application/json" },
- body: JSON.stringify({
- evaluationId: stored.id,
- testIndex: 0,
- prompt:
- "Read and follow skills/triage-finding/SKILL.md.\nTARGET=" +
- replayVariables.target_repo,
- variables: replayVariables,
- }),
- });
- assert.equal(replay.status, 200, viewerOutput);
- const replayResult = (await replay.json()) as { output: unknown };
- assert.equal(replayResult.output, "ok", JSON.stringify(replayResult));
- viewer.kill("SIGTERM");
- await closed;
const rows = fs
.readFileSync(capture, "utf8")
.trim()
@@ -1512,64 +1527,11 @@ if (fs.existsSync(${JSON.stringify(failure)})) {
.inputVars || JSON.parse(resumedResult.test_case).vars;
assert.equal(typeof replayVariables.target_repo, "string");
- const listener = createServer();
- await new Promise<void>((resolve) =>
- listener.listen(0, "127.0.0.1", resolve),
- );
- const address = listener.address();
- assert.ok(address && typeof address !== "string");
- const port = address.port;
- await new Promise<void>((resolve, reject) =>
- listener.close((error) => (error ? reject(error) : resolve())),
- );
- const viewer = spawn(
- process.execPath,
- [...replayRunnerArgs(), "view", "--port", String(port), "--no"],
- {
- cwd: evalRoot,
- env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },
- stdio: ["ignore", "pipe", "pipe"],
- },
- );
- let viewerOutput = "";
- for (const stream of [viewer.stdout, viewer.stderr])
- stream.on("data", (chunk) => {
- viewerOutput += chunk;
- });
- const closed = new Promise<number | null>((resolve, reject) => {
- viewer.once("error", reject);
- viewer.once("close", resolve);
- });
- t.after(async () => {
- if (viewer.exitCode === null) viewer.kill("SIGTERM");
- await closed;
- });
- const url = `http://127.0.0.1:${port}`;
- while (true) {
- try {
- const response = await fetch(`${url}/api/eval`);
- if (response.ok) break;
- } catch {
- /* The actual viewer is still building its staged runtime. */
- }
- assert.equal(viewer.exitCode, null, viewerOutput);
- await new Promise((resolve) => setTimeout(resolve, 50));
- }
- const replay = await fetch(`${url}/api/eval/replay`, {
- method: "POST",
- headers: { "content-type": "application/json" },
- body: JSON.stringify({
- evaluationId: stored.id,
- testIndex: 0,
- prompt: "hello",
- variables: replayVariables,
- }),
+ const viewerOutput = await replayInViewer(t, environment, {
+ evaluationId: stored.id,
+ prompt: "hello",
+ variables: replayVariables,
});
- assert.equal(replay.status, 200, viewerOutput);
- const replayResult = (await replay.json()) as { output: unknown };
- assert.equal(replayResult.output, "ok", JSON.stringify(replayResult));
- viewer.kill("SIGTERM");
- await closed;
assert.doesNotMatch(viewerOutput, /afterEach extension hook failed/);
const rows = fs
.readFileSync(capture, "utf8")
diff --git a/github-action/tests/main.test.ts b/github-action/tests/main.test.ts
index 439a94554..fbaf399d3 100644
--- a/github-action/tests/main.test.ts
+++ b/github-action/tests/main.test.ts
@@ -55,11 +55,12 @@ async function harness(t: TestContext, scenario: Scenario = 'schedule', serverUr
await writeFile(join(repository, 'app.txt'), 'synthetic source\n');
git('add', '.'); git('commit', '-qm', 'base');
const base = git('rev-parse', 'HEAD');
+ let sha = base;
if (scenario !== 'schedule') {
await writeFile(join(repository, scenario === 'policy-pr' ? 'SECURITY.md' : 'app.txt'), 'synthetic change\n');
git('add', '.'); git('commit', '-qm', 'PR change');
+ sha = git('rev-parse', 'HEAD');
}
- const sha = git('rev-parse', 'HEAD');
const previousEnv = { ...process.env };
const previousExitCode = process.exitCode;
t.after(async () => {
@@ -197,10 +198,18 @@ test('PR severity failure retains SARIF outputs despite unrelated scope notes',
});
test('scheduled complete report-only scan succeeds', async (t) => {
- const app = await harness(t); const result = await app.run();
+ const app = await harness(t);
+ process.env.INPUT_FOO = 'unrelated workflow value';
+ app.setInput('effort', 'medium');
+ app.configure({scanResult: value => {
+ value.turn = {usage: {input_tokens: 1200, output_tokens: 50}};
+ }});
+ const result = await app.run();
assertAttribution(result.args, 'openai');
+ assert.ok(result.args.includes('--effort=medium'));
assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed');
assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['sarif-upload-ready'], 'true');
+ assert.equal(result.outputs['report-status'], 'ready'); assert.equal(result.cleanups, 1);
assert.equal(result.outputs['analysis-ref'], 'refs/heads/main');
assert.ok(result.args.includes('--verbose'));
assert.match(result.logs, /Synthetic live CLI progress/);
@@ -209,34 +218,12 @@ test('scheduled complete report-only scan succeeds', async (t) => {
assert.match(result.logs, /Security scan exited after \d+m \d+s; exit code: 0/);
assert.match(result.logs, /Scan: completed; findings policy: passed; report: ready/);
assert.match(result.logs, /Estimated cost: \$0.1250/);
+ assert.equal(result.outputs['estimated-cost'], '0.125');
+ assert.ok(result.summary.includes('1,200 input (0 cached) · 50 output'));
assert.match(result.logs, /Scan completed\. Findings are reported without failing the job\./);
assert.match(result.summary, /\*\*Scan completed\. Findings are reported without failing the job\.\*\*/);
});
-test('usage without a cached count preserves a successful Action and its reports', async t => {
- const app = await harness(t);
- app.configure({scanResult: value => {
- value.turn = {usage: {input_tokens: 1200, output_tokens: 50}};
- }});
- const result = await app.run();
- assert.equal(result.exitCode, 0);
- assert.equal(result.outputs['scan-status'], 'completed');
- assert.equal(result.outputs['policy-status'], 'passed');
- assert.equal(result.outputs['report-status'], 'ready');
- assert.equal(result.outputs['sarif-upload-ready'], 'true');
- assert.equal(result.cleanups, 1);
- assert.ok(result.summary.includes('1,200 input (0 cached) · 50 output'));
- assert.equal(result.outputs['estimated-cost'], '0.125');
-});
-
-test('unrelated ambient input variables do not reject a valid scan', async (t) => {
- const app = await harness(t); process.env.INPUT_FOO = 'unrelated workflow value';
- app.setInput('effort', 'medium');
- const result = await app.run();
- assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed');
- assert.ok(result.args.includes('--effort=medium'));
-});
-
for (const [scenario, mode] of [['schedule', 'standard'], ['pr', 'standard'], ['schedule', 'deep']] as const) {
test(`enterprise ${scenario} ${mode} scans preserve reports and upload identity`, async t => {
const serverUrl = 'https://example-enterprise.ghe.com';
@@ -374,16 +361,6 @@ test('Bedrock dry-run forwards no credentials and remains a skipped scan', async
assert.equal(result.environment.AWS_BEARER_TOKEN_BEDROCK, undefined); assert.equal(result.environment.OPENAI_API_KEY, undefined);
});
-test('Bedrock authentication errors remain scan failures', async t => {
- const app = await harness(t);
- app.setInput('provider', 'amazon-bedrock'); app.setInput('model', 'openai.synthetic-model');
- delete process.env.OPENAI_API_KEY;
- app.configure({exitCode:2, cliFailure:true});
- const result = await app.run();
- assert.equal(result.exitCode, 1); assert.equal(result.processes, 1);
- assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false');
-});
-
for (const scenario of ['schedule', 'pr'] as const) {
test(`missing API key fails a ${scenario === 'pr' ? 'Dependabot PR' : 'scheduled scan'} with setup guidance`, async (t) => {
const app = await harness(t, scenario); delete process.env.OPENAI_API_KEY;
@@ -428,15 +405,6 @@ for (const threshold of ['none', 'high']) {
});
}
-test('verbose false suppresses CLI diagnostics but retains lifecycle and results', async (t) => {
- const app = await harness(t); app.setInput('verbose', 'false'); const result = await app.run();
- assert.equal(result.exitCode, 0);
- assert.ok(!result.args.includes('--verbose'));
- assert.doesNotMatch(result.logs, /Synthetic live CLI progress/);
- assert.match(result.logs, /Starting security scan/);
- assert.match(result.logs, /Scan: completed/);
-});
-
test('configuration diagnostics preserve values and cannot inject runner commands', async (t) => {
const app = await harness(t);
app.setInput('model', 'synthetic-offline-test-key\u2028::error::injected');
@@ -459,6 +427,10 @@ test('scope exclusions and deferred notes do not fail completed scans or make fi
assert.equal(result.outputs['exit-code'], '0');
assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['sarif-upload-ready'], 'true');
assert.equal(result.outputs['high-count'], '1'); assert.ok(result.outputs['json-path']);
+ assert.ok(!result.args.includes('--verbose'));
+ assert.doesNotMatch(result.logs, /Synthetic live CLI progress/);
+ assert.match(result.logs, /Starting security scan/);
+ assert.match(result.logs, /Scan: completed/);
assert.match(result.logs, /Findings:.*high: 1/);
assert.doesNotMatch(result.logs, /Provisional|Scan coverage is partial|::error::/);
assert.doesNotMatch(result.summary, /provisional|not evaluated|not-evaluated/i);
@@ -466,13 +438,15 @@ test('scope exclusions and deferred notes do not fail completed scans or make fi
assert.equal(coverage.deferred[0].reason, 'Dependency <example> unavailable; validation deferred.');
});
-test('execution failure diagnostics cannot emit legacy runner commands', async (t) => {
+test('CLI warnings fail a completed scan and cannot emit legacy runner commands', async (t) => {
const app = await harness(t);
- app.configure({exitCode: 2, scanResult: value => {
+ app.configure({scanResult: value => {
value.warnings = ['Synthetic diagnostic: ##[stop-commands]token'];
}});
const result = await app.run();
- assert.equal(result.exitCode, 1);
+ assert.equal(result.exitCode, 1); assert.equal(result.outputs['exit-code'], '0');
+ assert.equal(result.outputs['policy-status'], 'not-evaluated');
+ assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false');
assert.match(result.logs, /Synthetic diagnostic: ##\\\[stop-commands\]token/);
assert.doesNotMatch(result.logs, /##\[stop-commands\]/);
assert.match(result.summary, /Synthetic diagnostic: ##&#91;stop-commands&#93;token/);
@@ -492,8 +466,7 @@ test('exit 2 fails the step even when finalized reports were retained', async (t
for (const [name, scanProcess] of [
['timeout', {timedOut: true}], ['interruption', {interrupted: true}],
- ['signal', {signal: 'SIGTERM' as const}], ['terminated exit', {exitCode: 143}],
- ['malformed output', {stdout: 'not JSON'}],
+ ['signal', {signal: 'SIGTERM' as const}],
] as const) {
test(`retained reports cannot hide scan ${name}`, async (t) => {
const app = await harness(t); app.configure({scanProcess});
@@ -503,18 +476,6 @@ for (const [name, scanProcess] of [
});
}
-for (const [name, scanResult] of [
- ['failed manifest', (value: any) => { value.manifest.scan.status = 'failed'; }],
- ['target-change warning', (value: any) => { value.warnings = ['Synthetic target changed during scanning.']; }],
-] as const) {
- test(`${name} cannot become a successful scan`, async (t) => {
- const app = await harness(t); app.configure({scanResult});
- const result = await app.run();
- assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'failed');
- assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['sarif-upload-ready'], 'false');
- });
-}
-
for (const missingReport of ['scan-manifest.json', 'findings.json', 'coverage.json']) {
test(`completed scan fails when required ${missingReport} is missing without artifact upload`, async (t) => {
const app = await harness(t); app.configure({missingReport});
@@ -693,10 +654,13 @@ test('archive read failure preserves scan reports, summary and cleanup', {
await assert.rejects(stat(staging), {code: 'ENOENT'});
});
-test('CLI authentication failure leaves findings unavailable instead of reporting a threshold failure', async (t) => {
- const app = await harness(t); app.configure({exitCode: 2, cliFailure: true});
+test('Bedrock CLI authentication failure leaves findings unavailable', async (t) => {
+ const app = await harness(t);
+ app.setInput('provider', 'amazon-bedrock'); app.setInput('model', 'openai.synthetic-model');
+ delete process.env.OPENAI_API_KEY;
+ app.configure({exitCode: 2, cliFailure: true});
const result = await app.run();
- assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'failed');
+ assert.equal(result.exitCode, 1); assert.equal(result.processes, 1); assert.equal(result.outputs['scan-status'], 'failed');
assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['sarif-upload-ready'], 'false');
assert.equal(result.outputs['json-path'], '');
assert.match(result.summary, /Synthetic API authentication failure/);
@@ -706,13 +670,6 @@ test('CLI authentication failure leaves findings unavailable instead of reportin
assert.match(result.logs, /Findings: unavailable/);
});
-test('malformed findings keep count outputs empty', async (t) => {
- const app = await harness(t); app.configure({scanResult: value => { value.findings.findings[0].severity = null; }});
- const result = await app.run();
- assert.equal(result.exitCode, 1);
- for (const level of ['critical', 'high', 'medium', 'low', 'informational']) assert.equal(result.outputs[`${level}-count`], '');
-});
-
test('valid empty findings publish zero counts', async (t) => {
const app = await harness(t); app.configure({scanResult: value => { value.findings.findings = []; }});
const result = await app.run();
diff --git a/github-action/tests/results.test.ts b/github-action/tests/results.test.ts
index 7ed82ea28..a86f5a9c5 100644
--- a/github-action/tests/results.test.ts
+++ b/github-action/tests/results.test.ts
@@ -34,6 +34,7 @@ function change(options: ResultOptions, update: (value: any) => void): void {
test('adapts a synthetic CLI JSON result into findings, counts and report outputs', async (t) => {
const opts = await fixture(t);
+ change(opts, (value) => { value.findings.findings[0].title = 'Title from the CLI result'; });
const result = await analyzeResults(opts);
assert.deepEqual(result.errors, []);
assert.equal(result.scanStatus, 'completed');
@@ -44,6 +45,7 @@ test('adapts a synthetic CLI JSON result into findings, counts and report output
assert.deepEqual(result.usage, { input_tokens: 1200, cached_input_tokens: 200, output_tokens: 50 });
assert.equal(result.estimatedCost, 0.125);
assert.deepEqual(result.estimatedCostRange, { min: 0.125, max: 0.25, context: 'unknown' });
+ assert.equal(result.findings[0]?.title, 'Title from the CLI result');
assert.equal(result.findings[0]?.path, 'src/extract.py');
assert.equal(result.findings[0]?.startLine, 41);
assert.equal(result.paths.jsonPath, join(opts.resultsDirectory, 'findings.json'));
@@ -62,14 +64,6 @@ test('token usage remains available for unpriced models and distinguishes missin
}
});
-test('uses the structured CLI findings instead of reparsing report documents', async (t) => {
- const opts = await fixture(t);
- change(opts, (value) => { value.findings.findings[0].title = 'Title from the CLI result'; });
- const result = await analyzeResults(opts);
- assert.equal(result.scanStatus, 'completed');
- assert.equal(result.findings[0]?.title, 'Title from the CLI result');
-});
-
test('annotations prefer the root control over an earlier entry point', async (t) => {
const opts = await fixture(t);
change(opts, (value) => {
@@ -123,6 +117,7 @@ test('operational exits and signal termination cannot turn complete output into
assert.equal(result.policyStatus, 'not-evaluated');
assert.equal(result.sarifUploadReady, false);
assert.equal(result.findings.length, 1);
+ assert.equal(result.counts?.high, 1);
}
});
@@ -302,12 +297,10 @@ test('SARIF publication rejects external references and encoded traversal paths'
});
-test('valid empty and retained findings distinguish zero counts from unavailable results', async (t) => {
+test('valid empty findings have zero counts', async (t) => {
const opts = await fixture(t);
change(opts, (value) => { value.findings.findings = []; });
assert.deepEqual((await analyzeResults(opts)).counts, {critical: 0, high: 0, medium: 0, low: 0, informational: 0});
- const retained = await fixture(t);
- assert.equal((await analyzeResults({...retained, exitCode: 2})).counts?.high, 1);
});
test('accepted relative path spellings preserve completed findings and SARIF', async (t) => {