OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test(tooling): reduce repeated Action and eval setup (#2040)
Michael D'Angelob3e9d40f5a3d
diff --git a/.github/scripts/invoice-desk-target.test.mjs b/.github/scripts/invoice-desk-target.test.mjsindex 7a4be077a..8cce9de3d 100644--- a/.github/scripts/invoice-desk-target.test.mjs+++ b/.github/scripts/invoice-desk-target.test.mjs@@ -27,55 +27,61 @@ function pullRequest(overrides = {}) {};}-test("selects the requested PR head rather than the trusted workflow commit", async () => {- assert.deepEqual(- await resolveScanTargets(context, async (number) => {- assert.equal(number, "7");- return pullRequest();- }),- [{ pr: 7, sha: sourceSha }],- );-});--test("includes fork and draft PRs without depending on a PR workflow run", async () => {- assert.deepEqual(- await resolveScanTargets(context, async () =>- pullRequest({- draft: true,- head: { sha: sourceSha, repo: { full_name: "contributor/invoices" } },- }),- ),- [{ pr: 7, sha: sourceSha }],- );+test("initial selection and approval recheck require the selected PR and revision to remain eligible", async () => {+ const target = { pr: 7, sha: sourceSha };+ // The approval step must use its matrix target even if dispatch inputs differ.+ const recheck = {+ ...context,+ target: { ...target },+ event: { inputs: { pr_number: "9", source_sha: mainSha } },+ };+ for (const request of [context, recheck]) {+ for (const [overrides, expected] of [+ [{}, [target]],+ [+ {+ draft: true,+ head: { sha: sourceSha, repo: { full_name: "contributor/invoices" } },+ },+ [target],+ ],+ [{ state: "closed" }, []],+ [{ base: { ref: "release", repo: { full_name: repository } } }, []],+ [{ base: { ref: "main", repo: { full_name: "other/invoices" } } }, []],+ [{ head: { sha: mainSha } }, []],+ [{ head: { sha: "3".repeat(40) } }, []],+ ]) {+ const actual = await resolveScanTargets(request, async (number) => {+ assert.equal(number, "7");+ return pullRequest(overrides);+ });+ assert.deepEqual(actual, expected);+ }+ }});-test("rechecking after approval excludes closed, retargeted, and superseded PRs", async () => {- assert.deepEqual(- await resolveScanTargets(context, async () => pullRequest()),- [{ pr: 7, sha: sourceSha }],- );- for (const changed of [- { state: "closed" },- { base: { ref: "release", repo: { full_name: repository } } },- { base: { ref: "main", repo: { full_name: "other/invoices" } } },- { head: { sha: "3".repeat(40) } },+test("manual and scheduled baselines use the trusted workflow commit before and after approval", async () => {+ const target = { pr: 0, sha: mainSha };+ const scheduled = {+ ...context,+ eventName: "schedule",+ event: { schedule: "17 3 * * 1" },+ };+ for (const request of [+ { ...context, event: { inputs: {} } },+ scheduled,+ { ...context, target: { ...target } },+ { ...scheduled, target: { ...target } },]) {assert.deepEqual(- await resolveScanTargets(context, async () => pullRequest(changed)),- [],+ await resolveScanTargets(request, () =>+ assert.fail("unexpected API call"),+ ),+ [target],);}});-test("manual baseline scans use the trusted workflow commit", async () => {- assert.deepEqual(- await resolveScanTargets({ ...context, event: { inputs: {} } }, () =>- assert.fail("unexpected API call"),- ),- [{ pr: 0, sha: mainSha }],- );-});-test("incomplete or invalid PR inputs fail instead of falling back to a baseline", async () => {for (const inputs of [{ pr_number: "7" },@@ -117,48 +123,6 @@ test("PR events and completed PR workflows do not queue inference or query GitHu}});-test("scheduled baseline scans use the trusted workflow commit", async () => {- assert.deepEqual(- await resolveScanTargets(- { ...context, eventName: "schedule", event: { schedule: "17 3 * * 1" } },- () => assert.fail("unexpected API call"),- ),- [{ pr: 0, sha: mainSha }],- );-});--test("approval recheck stays bound to its matrix PR and source revision", async () => {- const run = { ...context, target: { pr: 7, sha: sourceSha } };- for (const changed of [- { state: "closed" },- { base: { ref: "release", repo: { full_name: repository } } },- { head: { sha: mainSha } },- ]) {- assert.deepEqual(- await resolveScanTargets(run, async (number) => {- assert.equal(number, "7");- return pullRequest(changed);- }),- [],- );- }- assert.deepEqual(await resolveScanTargets(run, async () => pullRequest()), [- { pr: 7, sha: sourceSha },- ]);-});--test("baseline approval recheck retains the original trusted workflow commit", async () => {- for (const eventName of ["workflow_dispatch", "schedule"]) {- assert.deepEqual(- await resolveScanTargets(- { ...context, eventName, target: { pr: 0, sha: mainSha } },- () => assert.fail("unexpected API call"),- ),- [{ pr: 0, sha: mainSha }],- );- }-});-test("resolver CLI emits the requested target without dispatching another workflow", (t) => {const root = mkdtempSync(join(tmpdir(), "invoice-desk-target-"));t.after(() => rmSync(root, { recursive: true, force: true }));diff --git a/.github/scripts/test_check_plugin_source_compatibility.mts b/.github/scripts/test_check_plugin_source_compatibility.mtsindex a99528dd3..158398027 100644--- a/.github/scripts/test_check_plugin_source_compatibility.mts+++ b/.github/scripts/test_check_plugin_source_compatibility.mts@@ -92,10 +92,10 @@ test("checkout preserves source size with autocrlf", () => {git(root, "add", "--", ".gitattributes", name);unlinkSync(source);git(root, "checkout-index", "--", name);- const result = runChecker("--plugin-root", root);- assert.equal(result.status, 0, result.stderr);assert.deepEqual(readFileSync(source), content);}+ const result = runChecker("--plugin-root", root);+ assert.equal(result.status, 0, result.stderr);});test("accepts Markdown structures, natural line endings, and inline markup", () => {diff --git a/evals/secret-discovery/test_discovery.mts b/evals/secret-discovery/test_discovery.mtsindex 6282e8df1..4f043966e 100644--- a/evals/secret-discovery/test_discovery.mts+++ b/evals/secret-discovery/test_discovery.mts@@ -58,16 +58,18 @@ interface TestResult extends Result {};resolved_questions?: TestFinding[];}-function retainedResult(fixture: ReturnType<typeof createFixture>): TestResult {+const fixture = createFixture();++function retainedResult(source = fixture): TestResult {return {- findings: fixture.positives.map((expected) => ({+ findings: source.positives.map((expected) => ({title: "Embedded credential",taxonomy: { category: "hardcoded-credentials", cwe: [expected.cwes[0]] },locations: [{ path: expected.path, startLine: expected.line, role: "root_control" },],codeEvidence: [- sourceEvidence(fixture, expected.path, expected.line, expected.endLine),+ sourceEvidence(source, expected.path, expected.line, expected.endLine),],})),coverage: {@@ -79,14 +81,8 @@ function retainedResult(fixture: ReturnType<typeof createFixture>): TestResult {};}-function retainedFixture() {- const fixture = createFixture();- return { fixture, result: retainedResult(fixture) };-}-test("counts every retained secret location, including unused and integration source", () => {- const fixture = createFixture();- const report = gradeResult(retainedResult(fixture), fixture);+ const report = gradeResult(retainedResult(), fixture);assert.equal(report.passed, true);assert.equal(report.recall, 1);assert.deepEqual(@@ -103,7 +99,7 @@ test("counts every retained secret location, including unused and integration so});test("resolved, deferred, and intermediate findings do not satisfy final recall", () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();result.coverage.deferred = result.findings;result.resolved_questions = result.findings;result.findings = [];@@ -145,7 +141,7 @@ for (const [name, mutate] of [],] as const) {test(`rejects ${name}`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();mutate(result.findings[0]);const report = gradeResult(result, fixture);assert.equal(report.passed, false);@@ -173,7 +169,7 @@ for (const category of ["without-credentials",]) {test(`rejects negated category ${category}`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();result.findings[0].taxonomy.category = category;const report = gradeResult(result, fixture);assert.equal(report.passed, false);@@ -182,10 +178,9 @@ for (const category of [}test("rejects unrelated categories containing credential-term substrings", () => {- const fixture = createFixture();for (const category of ["tokenization-error", "secretary-data"]) {for (const cwe of ["CWE-798", "CWE-200"]) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].taxonomy = { category, cwe: [cwe] };const report = gradeResult(result, fixture);assert.equal(report.passed, false, `${category}:${cwe}`);@@ -195,7 +190,6 @@ test("rejects unrelated categories containing credential-term substrings", () =>});test("accepts plural and identifier-style credential categories", () => {- const fixture = createFixture();for (const category of ["tokens-exposed","secrets_in_source",@@ -203,14 +197,14 @@ test("accepts plural and identifier-style credential categories", () => {"hardcodedCredentials","credentialExposure",]) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].taxonomy.category = category;assert.equal(gradeResult(result, fixture).passed, true, category);}});test("accepts credential exposure with non-secret context in the category", () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();for (const category of ["credential exposure with non-secret context","credential exposure with no-secret context",@@ -223,7 +217,7 @@ test("accepts credential exposure with non-secret context in the category", () =});test("accepts a secret's valid source range and sensitive-data category", () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();result.findings[0].locations[0].endLine = fixture.positives[0].line + 1;result.findings[0].taxonomy.category = "sensitive-data-exposure";assert.equal(gradeResult(result, fixture).passed, true);@@ -235,14 +229,13 @@ for (const category of ["api-key-exposure",]) {test(`accepts credential category ${category}`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();result.findings[0].taxonomy.category = category;assert.equal(gradeResult(result, fixture).passed, true);});}test("accepts equivalent CWE spelling without weakening generic disclosure categories", () => {- const fixture = createFixture();for (const [cwe, generic] of [["cwe-798", false],["CWE-0798", false],@@ -250,7 +243,7 @@ test("accepts equivalent CWE spelling without weakening generic disclosure categ[" cwe-00200 ", true],["CWE-0540", true],] as const) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].taxonomy.cwe = ["CWE-798", cwe];assert.equal(gradeResult(result, fixture).passed, true, cwe);if (generic) {@@ -261,7 +254,6 @@ test("accepts equivalent CWE spelling without weakening generic disclosure categ});test("rejects unrelated or malformed CWE identifiers after normalization", () => {- const fixture = createFixture();for (const cwe of [" cwe-0089 ","CWE-798-extra",@@ -270,7 +262,7 @@ test("rejects unrelated or malformed CWE identifiers after normalization", () =>"CWE-321",]) {for (const withValidCwe of [false, true]) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].taxonomy.cwe = withValidCwe ? ["CWE-798", cwe] : [cwe];const report = gradeResult(result, fixture);assert.equal(report.passed, false, cwe);@@ -281,12 +273,11 @@ test("rejects unrelated or malformed CWE identifiers after normalization", () =>});test("matches path casing only on Windows while preserving citation checks", () => {- const fixture = createFixture();const descriptor = Object.getOwnPropertyDescriptor(process, "platform");try {for (const platform of ["linux", "win32"]) {Object.defineProperty(process, "platform", { value: platform });- const result = retainedResult(fixture);+ const result = retainedResult();const finding = result.findings[0];finding.locations.push({ path: "src/client.py", startLine: 7, role: "sink" },@@ -315,7 +306,7 @@ test("matches path casing only on Windows while preserving citation checks", ()for (const cwe of ["CWE-256", "CWE-259", "CWE-260"]) {test(`accepts ${cwe} for the password in the database configuration`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();const databaseIndex = fixture.positives.findIndex((entry) => entry.id === "dotenv-url",);@@ -328,7 +319,7 @@ for (const cwe of ["CWE-256", "CWE-259", "CWE-260"]) {for (const cwe of ["CWE-312", "CWE-313"]) {test(`accepts ${cwe} for credentials stored in source and configuration files`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();for (const finding of result.findings) finding.taxonomy.cwe = [cwe];assert.equal(gradeResult(result, fixture).passed, true);@@ -340,7 +331,6 @@ for (const cwe of ["CWE-312", "CWE-313"]) {}test("keeps password and cryptographic-key CWEs specific to those exposures", () => {- const fixture = createFixture();for (const [index, expected] of fixture.positives.entries()) {for (const [cwe, applicableId] of [["CWE-256", "dotenv-url"],@@ -349,7 +339,7 @@ test("keeps password and cryptographic-key CWEs specific to those exposures", ()["CWE-321", "private-key"],] as const) {if (expected.id === applicableId) continue;- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[index].taxonomy.cwe = [cwe];const report = gradeResult(result, fixture);assert.equal(report.passed, false, `${expected.id}:${cwe}`);@@ -360,7 +350,7 @@ test("keeps password and cryptographic-key CWEs specific to those exposures", ()for (const cwe of ["CWE-200", "CWE-540"]) {test(`accepts ${cwe} for specifically categorized credential exposure`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();for (const finding of result.findings) finding.taxonomy.cwe = [cwe];assert.equal(gradeResult(result, fixture).passed, true);@@ -373,7 +363,7 @@ for (const cwe of ["CWE-200", "CWE-540"]) {for (const negativeIndex of [0, 1, 2, 3]) {test(`rejects a finding against negative control ${negativeIndex}`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();result.findings.push({taxonomy: { category: "hardcoded-credentials", cwe: ["CWE-798"] },locations: [@@ -399,13 +389,12 @@ for (const role of ["expected_control",]) {test(`benign ${role} context does not become a false positive`, () => {- const fixture = createFixture();for (const path of [fixture.negatives[0],`./${fixture.negatives[0]}`,`.\\${fixture.negatives[0].replaceAll("/", "\\")}`,]) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].locations.push({path,startLine: 1,@@ -419,7 +408,6 @@ for (const role of [});test(`rejects invalid ${role} source citations`, () => {- const fixture = createFixture();for (const location of [{ path: "src/absent.py", startLine: 999 },{ path: "src/runtime_config.py", startLine: 0 },@@ -428,7 +416,7 @@ for (const role of [{ path: "src/runtime_config.py", startLine: 3, endLine: 1 },{ path: "src/runtime_config.py", startLine: 1, endLine: 999 },]) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].locations.push({ ...location, role });const report = gradeResult(result, fixture);assert.equal(report.recall, 1);@@ -445,7 +433,7 @@ for (const [name, location] of [["negative control", { path: "src/runtime_config.py", startLine: 1 }],] as const) {test(`rejects an additional non-supporting location at ${name}`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();result.findings[0].locations.push({ ...location, role: "root_control" });const report = gradeResult(result, fixture);assert.equal(report.recall, 1);@@ -456,7 +444,7 @@ for (const [name, location] of [for (const role of ["root_control", "sink"]) {test(`rejects a ${role} citation classified only by a different finding`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();const finding = result.findings[0];finding.taxonomy.cwe = ["CWE-522"];const key = fixture.positives.find(@@ -481,7 +469,6 @@ for (const role of ["root_control", "sink"]) {}test("correct finding locations cannot hide missing or unrelated code evidence", () => {- const fixture = createFixture();for (const evidence of [undefined,[],@@ -489,7 +476,7 @@ test("correct finding locations cannot hide missing or unrelated code evidence",[sourceEvidence(fixture, "src/runtime_config.py", 3)],[{ ...sourceEvidence(fixture, "src/client.py", 3), code: " \n" }],]) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].codeEvidence = evidence as CodeEvidence[];const report = gradeResult(result, fixture);assert.equal(report.recall, 1);@@ -499,7 +486,6 @@ test("correct finding locations cannot hide missing or unrelated code evidence",});test("rejects fabricated source text at otherwise valid evidence locations", () => {- const fixture = createFixture();for (const code of ["invented source",sourceEvidence(fixture, "src/client.py", 1).code,@@ -508,7 +494,7 @@ test("rejects fabricated source text at otherwise valid evidence locations", ()"OTHER_TOKEN",),]) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].codeEvidence[0].code = code;const report = gradeResult(result, fixture);assert.equal(report.recall, 1);@@ -516,7 +502,7 @@ test("rejects fabricated source text at otherwise valid evidence locations", ()assert.deepEqual(report.errors, ["missing or invalid code evidence"]);}- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].codeEvidence.push({...sourceEvidence(fixture, "README.md", 1),code: "invented supporting context",@@ -527,14 +513,13 @@ test("rejects fabricated source text at otherwise valid evidence locations", ()});test("rejects invented and out-of-bounds citations alongside valid evidence", () => {- const fixture = createFixture();for (const citation of [{ path: "src/absent.py", startLine: 1 },{ path: "src/client.py", startLine: 0 },{ path: "src/client.py", startLine: 999 },{ path: "src/client.py", startLine: 7, code: "first line\nsecond line" },]) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].codeEvidence.push({...sourceEvidence(fixture, "src/client.py", 3),...citation,@@ -546,12 +531,11 @@ test("rejects invented and out-of-bounds citations alongside valid evidence", ()});test("accepts multiline exposure and consumer excerpts with supporting context", () => {- const fixture = createFixture();for (const [startLine, endLine] of [[1, 3],[4, 7],] as const) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].codeEvidence = [sourceEvidence(fixture, "src/client.py", startLine, endLine),sourceEvidence(fixture, "src/runtime_config.py", 3),@@ -564,7 +548,6 @@ test("accepts multiline exposure and consumer excerpts with supporting context",});test("consumer evidence and sinks must cover credential use, not function framing", () => {- const fixture = createFixture();for (const [id, line] of [["active-source", 5],["active-source", 6],@@ -575,7 +558,7 @@ test("consumer evidence and sinks must cover credential use, not function framin] as const) {const index = fixture.positives.findIndex((expected) => expected.id === id);const expected = fixture.positives[index];- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[index].codeEvidence = [sourceEvidence(fixture, expected.path, line),];@@ -585,7 +568,7 @@ test("consumer evidence and sinks must cover credential use, not function framin`${id}:${line}`,);- const invalidSink = retainedResult(fixture);+ const invalidSink = retainedResult();invalidSink.findings[index].locations.push({path: expected.path,startLine: line,@@ -600,14 +583,13 @@ test("consumer evidence and sinks must cover credential use, not function framin});test("accepts actual credential-use lines and excerpts containing them", () => {- const fixture = createFixture();for (const [index, expected] of fixture.positives.entries()) {if (expected.consumerLine === null) continue;for (const [startLine, endLine] of [[expected.consumerLine, expected.consumerLine],[5, expected.lineCount],] as const) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[index].codeEvidence = [sourceEvidence(fixture, expected.path, startLine, endLine),];@@ -623,7 +605,7 @@ test("accepts actual credential-use lines and excerpts containing them", () => {});test("multiple expected exposures can share a finding", () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();const unused = result.findings.splice(1, 1)[0];result.findings[0].locations.push(...unused.locations);assert.deepEqual(gradeResult(result, fixture).errors, [@@ -634,7 +616,7 @@ test("multiple expected exposures can share a finding", () => {});test("grouped exposures retain their applicable CWE classifications", () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();const indices = ["dotenv-url", "private-key"].map((id) =>fixture.positives.findIndex((expected) => expected.id === id),);@@ -663,7 +645,7 @@ test("grouped exposures retain their applicable CWE classifications", () => {for (const role of ["supporting", "expected_control"]) {test(`${role} citations cannot satisfy an independently missed exposure`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();const unused = result.findings.splice(1, 1)[0];result.findings[0].locations.push({...unused.locations[0],@@ -676,7 +658,7 @@ for (const role of ["supporting", "expected_control"]) {}test("accepts real credential-use sinks without treating them as separate exposures", () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();for (const [index, expected] of fixture.positives.entries()) {if (expected.consumerLine === null) continue;const sink = {@@ -686,7 +668,7 @@ test("accepts real credential-use sinks without treating them as separate exposurole: "sink",};result.findings[index].locations.push(sink);- const onlyConsumer = retainedResult(fixture);+ const onlyConsumer = retainedResult();onlyConsumer.findings[index].locations = [sink];assert.equal(gradeResult(onlyConsumer, fixture).cases[index].found, false);}@@ -697,7 +679,7 @@ test("accepts real credential-use sinks without treating them as separate exposu{ path: "src/absent.py", startLine: 5 },{ path: "src/client.py", startLine: 1 },]) {- const invalid = retainedResult(fixture);+ const invalid = retainedResult();invalid.findings[0].locations.push({ ...location, role: "sink" });assert.equal(gradeResult(invalid, fixture).falsePositiveCount, 1);}@@ -708,9 +690,8 @@ for (const [name, separator] of [["Windows", "\\"],] as const) {test(`accepts ${name} source and credential-use paths with relative prefixes`, () => {- const fixture = createFixture();for (const prefix of ["", "./", "././", ".\\", ".\\./", "./.\\"]) {- const result = retainedResult(fixture);+ const result = retainedResult();for (const [index, expected] of fixture.positives.entries()) {const path = `${prefix}${expected.path.replaceAll("/", separator)}`;result.findings[index].locations[0].path = path;@@ -730,7 +711,6 @@ for (const [name, separator] of [}test("path normalization does not make unrelated or unsafe source locations valid", () => {- const fixture = createFixture();for (const path of ["./src/absent.py","./src/../src/client.py",@@ -740,7 +720,7 @@ test("path normalization does not make unrelated or unsafe source locations vali"\\src\\client.py","C:\\src\\client.py",]) {- const result = retainedResult(fixture);+ const result = retainedResult();result.findings[0].locations[0].path = path;const report = gradeResult(result, fixture);assert.equal(report.cases[0].found, false, path);@@ -750,7 +730,7 @@ test("path normalization does not make unrelated or unsafe source locations vali});test("accepts the private-key body and rejects reversed source ranges", () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();const keyIndex = fixture.positives.findIndex((entry) => entry.id === "private-key",);@@ -763,7 +743,7 @@ test("accepts the private-key body and rejects reversed source ranges", () => {for (const footerLocation of [true, false]) {test(`rejects footer-only private-key evidence with ${footerLocation ? "footer" : "declaration"} location`, () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();const keyIndex = fixture.positives.findIndex((entry) => entry.id === "private-key",);@@ -784,7 +764,7 @@ for (const footerLocation of [true, false]) {}test("accepts public-key context alongside credential evidence", () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();result.findings[0].codeEvidence.push(sourceEvidence(fixture, "src/signing.mjs", 2, 4),sourceEvidence(fixture, "config/public.pem", 1, 3),@@ -793,7 +773,7 @@ test("accepts public-key context alongside credential evidence", () => {});test("scope notes neither fail correct findings nor compensate for missing findings", () => {- const { fixture, result } = retainedFixture();+ const result = retainedResult();Object.assign(result.coverage, {completeness: "partial",explicitExclusions: [diff --git a/evals/secret-discovery/test_runtime.mts b/evals/secret-discovery/test_runtime.mtsindex 302e3e715..bfb178497 100644--- a/evals/secret-discovery/test_runtime.mts+++ b/evals/secret-discovery/test_runtime.mts@@ -289,11 +289,9 @@ async function nativeFixture(await mkdir(dirname(executable), { recursive: true });const tools = join(dirname(dirname(executable)), "codex-path");await mkdir(tools);- await writeFile(- join(tools, "rg"),- `#!${process.execPath}\nconsole.log("synthetic rg");\n`,- { mode: 0o755 },- );+ await writeFile(join(tools, "rg"), "#!/bin/sh\necho synthetic rg\n", {+ mode: 0o755,+ });const result = {findings: prepared.fixture.positives.map((expected) => ({taxonomy: { category: "hardcoded-credentials", cwe: [expected.cwes[0]] },diff --git a/evals/triage-finding/package.json b/evals/triage-finding/package.jsonindex 648abad5c..9e706e116 100644--- a/evals/triage-finding/package.json+++ b/evals/triage-finding/package.json@@ -14,9 +14,9 @@"test:calibration-generate": "node --experimental-strip-types scripts/test-generate-calibration-tests.mts","test:calibration-hydrate": "node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts","test:dataset": "node --experimental-strip-types scripts/test-calibration-dataset.mts",- "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-intake-assertions.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-promptfoo-runtime.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts && node --experimental-strip-types --test scripts/test-promptfoo-runner.mts scripts/test-triage-provider.mts",+ "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-intake-assertions.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-promptfoo-runtime.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts && node --experimental-strip-types --test scripts/test-promptfoo-runner.mts scripts/test-triage-provider.mts","test:instructions": "node --experimental-strip-types scripts/test-app-surface-instructions.mts",- "test:sastbench": "node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts",+ "test:sastbench": "node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts","validate": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./promptfooconfig.yaml","validate:calibration": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./promptfooconfig.calibration.yaml","validate:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./sastbench/promptfooconfig.sastbench.yaml",diff --git a/evals/triage-finding/sastbench/scripts/test-hydrate-sastbench-repos.mts b/evals/triage-finding/sastbench/scripts/test-hydrate-sastbench-repos.mtsdeleted file mode 100644index b09314472..000000000--- a/evals/triage-finding/sastbench/scripts/test-hydrate-sastbench-repos.mts+++ /dev/null@@ -1,68 +0,0 @@-#!/usr/bin/env node-import assert from "node:assert/strict";-import path from "node:path";--import { buildHydrationPlan, parseArgs } from "./hydrate-sastbench-repos.mts";--const repoUrl = "https://github.com/example/project";-const firstCommit = "1".repeat(40);-const secondCommit = "2".repeat(40);-const baseRecord = {- repo_name: "example/project",- repo_url: repoUrl,- commit_hash: firstCommit,- ground_truth: "true_positive",-};-const records = [- { ...baseRecord },- { ...baseRecord, ground_truth: "false_positive" },- { ...baseRecord, commit_hash: secondCommit },-];-const cacheRoot = path.resolve("/tmp/sastbench-git-cache");-const targetRoot = path.resolve("/tmp/sastbench-targets");--assert.equal(parseArgs(["--", "--dry-run"]).dryRun, true);--const jobs = buildHydrationPlan(records, { cacheRoot, targetRoot });-assert.equal(- buildHydrationPlan(records, { cacheRoot, targetRoot })[0].cacheDir,- jobs[0].cacheDir,-);-assert.equal(jobs.length, 2);-assert.equal(jobs[0].caseCount, 2);-assert.equal(jobs[1].caseCount, 1);-assert.equal(- jobs[0].cacheDir,- path.join(cacheRoot, "repo-acb9cf7430784a93.git"),-);-for (const job of jobs) {- assert.equal(- path.relative(targetRoot, job.targetDir).startsWith(".."),- false,- );- assert.equal(path.relative(cacheRoot, job.cacheDir).startsWith(".."), false);- assert.equal(job.repoUrl, repoUrl);-}--assert.throws(- () =>- buildHydrationPlan(- [- {- ...baseRecord,- repo_name: "same/name",- repo_url: "https://github.com/example/one",- },- {- ...baseRecord,- repo_name: "same/name",- repo_url: "https://github.com/example/two",- commit_hash: secondCommit,- },- ],- { cacheRoot, targetRoot },- ),- /maps to multiple repository URLs/,-);--console.log("sastbench hydration planning tests passed");diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mtsindex fb124b869..69e85e00e 100644--- a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts@@ -1,7 +1,7 @@#!/usr/bin/env nodeimport assert from "node:assert/strict";import path from "node:path";-import { buildHydrationPlan } from "./hydrate-sastbench-repos.mts";+import { buildHydrationPlan, parseArgs } from "./hydrate-sastbench-repos.mts";import {REPRESENTATIVE_SAMPLE_SPEC,@@ -119,15 +119,36 @@ assert.equal();const stateId = repositoryStateId(positive);-assert.match(stateId, /^state-[a-f0-9]{16}$/);assert.equal(repositoryStateId(negative), stateId);-const jobs = buildHydrationPlan([positive, negative], { targetRoot });-assert.equal(jobs.length, 1);-assert.equal(jobs[0].caseCount, 2);-assert.equal(jobs[0].repoUrl, positive.repo_url);-assert.equal(jobs[0].commitHash, positive.commit_hash);-assert.equal(jobs[0].targetDir, path.join(targetRoot, stateId));+assert.equal(parseArgs(["--", "--dry-run"]).dryRun, true);+const cacheRoot = path.resolve("/tmp/sastbench-git-cache");+const otherRevision = { ...positive, commit_hash: "2".repeat(40) };+const jobs = buildHydrationPlan([positive, negative, otherRevision], {+ cacheRoot,+ targetRoot,+});+assert.deepEqual(+ jobs.map((job) => job.caseCount),+ [2, 1],+);+for (const [index, record] of [positive, otherRevision].entries()) {+ const job = jobs[index]!;+ assert.equal(job.repoUrl, record.repo_url);+ assert.equal(job.commitHash, record.commit_hash);+ const target = repositoryStateId(record);+ assert.match(target, /^state-[a-f0-9]{16}$/);+ assert.equal(job.targetDir, path.join(targetRoot, target));+ assert.equal(job.cacheDir, path.join(cacheRoot, "repo-acb9cf7430784a93.git"));+}+assert.throws(+ () =>+ buildHydrationPlan([+ positive,+ { ...otherRevision, repo_url: "https://github.com/example/other" },+ ]),+ /maps to multiple repository URLs/,+);const promptVars = buildPromptVars(positive, 0, targetRoot);assert.equal(promptVars.case_id, "sastbench-000000");diff --git a/evals/triage-finding/scripts/test-promptfoo-runner.mts b/evals/triage-finding/scripts/test-promptfoo-runner.mtsindex aeb619421..39f19cb0d 100644--- a/evals/triage-finding/scripts/test-promptfoo-runner.mts+++ b/evals/triage-finding/scripts/test-promptfoo-runner.mts@@ -1,8 +1,9 @@import assert from "node:assert/strict";-import { spawn } from "node:child_process";+import { spawn, type ChildProcess } from "node:child_process";+import { once } from "node:events";import fs from "node:fs";import path from "node:path";-import test from "node:test";+import test, { type TestContext } from "node:test";import { pathToFileURL } from "node:url";import { stageSkillRuntime } from "./run-promptfoo.mts";@@ -88,6 +89,43 @@ test("initializes native providers only in the Promptfoo child", async () => {}});+async function interruptRunner(+ t: TestContext,+ child: ChildProcess,+ interrupt: () => void,+ exitCode: number,+) {+ t.after(() => {+ if (child.exitCode === null) child.kill("SIGKILL");+ });+ let stderr = "";+ child.stderr!.on("data", (chunk) => {+ stderr += chunk;+ });+ const exited = once(child, "exit");+ const outputEnded = once(child.stdout!, "end");+ let output = "";+ const ready = new Promise<{ root: string }>((resolve) => {+ child.stdout!.on("data", (chunk) => {+ output += chunk;+ if (output.includes("\n")) resolve(JSON.parse(output.split("\n")[0]));+ });+ });+ const { root } = await Promise.race([+ ready,+ exited.then(() => {+ throw new Error(`runner exited early: ${stderr}`);+ }),+ ]);+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ assert.ok(fs.existsSync(root));+ interrupt();+ assert.deepEqual(await exited, [exitCode, null], stderr);+ assert.equal(fs.existsSync(root), false);+ await outputEnded;+ return output;+}+test("Windows console Ctrl+C lets the child save progress before runtime cleanup",{ timeout: 15000 },@@ -137,112 +175,64 @@ test(["--experimental-strip-types", "--eval", driver],{ stdio: ["ignore", "pipe", "pipe", "ipc"] },);- t.after(() => {- if (child.exitCode === null) child.kill("SIGKILL");- });- let stderr = "";- child.stderr!.on("data", (chunk) => {- stderr += chunk;- });- const exited = new Promise<number | null>((resolve, reject) => {- child.once("error", reject);- child.once("close", resolve);- });- let output = "";- const ready = new Promise<{ root: string }>((resolve) => {- child.stdout!.on("data", (chunk) => {- output += chunk;- if (output.includes("\n")) resolve(JSON.parse(output.split("\n")[0]));- });- });- const { root } = await Promise.race([- ready,- exited.then(() => {- throw new Error(`runner exited early: ${stderr}`);- }),- ]);- t.after(() => fs.rmSync(root, { recursive: true, force: true }));- assert.ok(fs.existsSync(root));- child.send("console-interrupt");- assert.equal(await exited, 130, stderr);+ const output = await interruptRunner(+ t,+ child,+ () => child.send("console-interrupt"),+ 130,+ );assert.match(output, /saved:1:true/);assert.doesNotMatch(output, /forced:/);- assert.equal(fs.existsSync(root), false);},);-for (const [signal, exitCode] of [- ["SIGINT", 130],- ["SIGTERM", 143],+for (const [signal, exitCode, delivery] of [+ ["SIGINT", 130, "pid"],+ ["SIGINT", 130, "group"],+ ["SIGTERM", 143, "pid"],+ ["SIGTERM", 143, "group"],] as const) {- for (const delivery of ["pid", "group"]) {- test(- `runner forwards ${signal} once (${delivery}) and removes its runtime after the child exits`,- { skip: process.platform === "win32", timeout: 15000 },- async (t) => {- const target = `- let received = 0;- process.on(${JSON.stringify(signal)}, () => {- received++;- if (received > 1) process.exit(99);- setTimeout(() => { console.log("graceful:" + received); process.exit(0); }, 150);- });- console.log(JSON.stringify({ root: process.env.TRIAGE_RUNTIME_ROOT }));- setInterval(() => {}, 1000);- `;- const driver = `- const cp = require('node:child_process');- const spawn = cp.spawn;- cp.spawn = (command, args, options) => spawn(process.execPath, ['--eval', args[0].endsWith("build_mcp_app.mjs") ? "" : ${JSON.stringify(target)}], options);- import(${JSON.stringify(new URL("./run-promptfoo.mts", import.meta.url).href)}).then(({ runPromptfoo }) => runPromptfoo([])).then(code => { process.exitCode = code; });+ test(+ `runner forwards ${signal} once (${delivery}) and removes its runtime after the child exits`,+ { skip: process.platform === "win32", timeout: 15000 },+ async (t) => {+ const target = `+ let received = 0;+ process.on(${JSON.stringify(signal)}, () => {+ received++;+ if (received > 1) process.exit(99);+ setTimeout(() => {+ console.log("graceful:" + received + ":" + require("node:fs").existsSync(process.env.TRIAGE_RUNTIME_ROOT));+ process.exit(0);+ }, 150);+ });+ console.log(JSON.stringify({ root: process.env.TRIAGE_RUNTIME_ROOT }));+ setInterval(() => {}, 1000);`;- const child = spawn(- process.execPath,- ["--experimental-strip-types", "--eval", driver],- {- stdio: ["ignore", "pipe", "pipe"],- detached: true,- },- );- t.after(() => {- if (child.exitCode === null) child.kill("SIGKILL");- });- let stderr = "";- child.stderr.on("data", (chunk) => {- stderr += chunk;- });- const exited = new Promise<{- code: number | null;- signal: NodeJS.Signals | null;- }>((resolve, reject) => {- child.once("error", reject);- child.once("exit", (code, signal) => resolve({ code, signal }));- });- let text = "";- const ready = new Promise<{ root: string }>((resolve) => {- child.stdout.on("data", (chunk) => {- text += chunk;- if (text.includes("\n")) resolve(JSON.parse(text.split("\n")[0]));- });- });- const { root } = await Promise.race([- ready,- exited.then(() => {- throw new Error(`runner exited early: ${stderr}`);- }),- ]);- t.after(() => fs.rmSync(root, { recursive: true, force: true }));- assert.ok(fs.existsSync(root));- if (delivery === "group") process.kill(-child.pid!, signal);- else child.kill(signal);- assert.deepEqual(- await exited,- { code: exitCode, signal: null },- stderr,- );- assert.match(text, /graceful:1/);- assert.equal(fs.existsSync(root), false);- },- );- }+ const driver = `+ const cp = require('node:child_process');+ const spawn = cp.spawn;+ cp.spawn = (command, args, options) => spawn(process.execPath, ['--eval', args[0].endsWith("build_mcp_app.mjs") ? "" : ${JSON.stringify(target)}], options);+ import(${JSON.stringify(new URL("./run-promptfoo.mts", import.meta.url).href)}).then(({ runPromptfoo }) => runPromptfoo([])).then(code => { process.exitCode = code; });+ `;+ const child = spawn(+ process.execPath,+ ["--experimental-strip-types", "--eval", driver],+ {+ stdio: ["ignore", "pipe", "pipe"],+ detached: true,+ },+ );+ const output = await interruptRunner(+ t,+ child,+ () => {+ if (delivery === "group") process.kill(-child.pid!, signal);+ else child.kill(signal);+ },+ exitCode,+ );+ assert.match(output, /graceful:1:true/);+ },+ );}diff --git a/evals/triage-finding/scripts/test-triage-provider.mts b/evals/triage-finding/scripts/test-triage-provider.mtsindex 35520e993..b7cd3a93c 100644--- a/evals/triage-finding/scripts/test-triage-provider.mts+++ b/evals/triage-finding/scripts/test-triage-provider.mts@@ -49,6 +49,71 @@ fs.writeFileSync(path.join(fixture, 'SECURITY.md'), '# Synthetic policy\\n');const policy = require('node:child_process').execFileSync(launcher, ['--helper', 'resolve-security-md', '--repo', fixture, '--scope', 'src/server.js', '--out', '-'], {encoding:'utf8'});`;+async function replayInViewer(+ t: TestContext,+ environment: NodeJS.ProcessEnv,+ request: {+ evaluationId: unknown;+ prompt: string;+ variables: Record<string, unknown>;+ },+) {+ const listener = createServer();+ await new Promise<void>((resolve) =>+ listener.listen(0, "127.0.0.1", resolve),+ );+ const address = listener.address();+ assert.ok(address && typeof address !== "string");+ const port = address.port;+ await new Promise<void>((resolve, reject) =>+ listener.close((error) => (error ? reject(error) : resolve())),+ );+ const viewer = spawn(+ process.execPath,+ [...replayRunnerArgs(), "view", "--port", String(port), "--no"],+ {+ cwd: evalRoot,+ env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },+ stdio: ["ignore", "pipe", "pipe"],+ },+ );+ let viewerOutput = "";+ for (const stream of [viewer.stdout, viewer.stderr])+ stream.on("data", (chunk) => {+ viewerOutput += chunk;+ });+ const closed = new Promise<number | null>((resolve, reject) => {+ viewer.once("error", reject);+ viewer.once("close", resolve);+ });+ t.after(async () => {+ if (viewer.exitCode === null) viewer.kill("SIGTERM");+ await closed;+ });+ const url = `http://127.0.0.1:${port}`;+ while (true) {+ try {+ const response = await fetch(`${url}/api/eval`);+ if (response.ok) break;+ } catch {+ /* The actual viewer is still building its staged runtime. */+ }+ assert.equal(viewer.exitCode, null, viewerOutput);+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ const replay = await fetch(`${url}/api/eval/replay`, {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({ testIndex: 0, ...request }),+ });+ assert.equal(replay.status, 200, viewerOutput);+ const replayResult = (await replay.json()) as { output: unknown };+ assert.equal(replayResult.output, "ok", JSON.stringify(replayResult));+ viewer.kill("SIGTERM");+ await closed;+ return viewerOutput;+}+function providerConfig(name: string) {return parse(fs.readFileSync(path.join(evalRoot, "providers", `${name}.yaml`), "utf8"),@@ -1191,63 +1256,13 @@ if (fs.existsSync(${JSON.stringify(failure)})) {const replayVariables =JSON.parse((resumedResult.metadata as string) || "{}").inputVars ||JSON.parse(resumedResult.test_case).vars;- const listener = createServer();- await new Promise<void>((resolve) =>- listener.listen(0, "127.0.0.1", resolve),- );- const address = listener.address();- assert.ok(address && typeof address !== "string");- const port = address.port;- await new Promise<void>((resolve, reject) =>- listener.close((error) => (error ? reject(error) : resolve())),- );- const viewer = spawn(- process.execPath,- [...replayRunnerArgs(), "view", "--port", String(port), "--no"],- {- cwd: evalRoot,- env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },- stdio: ["ignore", "pipe", "pipe"],- },- );- let viewerOutput = "";- for (const stream of [viewer.stdout, viewer.stderr])- stream.on("data", (chunk) => {- viewerOutput += chunk;- });- const closed = new Promise<number | null>((resolve, reject) => {- viewer.once("error", reject);- viewer.once("close", resolve);- });- t.after(async () => {- if (viewer.exitCode === null) viewer.kill("SIGTERM");- await closed;+ await replayInViewer(t, environment, {+ evaluationId: stored.id,+ prompt:+ "Read and follow skills/triage-finding/SKILL.md.\nTARGET=" ++ replayVariables.target_repo,+ variables: replayVariables,});- const url = `http://127.0.0.1:${port}`;- while (true) {- try {- if ((await fetch(`${url}/api/eval`)).ok) break;- } catch {}- assert.equal(viewer.exitCode, null, viewerOutput);- await new Promise((resolve) => setTimeout(resolve, 50));- }- const replay = await fetch(`${url}/api/eval/replay`, {- method: "POST",- headers: { "content-type": "application/json" },- body: JSON.stringify({- evaluationId: stored.id,- testIndex: 0,- prompt:- "Read and follow skills/triage-finding/SKILL.md.\nTARGET=" +- replayVariables.target_repo,- variables: replayVariables,- }),- });- assert.equal(replay.status, 200, viewerOutput);- const replayResult = (await replay.json()) as { output: unknown };- assert.equal(replayResult.output, "ok", JSON.stringify(replayResult));- viewer.kill("SIGTERM");- await closed;const rows = fs.readFileSync(capture, "utf8").trim()@@ -1512,64 +1527,11 @@ if (fs.existsSync(${JSON.stringify(failure)})) {.inputVars || JSON.parse(resumedResult.test_case).vars;assert.equal(typeof replayVariables.target_repo, "string");- const listener = createServer();- await new Promise<void>((resolve) =>- listener.listen(0, "127.0.0.1", resolve),- );- const address = listener.address();- assert.ok(address && typeof address !== "string");- const port = address.port;- await new Promise<void>((resolve, reject) =>- listener.close((error) => (error ? reject(error) : resolve())),- );- const viewer = spawn(- process.execPath,- [...replayRunnerArgs(), "view", "--port", String(port), "--no"],- {- cwd: evalRoot,- env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },- stdio: ["ignore", "pipe", "pipe"],- },- );- let viewerOutput = "";- for (const stream of [viewer.stdout, viewer.stderr])- stream.on("data", (chunk) => {- viewerOutput += chunk;- });- const closed = new Promise<number | null>((resolve, reject) => {- viewer.once("error", reject);- viewer.once("close", resolve);- });- t.after(async () => {- if (viewer.exitCode === null) viewer.kill("SIGTERM");- await closed;- });- const url = `http://127.0.0.1:${port}`;- while (true) {- try {- const response = await fetch(`${url}/api/eval`);- if (response.ok) break;- } catch {- /* The actual viewer is still building its staged runtime. */- }- assert.equal(viewer.exitCode, null, viewerOutput);- await new Promise((resolve) => setTimeout(resolve, 50));- }- const replay = await fetch(`${url}/api/eval/replay`, {- method: "POST",- headers: { "content-type": "application/json" },- body: JSON.stringify({- evaluationId: stored.id,- testIndex: 0,- prompt: "hello",- variables: replayVariables,- }),+ const viewerOutput = await replayInViewer(t, environment, {+ evaluationId: stored.id,+ prompt: "hello",+ variables: replayVariables,});- assert.equal(replay.status, 200, viewerOutput);- const replayResult = (await replay.json()) as { output: unknown };- assert.equal(replayResult.output, "ok", JSON.stringify(replayResult));- viewer.kill("SIGTERM");- await closed;assert.doesNotMatch(viewerOutput, /afterEach extension hook failed/);const rows = fs.readFileSync(capture, "utf8")diff --git a/github-action/tests/main.test.ts b/github-action/tests/main.test.tsindex 439a94554..fbaf399d3 100644--- a/github-action/tests/main.test.ts+++ b/github-action/tests/main.test.ts@@ -55,11 +55,12 @@ async function harness(t: TestContext, scenario: Scenario = 'schedule', serverUrawait writeFile(join(repository, 'app.txt'), 'synthetic source\n');git('add', '.'); git('commit', '-qm', 'base');const base = git('rev-parse', 'HEAD');+ let sha = base;if (scenario !== 'schedule') {await writeFile(join(repository, scenario === 'policy-pr' ? 'SECURITY.md' : 'app.txt'), 'synthetic change\n');git('add', '.'); git('commit', '-qm', 'PR change');+ sha = git('rev-parse', 'HEAD');}- const sha = git('rev-parse', 'HEAD');const previousEnv = { ...process.env };const previousExitCode = process.exitCode;t.after(async () => {@@ -197,10 +198,18 @@ test('PR severity failure retains SARIF outputs despite unrelated scope notes',});test('scheduled complete report-only scan succeeds', async (t) => {- const app = await harness(t); const result = await app.run();+ const app = await harness(t);+ process.env.INPUT_FOO = 'unrelated workflow value';+ app.setInput('effort', 'medium');+ app.configure({scanResult: value => {+ value.turn = {usage: {input_tokens: 1200, output_tokens: 50}};+ }});+ const result = await app.run();assertAttribution(result.args, 'openai');+ assert.ok(result.args.includes('--effort=medium'));assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed');assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['sarif-upload-ready'], 'true');+ assert.equal(result.outputs['report-status'], 'ready'); assert.equal(result.cleanups, 1);assert.equal(result.outputs['analysis-ref'], 'refs/heads/main');assert.ok(result.args.includes('--verbose'));assert.match(result.logs, /Synthetic live CLI progress/);@@ -209,34 +218,12 @@ test('scheduled complete report-only scan succeeds', async (t) => {assert.match(result.logs, /Security scan exited after \d+m \d+s; exit code: 0/);assert.match(result.logs, /Scan: completed; findings policy: passed; report: ready/);assert.match(result.logs, /Estimated cost: \$0.1250/);+ assert.equal(result.outputs['estimated-cost'], '0.125');+ assert.ok(result.summary.includes('1,200 input (0 cached) · 50 output'));assert.match(result.logs, /Scan completed\. Findings are reported without failing the job\./);assert.match(result.summary, /\*\*Scan completed\. Findings are reported without failing the job\.\*\*/);});-test('usage without a cached count preserves a successful Action and its reports', async t => {- const app = await harness(t);- app.configure({scanResult: value => {- value.turn = {usage: {input_tokens: 1200, output_tokens: 50}};- }});- const result = await app.run();- assert.equal(result.exitCode, 0);- assert.equal(result.outputs['scan-status'], 'completed');- assert.equal(result.outputs['policy-status'], 'passed');- assert.equal(result.outputs['report-status'], 'ready');- assert.equal(result.outputs['sarif-upload-ready'], 'true');- assert.equal(result.cleanups, 1);- assert.ok(result.summary.includes('1,200 input (0 cached) · 50 output'));- assert.equal(result.outputs['estimated-cost'], '0.125');-});--test('unrelated ambient input variables do not reject a valid scan', async (t) => {- const app = await harness(t); process.env.INPUT_FOO = 'unrelated workflow value';- app.setInput('effort', 'medium');- const result = await app.run();- assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed');- assert.ok(result.args.includes('--effort=medium'));-});-for (const [scenario, mode] of [['schedule', 'standard'], ['pr', 'standard'], ['schedule', 'deep']] as const) {test(`enterprise ${scenario} ${mode} scans preserve reports and upload identity`, async t => {const serverUrl = 'https://example-enterprise.ghe.com';@@ -374,16 +361,6 @@ test('Bedrock dry-run forwards no credentials and remains a skipped scan', asyncassert.equal(result.environment.AWS_BEARER_TOKEN_BEDROCK, undefined); assert.equal(result.environment.OPENAI_API_KEY, undefined);});-test('Bedrock authentication errors remain scan failures', async t => {- const app = await harness(t);- app.setInput('provider', 'amazon-bedrock'); app.setInput('model', 'openai.synthetic-model');- delete process.env.OPENAI_API_KEY;- app.configure({exitCode:2, cliFailure:true});- const result = await app.run();- assert.equal(result.exitCode, 1); assert.equal(result.processes, 1);- assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false');-});-for (const scenario of ['schedule', 'pr'] as const) {test(`missing API key fails a ${scenario === 'pr' ? 'Dependabot PR' : 'scheduled scan'} with setup guidance`, async (t) => {const app = await harness(t, scenario); delete process.env.OPENAI_API_KEY;@@ -428,15 +405,6 @@ for (const threshold of ['none', 'high']) {});}-test('verbose false suppresses CLI diagnostics but retains lifecycle and results', async (t) => {- const app = await harness(t); app.setInput('verbose', 'false'); const result = await app.run();- assert.equal(result.exitCode, 0);- assert.ok(!result.args.includes('--verbose'));- assert.doesNotMatch(result.logs, /Synthetic live CLI progress/);- assert.match(result.logs, /Starting security scan/);- assert.match(result.logs, /Scan: completed/);-});-test('configuration diagnostics preserve values and cannot inject runner commands', async (t) => {const app = await harness(t);app.setInput('model', 'synthetic-offline-test-key\u2028::error::injected');@@ -459,6 +427,10 @@ test('scope exclusions and deferred notes do not fail completed scans or make fiassert.equal(result.outputs['exit-code'], '0');assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['sarif-upload-ready'], 'true');assert.equal(result.outputs['high-count'], '1'); assert.ok(result.outputs['json-path']);+ assert.ok(!result.args.includes('--verbose'));+ assert.doesNotMatch(result.logs, /Synthetic live CLI progress/);+ assert.match(result.logs, /Starting security scan/);+ assert.match(result.logs, /Scan: completed/);assert.match(result.logs, /Findings:.*high: 1/);assert.doesNotMatch(result.logs, /Provisional|Scan coverage is partial|::error::/);assert.doesNotMatch(result.summary, /provisional|not evaluated|not-evaluated/i);@@ -466,13 +438,15 @@ test('scope exclusions and deferred notes do not fail completed scans or make fiassert.equal(coverage.deferred[0].reason, 'Dependency <example> unavailable; validation deferred.');});-test('execution failure diagnostics cannot emit legacy runner commands', async (t) => {+test('CLI warnings fail a completed scan and cannot emit legacy runner commands', async (t) => {const app = await harness(t);- app.configure({exitCode: 2, scanResult: value => {+ app.configure({scanResult: value => {value.warnings = ['Synthetic diagnostic: ##[stop-commands]token'];}});const result = await app.run();- assert.equal(result.exitCode, 1);+ assert.equal(result.exitCode, 1); assert.equal(result.outputs['exit-code'], '0');+ assert.equal(result.outputs['policy-status'], 'not-evaluated');+ assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false');assert.match(result.logs, /Synthetic diagnostic: ##\\\[stop-commands\]token/);assert.doesNotMatch(result.logs, /##\[stop-commands\]/);assert.match(result.summary, /Synthetic diagnostic: ##[stop-commands]token/);@@ -492,8 +466,7 @@ test('exit 2 fails the step even when finalized reports were retained', async (tfor (const [name, scanProcess] of [['timeout', {timedOut: true}], ['interruption', {interrupted: true}],- ['signal', {signal: 'SIGTERM' as const}], ['terminated exit', {exitCode: 143}],- ['malformed output', {stdout: 'not JSON'}],+ ['signal', {signal: 'SIGTERM' as const}],] as const) {test(`retained reports cannot hide scan ${name}`, async (t) => {const app = await harness(t); app.configure({scanProcess});@@ -503,18 +476,6 @@ for (const [name, scanProcess] of [});}-for (const [name, scanResult] of [- ['failed manifest', (value: any) => { value.manifest.scan.status = 'failed'; }],- ['target-change warning', (value: any) => { value.warnings = ['Synthetic target changed during scanning.']; }],-] as const) {- test(`${name} cannot become a successful scan`, async (t) => {- const app = await harness(t); app.configure({scanResult});- const result = await app.run();- assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'failed');- assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['sarif-upload-ready'], 'false');- });-}-for (const missingReport of ['scan-manifest.json', 'findings.json', 'coverage.json']) {test(`completed scan fails when required ${missingReport} is missing without artifact upload`, async (t) => {const app = await harness(t); app.configure({missingReport});@@ -693,10 +654,13 @@ test('archive read failure preserves scan reports, summary and cleanup', {await assert.rejects(stat(staging), {code: 'ENOENT'});});-test('CLI authentication failure leaves findings unavailable instead of reporting a threshold failure', async (t) => {- const app = await harness(t); app.configure({exitCode: 2, cliFailure: true});+test('Bedrock CLI authentication failure leaves findings unavailable', async (t) => {+ const app = await harness(t);+ app.setInput('provider', 'amazon-bedrock'); app.setInput('model', 'openai.synthetic-model');+ delete process.env.OPENAI_API_KEY;+ app.configure({exitCode: 2, cliFailure: true});const result = await app.run();- assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'failed');+ assert.equal(result.exitCode, 1); assert.equal(result.processes, 1); assert.equal(result.outputs['scan-status'], 'failed');assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['sarif-upload-ready'], 'false');assert.equal(result.outputs['json-path'], '');assert.match(result.summary, /Synthetic API authentication failure/);@@ -706,13 +670,6 @@ test('CLI authentication failure leaves findings unavailable instead of reportinassert.match(result.logs, /Findings: unavailable/);});-test('malformed findings keep count outputs empty', async (t) => {- const app = await harness(t); app.configure({scanResult: value => { value.findings.findings[0].severity = null; }});- const result = await app.run();- assert.equal(result.exitCode, 1);- for (const level of ['critical', 'high', 'medium', 'low', 'informational']) assert.equal(result.outputs[`${level}-count`], '');-});-test('valid empty findings publish zero counts', async (t) => {const app = await harness(t); app.configure({scanResult: value => { value.findings.findings = []; }});const result = await app.run();diff --git a/github-action/tests/results.test.ts b/github-action/tests/results.test.tsindex 7ed82ea28..a86f5a9c5 100644--- a/github-action/tests/results.test.ts+++ b/github-action/tests/results.test.ts@@ -34,6 +34,7 @@ function change(options: ResultOptions, update: (value: any) => void): void {test('adapts a synthetic CLI JSON result into findings, counts and report outputs', async (t) => {const opts = await fixture(t);+ change(opts, (value) => { value.findings.findings[0].title = 'Title from the CLI result'; });const result = await analyzeResults(opts);assert.deepEqual(result.errors, []);assert.equal(result.scanStatus, 'completed');@@ -44,6 +45,7 @@ test('adapts a synthetic CLI JSON result into findings, counts and report outputassert.deepEqual(result.usage, { input_tokens: 1200, cached_input_tokens: 200, output_tokens: 50 });assert.equal(result.estimatedCost, 0.125);assert.deepEqual(result.estimatedCostRange, { min: 0.125, max: 0.25, context: 'unknown' });+ assert.equal(result.findings[0]?.title, 'Title from the CLI result');assert.equal(result.findings[0]?.path, 'src/extract.py');assert.equal(result.findings[0]?.startLine, 41);assert.equal(result.paths.jsonPath, join(opts.resultsDirectory, 'findings.json'));@@ -62,14 +64,6 @@ test('token usage remains available for unpriced models and distinguishes missin}});-test('uses the structured CLI findings instead of reparsing report documents', async (t) => {- const opts = await fixture(t);- change(opts, (value) => { value.findings.findings[0].title = 'Title from the CLI result'; });- const result = await analyzeResults(opts);- assert.equal(result.scanStatus, 'completed');- assert.equal(result.findings[0]?.title, 'Title from the CLI result');-});-test('annotations prefer the root control over an earlier entry point', async (t) => {const opts = await fixture(t);change(opts, (value) => {@@ -123,6 +117,7 @@ test('operational exits and signal termination cannot turn complete output intoassert.equal(result.policyStatus, 'not-evaluated');assert.equal(result.sarifUploadReady, false);assert.equal(result.findings.length, 1);+ assert.equal(result.counts?.high, 1);}});@@ -302,12 +297,10 @@ test('SARIF publication rejects external references and encoded traversal paths'});-test('valid empty and retained findings distinguish zero counts from unavailable results', async (t) => {+test('valid empty findings have zero counts', async (t) => {const opts = await fixture(t);change(opts, (value) => { value.findings.findings = []; });assert.deepEqual((await analyzeResults(opts)).counts, {critical: 0, high: 0, medium: 0, low: 0, informational: 0});- const retained = await fixture(t);- assert.equal((await analyzeResults({...retained, exitCode: 2})).counts?.high, 1);});test('accepted relative path spellings preserve completed findings and SARIF', async (t) => {