OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test: reuse triage replay builds and exercise native Action SARIF (#1977)
Michael D'Angelo3dfed19297d6
diff --git a/.github/scripts/invoice-desk-target.test.mjs b/.github/scripts/invoice-desk-target.test.mjsindex 237eef4ea..7a4be077a 100644--- a/.github/scripts/invoice-desk-target.test.mjs+++ b/.github/scripts/invoice-desk-target.test.mjs@@ -92,13 +92,29 @@ test("incomplete or invalid PR inputs fail instead of falling back to a baseline}});-test("ignores unexpected events without querying GitHub", async () => {- assert.deepEqual(- await resolveScanTargets({ ...context, eventName: "pull_request" }, () =>- assert.fail("unexpected API call"),- ),- [],- );+test("PR events and completed PR workflows do not queue inference or query GitHub", async () => {+ for (const trigger of [+ { eventName: "pull_request", event: { pull_request: pullRequest() } },+ {+ eventName: "workflow_run",+ event: {+ workflow_run: {+ event: "pull_request",+ conclusion: "success",+ head_sha: sourceSha,+ pull_requests: [{ number: 7 }],+ },+ },+ },+ ]) {+ assert.deepEqual(+ await resolveScanTargets({ ...context, ...trigger }, () =>+ assert.fail("unexpected API call"),+ ),+ [],+ trigger.eventName,+ );+ }});test("scheduled baseline scans use the trusted workflow commit", async () => {@@ -111,27 +127,6 @@ test("scheduled baseline scans use the trusted workflow commit", async () => {);});-test("completed behavior workflows no longer queue inference", async () => {- assert.deepEqual(- await resolveScanTargets(- {- ...context,- eventName: "workflow_run",- event: {- workflow_run: {- event: "pull_request",- conclusion: "success",- head_sha: sourceSha,- pull_requests: [{ number: 7 }],- },- },- },- () => assert.fail("unexpected API call"),- ),- [],- );-});-test("approval recheck stays bound to its matrix PR and source revision", async () => {const run = { ...context, target: { pr: 7, sha: sourceSha } };for (const changed of [diff --git a/evals/triage-finding/scripts/run-promptfoo.mts b/evals/triage-finding/scripts/run-promptfoo.mtsindex ea9bbf8e4..90339197c 100644--- a/evals/triage-finding/scripts/run-promptfoo.mts+++ b/evals/triage-finding/scripts/run-promptfoo.mts@@ -74,6 +74,7 @@ function stageSkillRuntime() {async function runPromptfoo(promptfooArgs: string[],environment: NodeJS.ProcessEnv = {},+ builtMcp?: string,) {const runtimeRoot = stageSkillRuntime();const artifacts = path.join(EVAL_ROOT, "artifacts");@@ -131,17 +132,22 @@ async function runPromptfoo(});}try {- const built = await runChild(- [- path.join(PLUGIN_ROOT, "mcp-app", "scripts", "build_mcp_app.mjs"),- "--output",- path.join(runtimeRoot, "plugins", "codex-security", "mcp"),- "--native",- "host",- ],- path.join(PLUGIN_ROOT, "mcp-app"),- );- if (built !== 0) return built;+ const mcp = path.join(runtimeRoot, "plugins", "codex-security", "mcp");+ if (builtMcp) {+ copyDirectory(builtMcp, mcp);+ } else {+ const built = await runChild(+ [+ path.join(PLUGIN_ROOT, "mcp-app", "scripts", "build_mcp_app.mjs"),+ "--output",+ mcp,+ "--native",+ "host",+ ],+ path.join(PLUGIN_ROOT, "mcp-app"),+ );+ if (built !== 0) return built;+ }return await runChild(["--import",diff --git a/evals/triage-finding/scripts/run-provider-fixture.mts b/evals/triage-finding/scripts/run-provider-fixture.mtsnew file mode 100644index 000000000..507ae8a3b--- /dev/null+++ b/evals/triage-finding/scripts/run-provider-fixture.mts@@ -0,0 +1,4 @@+import { runPromptfoo } from "./run-promptfoo.mts";++const [builtMcp, ...args] = process.argv.slice(2);+process.exitCode = await runPromptfoo(args, {}, builtMcp);diff --git a/evals/triage-finding/scripts/test-app-surface-instructions.mts b/evals/triage-finding/scripts/test-app-surface-instructions.mtsindex 38cf5602d..91d4e0969 100644--- a/evals/triage-finding/scripts/test-app-surface-instructions.mts+++ b/evals/triage-finding/scripts/test-app-surface-instructions.mts@@ -20,9 +20,12 @@ const agent = fs.readFileSync(agentPath, "utf8");const pluginPath = path.join(pluginRoot, ".codex-plugin", "plugin.json");const plugin = JSON.parse(fs.readFileSync(pluginPath, "utf8"));-assert.match(skill, /### Jira and Linear intake/);assert.match(skill, /references\/ticket-intake\.md/);assert.match(ticketIntake, /Atlassian[\s\S]*JQL/);+assert.match(+ ticketIntake,+ /\bsearchJiraIssuesUsingJql`?\s+for\s+collections\b|\bfor\s+collections,\s+use\s+`?searchJiraIssuesUsingJql\b/i,+);assert.match(ticketIntake, /natural-language search[\s\S]*discover[\s\S]*JQL/);assert.match(skill, /security or vulnerability Jira\/Linear tickets/);assert.match(skill, /Atlassian and Linear mentions are connector hints/);@@ -35,9 +38,24 @@ assert.match(ticketIntake,/issue key[\s\S]*URL[\s\S]*project[\s\S]*status[\s\S]*labels[\s\S]*components[\s\S]*priority/,);-assert.match(- ticketIntake,- /Default to read-only import and triage[\s\S]*Do not add comments, transition or close issues, assign owners, or change labels/,+const sentences = ticketIntake+ .split(/[.!?]|\n\s*\n|^#{1,6}[ \t]+.*$/m)+ .map((sentence) => sentence.trim().replace(/\s+/g, " "));+assert.ok(+ sentences.some((sentence) =>+ sentence.startsWith("Default to read-only import and triage"),+ ),+);+assert.ok(+ sentences.some(+ (sentence) =>+ sentence.startsWith(+ "Do not add comments, transition or close issues, assign owners, or change labels",+ ) &&+ /\b(?:without|unless)\b.*\bexplicit(?:ly)?\b.*\brequest(?:ed|s)?\b/i.test(+ sentence,+ ),+ ),);assert.match(agent, /default_prompt:.*Use \$triage-finding/);assert.match(ticketIntake, /missing connector|connector.*unavailable/i);@@ -49,7 +67,7 @@ assert.match(ticketIntake, /exhaust[\s\S]*pag(?:es|ination)/i);assert.match(ticketIntake, /independent vulnerability claim/i);assert.match(ticketIntake, /ambiguous[\s\S]*ask/i);assert.match(ticketIntake, /deterministic[\s\S]*tree order/i);-assert.equal(plugin.interface.defaultPrompt.length, 3);+assert.ok(plugin.interface.defaultPrompt.length <= 3);assert(plugin.interface.defaultPrompt.every((prompt: string) => [...prompt].length <= 128,diff --git a/evals/triage-finding/scripts/test-promptfoo-runtime.mts b/evals/triage-finding/scripts/test-promptfoo-runtime.mtsindex 27ad04d52..eb4ee2014 100644--- a/evals/triage-finding/scripts/test-promptfoo-runtime.mts+++ b/evals/triage-finding/scripts/test-promptfoo-runtime.mts@@ -3,7 +3,6 @@import assert from "node:assert/strict";import fs from "node:fs";import path from "node:path";-import { pathToFileURL } from "node:url";import { parse } from "yaml";import runtimeVars from "./runtime-vars.mts";@@ -162,15 +161,4 @@ assert.ok("Read and follow plugins/codex-security/skills/triage-finding/SKILL.md",),);-// Load the installed scanner so its Git imports are checked at runtime.-const promptfooDist = path.join(evalRoot, "node_modules/promptfoo/dist/src");-const scanner = fs- .readdirSync(promptfooDist)- .find((name) => name.startsWith("scanner-") && name.endsWith(".js"));-assert.ok(scanner, "Promptfoo scanner entrypoint must be installed");-const scannerModule = await import(- pathToFileURL(path.join(promptfooDist, scanner)).href-);-assert.equal(typeof scannerModule.executeScan, "function");-console.log("triage Promptfoo runtime tests passed");diff --git a/evals/triage-finding/scripts/test-triage-provider.mts b/evals/triage-finding/scripts/test-triage-provider.mtsindex 5c5eda611..efae11302 100644--- a/evals/triage-finding/scripts/test-triage-provider.mts+++ b/evals/triage-finding/scripts/test-triage-provider.mts@@ -1,16 +1,54 @@import assert from "node:assert/strict";-import { spawn } from "node:child_process";+import { execFileSync, spawn } from "node:child_process";import fs from "node:fs";import os from "node:os";import { createServer } from "node:net";import path from "node:path";-import test from "node:test";+import test, { after } from "node:test";import { DatabaseSync } from "node:sqlite";import { parse } from "yaml";import { beforeAll } from "./triage-provider.mts";const evalRoot = path.resolve(import.meta.dirname, "..");const runner = path.join(import.meta.dirname, "run-promptfoo.mts");+const fixtureRunner = path.join(+ import.meta.dirname,+ "run-provider-fixture.mts",+);+const fixtureRoot = fs.realpathSync(+ fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-bundle-")),+);+after(() => fs.rmSync(fixtureRoot, { recursive: true, force: true }));+let builtMcp: string | undefined;++function replayRunnerArgs() {+ if (!builtMcp) {+ const output = path.join(fixtureRoot, "mcp");+ const mcpApp = path.join(evalRoot, "../../plugins/codex-security/mcp-app");+ execFileSync(+ process.execPath,+ [+ path.join(mcpApp, "scripts/build_mcp_app.mjs"),+ "--output",+ output,+ "--native",+ "host",+ ],+ { cwd: mcpApp },+ );+ builtMcp = output;+ }+ // Every child copies the same build into its own throwaway skill runtime.+ return ["--experimental-strip-types", fixtureRunner, builtMcp];+}++const probeFixturePolicy = `+const launcher = path.join(cwd, 'plugins/codex-security/scripts/launch_codex_security_mcp');+const fixture = path.join(cwd, 'evals/triage-finding/fixtures/repo');+fs.writeFileSync(path.join(fixture, 'SECURITY.md'), '# Synthetic policy\\n');+const policy = require('node:child_process').execFileSync(launcher, ['--helper', 'resolve-security-md', '--repo', fixture, '--scope', 'src/server.js', '--out', '-'], {encoding:'utf8'});+`;+function providerConfig(name: string) {return parse(fs.readFileSync(path.join(evalRoot, "providers", `${name}.yaml`), "utf8"),@@ -36,18 +74,18 @@ const proxies = {NO_PROXY: "localhost,.example.test",};-function invoke(args: string[], environment: NodeJS.ProcessEnv) {+function invoke(+ args: string[],+ environment: NodeJS.ProcessEnv,+ runnerArgs = replayRunnerArgs(),+) {return new Promise<{ code: number | null; output: string }>((resolve, reject) => {- const child = spawn(- process.execPath,- ["--experimental-strip-types", runner, ...args],- {- cwd: evalRoot,- env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },- stdio: ["ignore", "pipe", "pipe"],- },- );+ const child = spawn(process.execPath, [...runnerArgs, ...args], {+ cwd: evalRoot,+ env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },+ stdio: ["ignore", "pipe", "pipe"],+ });let output = "";child.stdout.on("data", (chunk) => {output += chunk;@@ -191,7 +229,8 @@ const fs = require('node:fs');const path = require('node:path');const cwd = process.argv[process.argv.indexOf('--cd') + 1];fs.readFileSync(path.join(cwd, 'plugins/codex-security/skills/triage-finding/SKILL.md'));-fs.writeFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,node:process.env.CODEX_MCP_NODE_PATH,directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])}));+${probeFixturePolicy}+fs.writeFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy,node:process.env.CODEX_MCP_NODE_PATH,directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])}));console.log(JSON.stringify({type:'thread.started',thread_id:'synthetic-cli-provider'}));console.log(JSON.stringify({type:'item.completed',item:{id:'message',type:'agent_message',text:'ok'}}));console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_tokens:1}}));@@ -234,10 +273,12 @@ console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_tPROMPTFOO_DISABLE_TELEMETRY: "1",PROMPTFOO_DISABLE_UPDATE: "1",},+ ["--experimental-strip-types", runner],);assert.equal(result.code, 0, result.output);const row = JSON.parse(fs.readFileSync(capture, "utf8"));assert.notEqual(row.cwd, evalRoot);+ assert.match(row.policy, /Synthetic policy/);assert.equal(fs.existsSync(row.cwd), false);assert.equal(row.node, process.execPath);assert.deepEqual(row.directories, [@@ -261,17 +302,15 @@ test(const capture = path.join(root, "captures.jsonl");const fakeCodex = path.join(root, "codex");const fail = path.join(root, "fail");- fs.writeFileSync(- fakeCodex,- `#!${process.execPath}+ function writeCaptureCodex(executable: string, checkPolicy: boolean) {+ fs.writeFileSync(+ executable,+ `#!${process.execPath}const fs = require('node:fs');const path = require('node:path');-const cp = require('node:child_process');const cwd = process.argv[process.argv.indexOf('--cd') + 1];-const launcher = path.join(cwd, 'plugins/codex-security/scripts/launch_codex_security_mcp');-const fixture = path.join(cwd, 'evals/triage-finding/fixtures/repo');-fs.writeFileSync(path.join(fixture, 'SECURITY.md'), '# Synthetic policy\\n');-const policy = cp.execFileSync(launcher, ['--helper', 'resolve-security-md', '--repo', fixture, '--scope', 'src/server.js', '--out', '-'], {encoding:'utf8'});+fs.readFileSync(path.join(cwd, 'plugins/codex-security/skills/triage-finding/SKILL.md'));+${checkPolicy ? probeFixturePolicy : "const policy = undefined;"}const directories = process.argv.flatMap((arg, index) => arg === '--add-dir' ? [process.argv[index + 1]] : []);const overrides = process.argv.flatMap((arg, index) => arg === '--config' ? [process.argv[index + 1]] : []);fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy,directories,overrides,nodePath:process.env.CODEX_MCP_NODE_PATH,nodeTarget:fs.realpathSync.native(process.env.CODEX_MCP_NODE_PATH), proxies: Object.fromEntries(${JSON.stringify(Object.keys(proxies))}.map(key => [key,process.env[key]]))}) + '\\n');@@ -283,8 +322,10 @@ if (fs.existsSync(${JSON.stringify(fail)})) {console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,cached_input_tokens:0,output_tokens:1}}));}`,- { mode: 0o755 },- );+ { mode: 0o755 },+ );+ }+ writeCaptureCodex(fakeCodex, true);const ambientNode = path.join(root, "ambient-node", "bin", "node");fs.mkdirSync(path.dirname(ambientNode), { recursive: true });fs.copyFileSync(@@ -442,6 +483,10 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "// Prompt-specific settings merge and render separately for each concurrent// case, including the original configuration saved for retry and resume.+ // The initial matrix already ran the helper through each selected Node+ // launcher; these calls capture the configuration saved for replay.+ const templatedCodex = path.join(root, "templated-codex");+ writeCaptureCodex(templatedCodex, false);const templatedConfig = path.join(root, "templated.json");fs.writeFileSync(templatedConfig,@@ -451,6 +496,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "...provider,config: {...provider.config,+ codex_path_override: templatedCodex,cli_env: {...provider.config.cli_env,CODEX_MCP_NODE_PATH: "{{node_path}}",@@ -554,7 +600,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "path.dirname(row.nodePath),path.dirname(row.nodePath),path.dirname(row.nodePath),- path.dirname(fakeCodex),+ path.dirname(templatedCodex),]);}},@@ -1136,14 +1182,7 @@ if (fs.existsSync(${JSON.stringify(failure)})) {);const viewer = spawn(process.execPath,- [- "--experimental-strip-types",- runner,- "view",- "--port",- String(port),- "--no",- ],+ [...replayRunnerArgs(), "view", "--port", String(port), "--no"],{cwd: evalRoot,env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },@@ -1464,14 +1503,7 @@ if (fs.existsSync(${JSON.stringify(failure)})) {);const viewer = spawn(process.execPath,- [- "--experimental-strip-types",- runner,- "view",- "--port",- String(port),- "--no",- ],+ [...replayRunnerArgs(), "view", "--port", String(port), "--no"],{cwd: evalRoot,env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },diff --git a/evals/workflow-routing/test_routing.mts b/evals/workflow-routing/test_routing.mtsindex 2a4cb2c85..46b7d5319 100644--- a/evals/workflow-routing/test_routing.mts+++ b/evals/workflow-routing/test_routing.mts@@ -81,21 +81,22 @@ test("stages every production skill unchanged without the labels or harness", asassert.equal(entry.sha256, sha256(source));}for (const scenario of cases) {- if (scenario.skill) assert.ok(skill(scenario.skill));+ if (scenario.skill !== null) assert.ok(skill(scenario.skill), scenario.id);assert.ok(!routingPrompt(scenario).includes(scenario.id));assert.ok(!routingPrompt(scenario).includes("$"));}});-for (const scenario of cases) {- test(`accepts the expected observed route: ${scenario.id}`, () => {- const reads = scenario.skill ? [read(skill(scenario.skill))] : [];+test("accepts an observed skill read and a completed turn without a skill", () => {+ for (const entry of prepared.skills) {assert.equal(- grade(scenario.skill, [...reads, message, completed]).passed,+ grade(entry.name, [read(entry), message, completed]).passed,true,+ entry.name,);- });-}+ }+ assert.equal(grade(null, [message, completed]).passed, true);+});test("a claimed choice or filename listing cannot satisfy a skill read", () => {const expected = skill("security-scan");diff --git a/github-action/scripts/cli-integration.ts b/github-action/scripts/cli-integration.tsindex e62524c8e..495458e4c 100644--- a/github-action/scripts/cli-integration.ts+++ b/github-action/scripts/cli-integration.ts@@ -57,16 +57,23 @@ try {const cliResult = JSON.parse(stdout);assert.equal(cliResult.turn.mock, true, 'The integration scan must remain synthetic');assert.ok(cliResult.findings.findings.length > 0);- // Verify native SARIF, then exercise the CLI's manual export without model calls.assert.equal(cliResult.sarifPath, join(resultsDirectory, 'exports/results.sarif'));- run(['export', resultsDirectory, '--export-format', 'sarif', '--source-root', repository,- '--output', cliResult.sarifPath], 0);+ // Analyze one native scan export untouched; exercise manual export on the other run.+ if (threshold) {+ await rm(cliResult.sarifPath);+ run(['export', resultsDirectory, '--export-format', 'sarif', '--source-root', repository,+ '--output', cliResult.sarifPath], 0);+ }const result = await analyzeResults({ stdout, resultsDirectory, exitCode, publishable: true, scannedSha:git('rev-parse', 'HEAD').trim() });assert.equal(result.scanStatus, 'completed');assert.equal(result.policyStatus, threshold ? 'failed' : 'passed');assert.equal(result.reportStatus, 'ready');assert.equal(result.sarifUploadReady, true);const sarif = JSON.parse(await readFile(result.paths.sarifPath, 'utf8'));+ const expectedIds = cliResult.findings.findings.map((finding: any) => [finding.occurrenceId, finding.ruleId]);+ assert.ok(expectedIds.flat().every((id: unknown) => typeof id === 'string' && id.length > 0));+ assert.deepEqual(sarif.runs.flatMap((run: any) => run.results.map((finding: any) =>+ [finding.properties?.occurrenceId, finding.ruleId])).sort(), expectedIds.sort());assert.ok(sarif.runs.every((run: any) => run.automationDetails === undefined),'upload-sarif must be able to apply the workflow category to the real CLI export');assert.equal(result.findings.length, cliResult.findings.findings.length);@@ -82,7 +89,9 @@ try {}// Deep Scan cannot use --mock; validate the Action's actual arguments without model calls.- const deepInputs: Record<string, string> = {mode: 'deep', 'max-time-hours': '0.25', paths: 'example.ts', 'dry-run': 'true'};+ // Exercise all explicit cyber programs in existing CLI preflights; the Action input+ // tests cover forwarding every spelling in both modes.+ const deepInputs: Record<string, string> = {mode: 'deep', 'cyber-access-program': 'daybreak_blue', 'max-time-hours': '0.25', paths: 'example.ts', 'dry-run': 'true'};const deepArguments = scanArguments(parseInputs(name => deepInputs[name] ?? '', repository),{repository}, join(root, 'deep-results'));const deepPreflight = JSON.parse(run(deepArguments, 0));@@ -91,32 +100,25 @@ try {assert.equal(deepPreflight.maxTimeHours, 0.25);assert.deepEqual(deepPreflight.target.paths, ['example.ts']);- // Validate explicit program selection against the locked CLI, without model calls.- for (const mode of ['standard', 'deep']) for (const program of ['standard', 'daybreak_blue', 'daybreak_red']) {- const values: Record<string, string> = {mode, 'cyber-access-program':program, 'dry-run':'true'};- const args = scanArguments(parseInputs(name => values[name] ?? '', repository),- {repository}, join(root, `cyber-${mode}-${program}`));- const preflight = JSON.parse(run(args, 0));- assert.equal(preflight.dryRun, true);- assert.equal(preflight.mode, mode);- }-- // Pass provider/model through the locked CLI without copying its provider setup or validation.- for (const provider of ['amazon-bedrock', 'openrouter', 'fireworks']) for (const mode of ['standard', 'deep']) {+ // Exercise each installed provider while covering both scan modes.+ for (const [provider, mode] of [['amazon-bedrock', 'deep'], ['openrouter', 'standard'], ['fireworks', 'deep']]) {const values: Record<string, string> = {provider, model:'synthetic/model', mode, 'dry-run':'true'};const args = scanArguments(parseInputs(name => values[name] ?? '', repository),{repository}, join(root, `${provider}-${mode}`));const preflight = JSON.parse(run(args, 0));assert.equal(preflight.dryRun, true); assert.equal(preflight.mode, mode);+ assert.equal(preflight.model, 'synthetic/model');assert.equal(preflight.modelProvider, provider);if (provider === 'amazon-bedrock') assert.equal(preflight.authentication.method, 'aws_credentials');- values.model = '';- const missingModelArgs = scanArguments(parseInputs(name => values[name] ?? '', repository), {repository}, join(root, 'missing-model'));- const missing = spawnSync(process.execPath, [cli, ...missingModelArgs], {cwd:repository, env, encoding:'utf8', timeout:120_000});- if (missing.error) throw missing.error;- assert.equal(missing.status, 2);- assert.match(missing.stderr + missing.stdout, /--model is required when using --provider/);}+ // The CLI shares missing-model validation across external providers and scan modes.+ const missingValues: Record<string, string> = {provider:'openrouter', 'dry-run':'true'};+ const missingArgs = scanArguments(parseInputs(name => missingValues[name] ?? '', repository),+ {repository}, join(root, 'missing-model'));+ const missing = spawnSync(process.execPath, [cli, ...missingArgs], {cwd:repository, env, encoding:'utf8', timeout:120_000});+ if (missing.error) throw missing.error;+ assert.equal(missing.status, 2);+ assert.match(missing.stderr + missing.stdout, /--model is required when using --provider openrouter/);// Exercise Action arguments through the installed CLI and native request serializer.// A terminal fixture error stops each scan before any model output or tool execution.@@ -184,7 +186,7 @@ try {await writeFile(join(repository, '--help/example.ts'), 'export const example = 1;\n');git('add', '--', '--help/example.ts');git('commit', '-m', 'Synthetic option-shaped path');- const pathInputs: Record<string, string> = {paths: './--help', 'dry-run': 'true'};+ const pathInputs: Record<string, string> = {paths: './--help', 'cyber-access-program':'daybreak_red', 'dry-run': 'true'};const pathArguments = scanArguments(parseInputs(name => pathInputs[name] ?? '', repository),{repository}, join(root, 'path-results'));const pathPreflight = JSON.parse(run(pathArguments, 0));@@ -200,7 +202,7 @@ try {}git('add', '.'); git('commit', '-m', 'Synthetic literal path scopes');const scopes = ['src/[slug]', ...literalFiles.slice(1)];- const literalInputs: Record<string, string> = {paths: scopes.join('\n'), 'dry-run': 'true'};+ const literalInputs: Record<string, string> = {paths: scopes.join('\n'), 'cyber-access-program':'standard', 'dry-run': 'true'};const python = await resolveTool('python3');const literalArguments = scanArguments(parseInputs(name => literalInputs[name] ?? '', repository),{repository}, join(root, 'literal-results'));diff --git a/github-action/src/targets.ts b/github-action/src/targets.tsindex f150e7fa1..c50b216c9 100644--- a/github-action/src/targets.ts+++ b/github-action/src/targets.ts@@ -75,7 +75,7 @@ export async function containedPath(repository: string, input: string): Promise<return path;}-function originMatches(url: string, repository: string, serverUrl: string): boolean {+export function originMatches(url: string, repository: string, serverUrl: string): boolean {// Parse without printing potentially embedded tokens.try {const host = new URL(serverUrl).hostname;diff --git a/github-action/tests/results.test.ts b/github-action/tests/results.test.tsindex b2aeb8fe4..7ed82ea28 100644--- a/github-action/tests/results.test.ts+++ b/github-action/tests/results.test.ts@@ -325,7 +325,7 @@ test('accepted relative path spellings preserve completed findings and SARIF', a}});-test('prepared SARIF lets the uploader set distinct workflow categories', async (t) => {+test('SARIF preparation removes only automationDetails and preserves the published bytes', async (t) => {const opts = await fixture(t);const original = JSON.parse(await readFile(join(opts.resultsDirectory, 'exports/results.sarif'), 'utf8'));assert.ok(original.runs[0].automationDetails.id);@@ -334,14 +334,6 @@ test('prepared SARIF lets the uploader set distinct workflow categories', asyncconst expected = structuredClone(original);for (const run of expected.runs) delete run.automationDetails;assert.deepEqual(sarif, expected);- for (const category of ['security/frontend', 'security/backend']) {- const upload = structuredClone(sarif);- // Match upload-sarif's category precedence: it fills only missing automationDetails.- for (const run of upload.runs) {- if (run.automationDetails === undefined) run.automationDetails = {id: `${category}/`};- }- assert.ok(upload.runs.every((run: any) => run.automationDetails.id === `${category}/`));- }assert.deepEqual((await collectReports(result)).get('exports/results.sarif'), await readFile(result.paths.sarifPath));assert.equal((await analyzeResults(opts)).sarifUploadReady, true, 'preparation is repeatable');});diff --git a/github-action/tests/targets.test.ts b/github-action/tests/targets.test.tsindex 5f3204a8a..3203bcf23 100644--- a/github-action/tests/targets.test.ts+++ b/github-action/tests/targets.test.ts@@ -5,7 +5,19 @@ import { mkdtemp, realpath, writeFile, mkdir, rm, symlink } from 'node:fs/promisimport { tmpdir } from 'node:os';import { dirname, join } from 'node:path';import { parseInputs } from '../src/inputs.js';-import { resolveTarget, validateEvent, git, gitEnvironment, type EventContext } from '../src/targets.js';+import { resolveTarget, validateEvent, originMatches, git, gitEnvironment, type EventContext } from '../src/targets.js';++function eventContext(head = 'a'.repeat(40), eventName = 'pull_request', base = head): EventContext {+ const repository = 'example/repo';+ const pr = eventName === 'pull_request';+ return {+ eventName, repository, sha:head, serverUrl:'https://github.com',+ ref:pr ? 'refs/pull/7/merge' : 'refs/heads/main',+ payload:pr ? {number:7, pull_request:{+ number:7, head:{sha:head,repo:{full_name:repository}}, base:{sha:base,repo:{full_name:repository}},+ }} : {},+ };+}async function fixture(t: any) {const path = await realpath(await mkdtemp(join(tmpdir(),'action-target-')));@@ -18,9 +30,7 @@ async function fixture(t: any) {const change = async (file = 'src/app.ts', text = 'export const n = 2;\n') => {await writeFile(join(path,file),text); run('add','.'); run('commit','-m','change'); return run('rev-parse','HEAD');};- const event = (head: string, eventName = 'pull_request'): EventContext => ({eventName, repository:'example/repo',- sha:head, ref:eventName === 'pull_request' ? 'refs/pull/7/merge' : 'refs/heads/main',serverUrl:'https://github.com',- payload:{number:7,pull_request:{number:7,head:{sha:head,repo:{full_name:'example/repo'}},base:{sha:base,repo:{full_name:'example/repo'}}}}});+ const event = (head: string, eventName?: string) => eventContext(head, eventName, base);const inputs = (values: Record<string,string>={scope:'diff'}) => parseInputs(key=>values[key]??'',path);return {path,run,base,change,event,inputs};}@@ -46,8 +56,7 @@ test('PR target resolves merge base and preserves exact upload head identity', aassert.equal(target.diffBase,f.base); assert.equal(target.diffHead,head);assert.equal(target.analysisRef,'refs/pull/7/head'); assert.equal(target.publishable,true);});-test('GitHub Cloud checkout origins match the event host for HTTPS and SSH', async t => {- const f = await fixture(t); const head = await f.change();+test('GitHub Cloud matches HTTPS, SSH and SCP origin spellings for each event host', () => {for (const [host, sshUser] of [['github.com', 'git'], ['example-enterprise.ghe.com', 'example-enterprise']]) {for (const origin of [`https://${host}/example/repo.git`, `https://${host}:443/example/repo`,@@ -55,58 +64,46 @@ test('GitHub Cloud checkout origins match the event host for HTTPS and SSH', asy`${sshUser}@${host}:example/repo.git`, `${sshUser}@${host.toUpperCase()}:Example/Repo/`,`ssh://${sshUser}@${host}/example/repo`, `ssh://${sshUser}@${host.toUpperCase()}:22/Example/Repo.git/`,]) {- f.run('remote', 'set-url', 'origin', origin);- for (const eventName of ['schedule', 'pull_request']) {- const event = {...f.event(head, eventName), serverUrl:`https://${host}`};- const target = await resolveTarget(f.inputs({scope:eventName === 'schedule' ? 'repository' : 'diff'}), event);- assert.equal(target.scannedSha, head); assert.equal(target.publishable, true);- assert.equal(target.analysisRef, eventName === 'schedule' ? 'refs/heads/main' : 'refs/pull/7/head');- assert.equal(target.diffBase, eventName === 'schedule' ? undefined : f.base);- }+ assert.equal(originMatches(origin, 'example/repo', `https://${host}`), true, origin);}}});-test('GitHub.com checkout origins support its SSH-over-443 endpoint', async t => {- const f = await fixture(t);- f.run('remote', 'set-url', 'origin', 'ssh://[email protected]:443/example/repo.git');- const target = await resolveTarget(f.inputs({scope:'repository'}), f.event(f.base, 'schedule'));- assert.equal(target.scannedSha, f.base);- assert.equal(target.publishable, true);+test('target resolution checks the configured SSH origin against the triggering host and repository', async t => {+ const f = await fixture(t); const head = await f.change();+ const enterpriseEvent = {...f.event(head), serverUrl:'https://example-enterprise.ghe.com'};+ f.run('remote', 'set-url', 'origin', '[email protected]:example/repo.git');+ const pr = await resolveTarget(f.inputs(), enterpriseEvent);+ assert.equal(pr.scannedSha, head); assert.equal(pr.publishable, true);+ assert.equal(pr.analysisRef, 'refs/pull/7/head'); assert.equal(pr.diffBase, f.base);+ const schedule = f.event(head, 'schedule');+ await assert.rejects(resolveTarget(f.inputs({scope:'repository'}), schedule), /origin does not match/);- const enterpriseEvent = {...f.event(f.base), serverUrl:'https://example-enterprise.ghe.com'};+ f.run('remote', 'set-url', 'origin', 'ssh://[email protected]:443/example/repo.git');+ const target = await resolveTarget(f.inputs({scope:'repository'}), schedule);+ assert.equal(target.scannedSha, head); assert.equal(target.publishable, true);+ assert.equal(target.analysisRef, 'refs/heads/main'); assert.equal(target.diffBase, undefined);await assert.rejects(resolveTarget(f.inputs(), enterpriseEvent), /origin does not match/);- for (const origin of ['https://ssh.github.com/example/repo.git', 'ssh://[email protected]:22/example/repo.git',- 'ssh://[email protected]:443/other/repo.git', 'ssh://[email protected]:443/example/repo.git']) {- f.run('remote', 'set-url', 'origin', origin);- await assert.rejects(resolveTarget(f.inputs(), f.event(f.base)), /origin does not match/);- }+ await assert.rejects(resolveTarget(f.inputs({scope:'repository'}), {...schedule, repository:'example/other'}), /origin does not match/);});-test('SCP checkout paths remain literal when compared with the triggering repository', async t => {- const f = await fixture(t);- for (const origin of ['[email protected]:example/repo.git#suffix', '[email protected]:example/repo.git?suffix',+test('origins must match the event host, SSH endpoint and literal repository path', () => {+ for (const origin of ['https://ssh.github.com/example/repo.git', 'ssh://[email protected]:22/example/repo.git',+ 'ssh://[email protected]:443/other/repo.git', 'ssh://[email protected]:443/example/repo.git',+ '[email protected]:example/repo.git#suffix', '[email protected]:example/repo.git?suffix','[email protected]:other/../example/repo.git']) {- f.run('remote', 'set-url', 'origin', origin);- await assert.rejects(resolveTarget(f.inputs(), f.event(f.base)), /origin does not match/);+ assert.equal(originMatches(origin, 'example/repo', 'https://github.com'), false, origin);}-});-test('unsupported server URLs still fail eligibility', async t => {- const f = await fixture(t);- for (const serverUrl of ['', 'https://github.example.invalid', 'http://example-enterprise.ghe.com',- 'https://ghe.com', 'https://example-enterprise.ghe.com.example.invalid', 'https://example-enterprise-ghe.com']) {- assert.throws(() => validateEvent({...f.event(f.base), serverUrl}), /supports GitHub.com and GitHub Enterprise Cloud/);- }-});-test('enterprise checkout origins cannot select another host or repository', async t => {- const f = await fixture(t);- const event = {...f.event(f.base), serverUrl:'https://example-enterprise.ghe.com'};for (const origin of ['https://github.com/example/repo.git', 'https://other-enterprise.ghe.com/example/repo.git','[email protected]:example/repo.git', 'https://example-enterprise.ghe.com/other/repo.git','[email protected]:other/repo.git']) {- f.run('remote', 'set-url', 'origin', origin);- await assert.rejects(resolveTarget(f.inputs(), event), /origin does not match/);+ assert.equal(originMatches(origin, 'example/repo', 'https://example-enterprise.ghe.com'), false, origin);+ }+});+test('unsupported server URLs fail eligibility', () => {+ const event = eventContext();+ for (const serverUrl of ['', 'https://github.example.invalid', 'http://example-enterprise.ghe.com',+ 'https://ghe.com', 'https://example-enterprise.ghe.com.example.invalid', 'https://example-enterprise-ghe.com']) {+ assert.throws(() => validateEvent({...event, serverUrl}), /supports GitHub.com and GitHub Enterprise Cloud/, serverUrl);}- f.run('remote', 'set-url', 'origin', 'https://example-enterprise.ghe.com/example/repo.git');- await assert.rejects(resolveTarget(f.inputs(), f.event(f.base)), /origin does not match/);});test('manual diff scans resolve an explicit base against the checked-out head', async t => {const f=await fixture(t);@@ -129,7 +126,7 @@ test('large diffs are nonempty without collecting every changed pathname', asyncf.run('reset', '--hard', head);for (const eventName of ['pull_request', 'workflow_dispatch']) {const event = f.event(head, eventName);- event.payload.pull_request.base.sha = base;+ if (eventName === 'pull_request') event.payload.pull_request.base.sha = base;const inputs = f.inputs(eventName === 'pull_request' ? {scope:'diff'} : {scope:'diff', 'diff-base':base});const target = await resolveTarget(inputs, event);assert.equal(target.diffBase, base);@@ -169,8 +166,8 @@ test('wrong checkout, local changes, missing diff base and option revisions areawait assert.rejects(resolveTarget(f.inputs(),f.event(head)),/local changes/);await assert.rejects(resolveTarget(f.inputs({scope:'repository'}),f.event(head,'schedule')),/local changes/);});-test('forks and privileged events fail eligibility',async t=>{- const f=await fixture(t); const event=f.event(f.base);+test('forks and privileged events fail eligibility',()=>{+ const event=eventContext();event.payload.pull_request.head.repo.full_name='fork/repo'; assert.throws(()=>validateEvent(event),/Fork PR/);for(const eventName of ['pull_request_target','workflow_run']) assert.throws(()=>validateEvent({...event,eventName}),/Unsupported event/);});