Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: reuse triage replay builds and exercise native Action SARIF (#1977)

Michael D'Angelo3dfed19297d6
diff --git a/.github/scripts/invoice-desk-target.test.mjs b/.github/scripts/invoice-desk-target.test.mjs
index 237eef4ea..7a4be077a 100644
--- a/.github/scripts/invoice-desk-target.test.mjs
+++ b/.github/scripts/invoice-desk-target.test.mjs
@@ -92,13 +92,29 @@ test("incomplete or invalid PR inputs fail instead of falling back to a baseline
}
});
-test("ignores unexpected events without querying GitHub", async () => {
- assert.deepEqual(
- await resolveScanTargets({ ...context, eventName: "pull_request" }, () =>
- assert.fail("unexpected API call"),
- ),
- [],
- );
+test("PR events and completed PR workflows do not queue inference or query GitHub", async () => {
+ for (const trigger of [
+ { eventName: "pull_request", event: { pull_request: pullRequest() } },
+ {
+ eventName: "workflow_run",
+ event: {
+ workflow_run: {
+ event: "pull_request",
+ conclusion: "success",
+ head_sha: sourceSha,
+ pull_requests: [{ number: 7 }],
+ },
+ },
+ },
+ ]) {
+ assert.deepEqual(
+ await resolveScanTargets({ ...context, ...trigger }, () =>
+ assert.fail("unexpected API call"),
+ ),
+ [],
+ trigger.eventName,
+ );
+ }
});
test("scheduled baseline scans use the trusted workflow commit", async () => {
@@ -111,27 +127,6 @@ test("scheduled baseline scans use the trusted workflow commit", async () => {
);
});
-test("completed behavior workflows no longer queue inference", async () => {
- assert.deepEqual(
- await resolveScanTargets(
- {
- ...context,
- eventName: "workflow_run",
- event: {
- workflow_run: {
- event: "pull_request",
- conclusion: "success",
- head_sha: sourceSha,
- pull_requests: [{ number: 7 }],
- },
- },
- },
- () => assert.fail("unexpected API call"),
- ),
- [],
- );
-});
-
test("approval recheck stays bound to its matrix PR and source revision", async () => {
const run = { ...context, target: { pr: 7, sha: sourceSha } };
for (const changed of [
diff --git a/evals/triage-finding/scripts/run-promptfoo.mts b/evals/triage-finding/scripts/run-promptfoo.mts
index ea9bbf8e4..90339197c 100644
--- a/evals/triage-finding/scripts/run-promptfoo.mts
+++ b/evals/triage-finding/scripts/run-promptfoo.mts
@@ -74,6 +74,7 @@ function stageSkillRuntime() {
async function runPromptfoo(
promptfooArgs: string[],
environment: NodeJS.ProcessEnv = {},
+ builtMcp?: string,
) {
const runtimeRoot = stageSkillRuntime();
const artifacts = path.join(EVAL_ROOT, "artifacts");
@@ -131,17 +132,22 @@ async function runPromptfoo(
});
}
try {
- const built = await runChild(
- [
- path.join(PLUGIN_ROOT, "mcp-app", "scripts", "build_mcp_app.mjs"),
- "--output",
- path.join(runtimeRoot, "plugins", "codex-security", "mcp"),
- "--native",
- "host",
- ],
- path.join(PLUGIN_ROOT, "mcp-app"),
- );
- if (built !== 0) return built;
+ const mcp = path.join(runtimeRoot, "plugins", "codex-security", "mcp");
+ if (builtMcp) {
+ copyDirectory(builtMcp, mcp);
+ } else {
+ const built = await runChild(
+ [
+ path.join(PLUGIN_ROOT, "mcp-app", "scripts", "build_mcp_app.mjs"),
+ "--output",
+ mcp,
+ "--native",
+ "host",
+ ],
+ path.join(PLUGIN_ROOT, "mcp-app"),
+ );
+ if (built !== 0) return built;
+ }
return await runChild(
[
"--import",
diff --git a/evals/triage-finding/scripts/run-provider-fixture.mts b/evals/triage-finding/scripts/run-provider-fixture.mts
new file mode 100644
index 000000000..507ae8a3b
--- /dev/null
+++ b/evals/triage-finding/scripts/run-provider-fixture.mts
@@ -0,0 +1,4 @@
+import { runPromptfoo } from "./run-promptfoo.mts";
+
+const [builtMcp, ...args] = process.argv.slice(2);
+process.exitCode = await runPromptfoo(args, {}, builtMcp);
diff --git a/evals/triage-finding/scripts/test-app-surface-instructions.mts b/evals/triage-finding/scripts/test-app-surface-instructions.mts
index 38cf5602d..91d4e0969 100644
--- a/evals/triage-finding/scripts/test-app-surface-instructions.mts
+++ b/evals/triage-finding/scripts/test-app-surface-instructions.mts
@@ -20,9 +20,12 @@ const agent = fs.readFileSync(agentPath, "utf8");
const pluginPath = path.join(pluginRoot, ".codex-plugin", "plugin.json");
const plugin = JSON.parse(fs.readFileSync(pluginPath, "utf8"));
-assert.match(skill, /### Jira and Linear intake/);
assert.match(skill, /references\/ticket-intake\.md/);
assert.match(ticketIntake, /Atlassian[\s\S]*JQL/);
+assert.match(
+ ticketIntake,
+ /\bsearchJiraIssuesUsingJql`?\s+for\s+collections\b|\bfor\s+collections,\s+use\s+`?searchJiraIssuesUsingJql\b/i,
+);
assert.match(ticketIntake, /natural-language search[\s\S]*discover[\s\S]*JQL/);
assert.match(skill, /security or vulnerability Jira\/Linear tickets/);
assert.match(skill, /Atlassian and Linear mentions are connector hints/);
@@ -35,9 +38,24 @@ assert.match(
ticketIntake,
/issue key[\s\S]*URL[\s\S]*project[\s\S]*status[\s\S]*labels[\s\S]*components[\s\S]*priority/,
);
-assert.match(
- ticketIntake,
- /Default to read-only import and triage[\s\S]*Do not add comments, transition or close issues, assign owners, or change labels/,
+const sentences = ticketIntake
+ .split(/[.!?]|\n\s*\n|^#{1,6}[ \t]+.*$/m)
+ .map((sentence) => sentence.trim().replace(/\s+/g, " "));
+assert.ok(
+ sentences.some((sentence) =>
+ sentence.startsWith("Default to read-only import and triage"),
+ ),
+);
+assert.ok(
+ sentences.some(
+ (sentence) =>
+ sentence.startsWith(
+ "Do not add comments, transition or close issues, assign owners, or change labels",
+ ) &&
+ /\b(?:without|unless)\b.*\bexplicit(?:ly)?\b.*\brequest(?:ed|s)?\b/i.test(
+ sentence,
+ ),
+ ),
);
assert.match(agent, /default_prompt:.*Use \$triage-finding/);
assert.match(ticketIntake, /missing connector|connector.*unavailable/i);
@@ -49,7 +67,7 @@ assert.match(ticketIntake, /exhaust[\s\S]*pag(?:es|ination)/i);
assert.match(ticketIntake, /independent vulnerability claim/i);
assert.match(ticketIntake, /ambiguous[\s\S]*ask/i);
assert.match(ticketIntake, /deterministic[\s\S]*tree order/i);
-assert.equal(plugin.interface.defaultPrompt.length, 3);
+assert.ok(plugin.interface.defaultPrompt.length <= 3);
assert(
plugin.interface.defaultPrompt.every(
(prompt: string) => [...prompt].length <= 128,
diff --git a/evals/triage-finding/scripts/test-promptfoo-runtime.mts b/evals/triage-finding/scripts/test-promptfoo-runtime.mts
index 27ad04d52..eb4ee2014 100644
--- a/evals/triage-finding/scripts/test-promptfoo-runtime.mts
+++ b/evals/triage-finding/scripts/test-promptfoo-runtime.mts
@@ -3,7 +3,6 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
-import { pathToFileURL } from "node:url";
import { parse } from "yaml";
import runtimeVars from "./runtime-vars.mts";
@@ -162,15 +161,4 @@ assert.ok(
"Read and follow plugins/codex-security/skills/triage-finding/SKILL.md",
),
);
-// Load the installed scanner so its Git imports are checked at runtime.
-const promptfooDist = path.join(evalRoot, "node_modules/promptfoo/dist/src");
-const scanner = fs
- .readdirSync(promptfooDist)
- .find((name) => name.startsWith("scanner-") && name.endsWith(".js"));
-assert.ok(scanner, "Promptfoo scanner entrypoint must be installed");
-const scannerModule = await import(
- pathToFileURL(path.join(promptfooDist, scanner)).href
-);
-assert.equal(typeof scannerModule.executeScan, "function");
-
console.log("triage Promptfoo runtime tests passed");
diff --git a/evals/triage-finding/scripts/test-triage-provider.mts b/evals/triage-finding/scripts/test-triage-provider.mts
index 5c5eda611..efae11302 100644
--- a/evals/triage-finding/scripts/test-triage-provider.mts
+++ b/evals/triage-finding/scripts/test-triage-provider.mts
@@ -1,16 +1,54 @@
import assert from "node:assert/strict";
-import { spawn } from "node:child_process";
+import { execFileSync, spawn } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import { createServer } from "node:net";
import path from "node:path";
-import test from "node:test";
+import test, { after } from "node:test";
import { DatabaseSync } from "node:sqlite";
import { parse } from "yaml";
import { beforeAll } from "./triage-provider.mts";
const evalRoot = path.resolve(import.meta.dirname, "..");
const runner = path.join(import.meta.dirname, "run-promptfoo.mts");
+const fixtureRunner = path.join(
+ import.meta.dirname,
+ "run-provider-fixture.mts",
+);
+const fixtureRoot = fs.realpathSync(
+ fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-bundle-")),
+);
+after(() => fs.rmSync(fixtureRoot, { recursive: true, force: true }));
+let builtMcp: string | undefined;
+
+function replayRunnerArgs() {
+ if (!builtMcp) {
+ const output = path.join(fixtureRoot, "mcp");
+ const mcpApp = path.join(evalRoot, "../../plugins/codex-security/mcp-app");
+ execFileSync(
+ process.execPath,
+ [
+ path.join(mcpApp, "scripts/build_mcp_app.mjs"),
+ "--output",
+ output,
+ "--native",
+ "host",
+ ],
+ { cwd: mcpApp },
+ );
+ builtMcp = output;
+ }
+ // Every child copies the same build into its own throwaway skill runtime.
+ return ["--experimental-strip-types", fixtureRunner, builtMcp];
+}
+
+const probeFixturePolicy = `
+const launcher = path.join(cwd, 'plugins/codex-security/scripts/launch_codex_security_mcp');
+const fixture = path.join(cwd, 'evals/triage-finding/fixtures/repo');
+fs.writeFileSync(path.join(fixture, 'SECURITY.md'), '# Synthetic policy\\n');
+const policy = require('node:child_process').execFileSync(launcher, ['--helper', 'resolve-security-md', '--repo', fixture, '--scope', 'src/server.js', '--out', '-'], {encoding:'utf8'});
+`;
+
function providerConfig(name: string) {
return parse(
fs.readFileSync(path.join(evalRoot, "providers", `${name}.yaml`), "utf8"),
@@ -36,18 +74,18 @@ const proxies = {
NO_PROXY: "localhost,.example.test",
};
-function invoke(args: string[], environment: NodeJS.ProcessEnv) {
+function invoke(
+ args: string[],
+ environment: NodeJS.ProcessEnv,
+ runnerArgs = replayRunnerArgs(),
+) {
return new Promise<{ code: number | null; output: string }>(
(resolve, reject) => {
- const child = spawn(
- process.execPath,
- ["--experimental-strip-types", runner, ...args],
- {
- cwd: evalRoot,
- env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },
- stdio: ["ignore", "pipe", "pipe"],
- },
- );
+ const child = spawn(process.execPath, [...runnerArgs, ...args], {
+ cwd: evalRoot,
+ env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },
+ stdio: ["ignore", "pipe", "pipe"],
+ });
let output = "";
child.stdout.on("data", (chunk) => {
output += chunk;
@@ -191,7 +229,8 @@ const fs = require('node:fs');
const path = require('node:path');
const cwd = process.argv[process.argv.indexOf('--cd') + 1];
fs.readFileSync(path.join(cwd, 'plugins/codex-security/skills/triage-finding/SKILL.md'));
-fs.writeFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,node:process.env.CODEX_MCP_NODE_PATH,directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])}));
+${probeFixturePolicy}
+fs.writeFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy,node:process.env.CODEX_MCP_NODE_PATH,directories:process.argv.flatMap((arg,index)=>arg==='--add-dir'?[process.argv[index+1]]:[])}));
console.log(JSON.stringify({type:'thread.started',thread_id:'synthetic-cli-provider'}));
console.log(JSON.stringify({type:'item.completed',item:{id:'message',type:'agent_message',text:'ok'}}));
console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_tokens:1}}));
@@ -234,10 +273,12 @@ console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_t
PROMPTFOO_DISABLE_TELEMETRY: "1",
PROMPTFOO_DISABLE_UPDATE: "1",
},
+ ["--experimental-strip-types", runner],
);
assert.equal(result.code, 0, result.output);
const row = JSON.parse(fs.readFileSync(capture, "utf8"));
assert.notEqual(row.cwd, evalRoot);
+ assert.match(row.policy, /Synthetic policy/);
assert.equal(fs.existsSync(row.cwd), false);
assert.equal(row.node, process.execPath);
assert.deepEqual(row.directories, [
@@ -261,17 +302,15 @@ test(
const capture = path.join(root, "captures.jsonl");
const fakeCodex = path.join(root, "codex");
const fail = path.join(root, "fail");
- fs.writeFileSync(
- fakeCodex,
- `#!${process.execPath}
+ function writeCaptureCodex(executable: string, checkPolicy: boolean) {
+ fs.writeFileSync(
+ executable,
+ `#!${process.execPath}
const fs = require('node:fs');
const path = require('node:path');
-const cp = require('node:child_process');
const cwd = process.argv[process.argv.indexOf('--cd') + 1];
-const launcher = path.join(cwd, 'plugins/codex-security/scripts/launch_codex_security_mcp');
-const fixture = path.join(cwd, 'evals/triage-finding/fixtures/repo');
-fs.writeFileSync(path.join(fixture, 'SECURITY.md'), '# Synthetic policy\\n');
-const policy = cp.execFileSync(launcher, ['--helper', 'resolve-security-md', '--repo', fixture, '--scope', 'src/server.js', '--out', '-'], {encoding:'utf8'});
+fs.readFileSync(path.join(cwd, 'plugins/codex-security/skills/triage-finding/SKILL.md'));
+${checkPolicy ? probeFixturePolicy : "const policy = undefined;"}
const directories = process.argv.flatMap((arg, index) => arg === '--add-dir' ? [process.argv[index + 1]] : []);
const overrides = process.argv.flatMap((arg, index) => arg === '--config' ? [process.argv[index + 1]] : []);
fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy,directories,overrides,nodePath:process.env.CODEX_MCP_NODE_PATH,nodeTarget:fs.realpathSync.native(process.env.CODEX_MCP_NODE_PATH), proxies: Object.fromEntries(${JSON.stringify(Object.keys(proxies))}.map(key => [key,process.env[key]]))}) + '\\n');
@@ -283,8 +322,10 @@ if (fs.existsSync(${JSON.stringify(fail)})) {
console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,cached_input_tokens:0,output_tokens:1}}));
}
`,
- { mode: 0o755 },
- );
+ { mode: 0o755 },
+ );
+ }
+ writeCaptureCodex(fakeCodex, true);
const ambientNode = path.join(root, "ambient-node", "bin", "node");
fs.mkdirSync(path.dirname(ambientNode), { recursive: true });
fs.copyFileSync(
@@ -442,6 +483,10 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
// Prompt-specific settings merge and render separately for each concurrent
// case, including the original configuration saved for retry and resume.
+ // The initial matrix already ran the helper through each selected Node
+ // launcher; these calls capture the configuration saved for replay.
+ const templatedCodex = path.join(root, "templated-codex");
+ writeCaptureCodex(templatedCodex, false);
const templatedConfig = path.join(root, "templated.json");
fs.writeFileSync(
templatedConfig,
@@ -451,6 +496,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
...provider,
config: {
...provider.config,
+ codex_path_override: templatedCodex,
cli_env: {
...provider.config.cli_env,
CODEX_MCP_NODE_PATH: "{{node_path}}",
@@ -554,7 +600,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
path.dirname(row.nodePath),
path.dirname(row.nodePath),
path.dirname(row.nodePath),
- path.dirname(fakeCodex),
+ path.dirname(templatedCodex),
]);
}
},
@@ -1136,14 +1182,7 @@ if (fs.existsSync(${JSON.stringify(failure)})) {
);
const viewer = spawn(
process.execPath,
- [
- "--experimental-strip-types",
- runner,
- "view",
- "--port",
- String(port),
- "--no",
- ],
+ [...replayRunnerArgs(), "view", "--port", String(port), "--no"],
{
cwd: evalRoot,
env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },
@@ -1464,14 +1503,7 @@ if (fs.existsSync(${JSON.stringify(failure)})) {
);
const viewer = spawn(
process.execPath,
- [
- "--experimental-strip-types",
- runner,
- "view",
- "--port",
- String(port),
- "--no",
- ],
+ [...replayRunnerArgs(), "view", "--port", String(port), "--no"],
{
cwd: evalRoot,
env: { ...process.env, ...environment, NODE_USE_ENV_PROXY: "" },
diff --git a/evals/workflow-routing/test_routing.mts b/evals/workflow-routing/test_routing.mts
index 2a4cb2c85..46b7d5319 100644
--- a/evals/workflow-routing/test_routing.mts
+++ b/evals/workflow-routing/test_routing.mts
@@ -81,21 +81,22 @@ test("stages every production skill unchanged without the labels or harness", as
assert.equal(entry.sha256, sha256(source));
}
for (const scenario of cases) {
- if (scenario.skill) assert.ok(skill(scenario.skill));
+ if (scenario.skill !== null) assert.ok(skill(scenario.skill), scenario.id);
assert.ok(!routingPrompt(scenario).includes(scenario.id));
assert.ok(!routingPrompt(scenario).includes("$"));
}
});
-for (const scenario of cases) {
- test(`accepts the expected observed route: ${scenario.id}`, () => {
- const reads = scenario.skill ? [read(skill(scenario.skill))] : [];
+test("accepts an observed skill read and a completed turn without a skill", () => {
+ for (const entry of prepared.skills) {
assert.equal(
- grade(scenario.skill, [...reads, message, completed]).passed,
+ grade(entry.name, [read(entry), message, completed]).passed,
true,
+ entry.name,
);
- });
-}
+ }
+ assert.equal(grade(null, [message, completed]).passed, true);
+});
test("a claimed choice or filename listing cannot satisfy a skill read", () => {
const expected = skill("security-scan");
diff --git a/github-action/scripts/cli-integration.ts b/github-action/scripts/cli-integration.ts
index e62524c8e..495458e4c 100644
--- a/github-action/scripts/cli-integration.ts
+++ b/github-action/scripts/cli-integration.ts
@@ -57,16 +57,23 @@ try {
const cliResult = JSON.parse(stdout);
assert.equal(cliResult.turn.mock, true, 'The integration scan must remain synthetic');
assert.ok(cliResult.findings.findings.length > 0);
- // Verify native SARIF, then exercise the CLI's manual export without model calls.
assert.equal(cliResult.sarifPath, join(resultsDirectory, 'exports/results.sarif'));
- run(['export', resultsDirectory, '--export-format', 'sarif', '--source-root', repository,
- '--output', cliResult.sarifPath], 0);
+ // Analyze one native scan export untouched; exercise manual export on the other run.
+ if (threshold) {
+ await rm(cliResult.sarifPath);
+ run(['export', resultsDirectory, '--export-format', 'sarif', '--source-root', repository,
+ '--output', cliResult.sarifPath], 0);
+ }
const result = await analyzeResults({ stdout, resultsDirectory, exitCode, publishable: true, scannedSha:git('rev-parse', 'HEAD').trim() });
assert.equal(result.scanStatus, 'completed');
assert.equal(result.policyStatus, threshold ? 'failed' : 'passed');
assert.equal(result.reportStatus, 'ready');
assert.equal(result.sarifUploadReady, true);
const sarif = JSON.parse(await readFile(result.paths.sarifPath, 'utf8'));
+ const expectedIds = cliResult.findings.findings.map((finding: any) => [finding.occurrenceId, finding.ruleId]);
+ assert.ok(expectedIds.flat().every((id: unknown) => typeof id === 'string' && id.length > 0));
+ assert.deepEqual(sarif.runs.flatMap((run: any) => run.results.map((finding: any) =>
+ [finding.properties?.occurrenceId, finding.ruleId])).sort(), expectedIds.sort());
assert.ok(sarif.runs.every((run: any) => run.automationDetails === undefined),
'upload-sarif must be able to apply the workflow category to the real CLI export');
assert.equal(result.findings.length, cliResult.findings.findings.length);
@@ -82,7 +89,9 @@ try {
}
// Deep Scan cannot use --mock; validate the Action's actual arguments without model calls.
- const deepInputs: Record<string, string> = {mode: 'deep', 'max-time-hours': '0.25', paths: 'example.ts', 'dry-run': 'true'};
+ // Exercise all explicit cyber programs in existing CLI preflights; the Action input
+ // tests cover forwarding every spelling in both modes.
+ const deepInputs: Record<string, string> = {mode: 'deep', 'cyber-access-program': 'daybreak_blue', 'max-time-hours': '0.25', paths: 'example.ts', 'dry-run': 'true'};
const deepArguments = scanArguments(parseInputs(name => deepInputs[name] ?? '', repository),
{repository}, join(root, 'deep-results'));
const deepPreflight = JSON.parse(run(deepArguments, 0));
@@ -91,32 +100,25 @@ try {
assert.equal(deepPreflight.maxTimeHours, 0.25);
assert.deepEqual(deepPreflight.target.paths, ['example.ts']);
- // Validate explicit program selection against the locked CLI, without model calls.
- for (const mode of ['standard', 'deep']) for (const program of ['standard', 'daybreak_blue', 'daybreak_red']) {
- const values: Record<string, string> = {mode, 'cyber-access-program':program, 'dry-run':'true'};
- const args = scanArguments(parseInputs(name => values[name] ?? '', repository),
- {repository}, join(root, `cyber-${mode}-${program}`));
- const preflight = JSON.parse(run(args, 0));
- assert.equal(preflight.dryRun, true);
- assert.equal(preflight.mode, mode);
- }
-
- // Pass provider/model through the locked CLI without copying its provider setup or validation.
- for (const provider of ['amazon-bedrock', 'openrouter', 'fireworks']) for (const mode of ['standard', 'deep']) {
+ // Exercise each installed provider while covering both scan modes.
+ for (const [provider, mode] of [['amazon-bedrock', 'deep'], ['openrouter', 'standard'], ['fireworks', 'deep']]) {
const values: Record<string, string> = {provider, model:'synthetic/model', mode, 'dry-run':'true'};
const args = scanArguments(parseInputs(name => values[name] ?? '', repository),
{repository}, join(root, `${provider}-${mode}`));
const preflight = JSON.parse(run(args, 0));
assert.equal(preflight.dryRun, true); assert.equal(preflight.mode, mode);
+ assert.equal(preflight.model, 'synthetic/model');
assert.equal(preflight.modelProvider, provider);
if (provider === 'amazon-bedrock') assert.equal(preflight.authentication.method, 'aws_credentials');
- values.model = '';
- const missingModelArgs = scanArguments(parseInputs(name => values[name] ?? '', repository), {repository}, join(root, 'missing-model'));
- const missing = spawnSync(process.execPath, [cli, ...missingModelArgs], {cwd:repository, env, encoding:'utf8', timeout:120_000});
- if (missing.error) throw missing.error;
- assert.equal(missing.status, 2);
- assert.match(missing.stderr + missing.stdout, /--model is required when using --provider/);
}
+ // The CLI shares missing-model validation across external providers and scan modes.
+ const missingValues: Record<string, string> = {provider:'openrouter', 'dry-run':'true'};
+ const missingArgs = scanArguments(parseInputs(name => missingValues[name] ?? '', repository),
+ {repository}, join(root, 'missing-model'));
+ const missing = spawnSync(process.execPath, [cli, ...missingArgs], {cwd:repository, env, encoding:'utf8', timeout:120_000});
+ if (missing.error) throw missing.error;
+ assert.equal(missing.status, 2);
+ assert.match(missing.stderr + missing.stdout, /--model is required when using --provider openrouter/);
// Exercise Action arguments through the installed CLI and native request serializer.
// A terminal fixture error stops each scan before any model output or tool execution.
@@ -184,7 +186,7 @@ try {
await writeFile(join(repository, '--help/example.ts'), 'export const example = 1;\n');
git('add', '--', '--help/example.ts');
git('commit', '-m', 'Synthetic option-shaped path');
- const pathInputs: Record<string, string> = {paths: './--help', 'dry-run': 'true'};
+ const pathInputs: Record<string, string> = {paths: './--help', 'cyber-access-program':'daybreak_red', 'dry-run': 'true'};
const pathArguments = scanArguments(parseInputs(name => pathInputs[name] ?? '', repository),
{repository}, join(root, 'path-results'));
const pathPreflight = JSON.parse(run(pathArguments, 0));
@@ -200,7 +202,7 @@ try {
}
git('add', '.'); git('commit', '-m', 'Synthetic literal path scopes');
const scopes = ['src/[slug]', ...literalFiles.slice(1)];
- const literalInputs: Record<string, string> = {paths: scopes.join('\n'), 'dry-run': 'true'};
+ const literalInputs: Record<string, string> = {paths: scopes.join('\n'), 'cyber-access-program':'standard', 'dry-run': 'true'};
const python = await resolveTool('python3');
const literalArguments = scanArguments(parseInputs(name => literalInputs[name] ?? '', repository),
{repository}, join(root, 'literal-results'));
diff --git a/github-action/src/targets.ts b/github-action/src/targets.ts
index f150e7fa1..c50b216c9 100644
--- a/github-action/src/targets.ts
+++ b/github-action/src/targets.ts
@@ -75,7 +75,7 @@ export async function containedPath(repository: string, input: string): Promise<
return path;
}
-function originMatches(url: string, repository: string, serverUrl: string): boolean {
+export function originMatches(url: string, repository: string, serverUrl: string): boolean {
// Parse without printing potentially embedded tokens.
try {
const host = new URL(serverUrl).hostname;
diff --git a/github-action/tests/results.test.ts b/github-action/tests/results.test.ts
index b2aeb8fe4..7ed82ea28 100644
--- a/github-action/tests/results.test.ts
+++ b/github-action/tests/results.test.ts
@@ -325,7 +325,7 @@ test('accepted relative path spellings preserve completed findings and SARIF', a
}
});
-test('prepared SARIF lets the uploader set distinct workflow categories', async (t) => {
+test('SARIF preparation removes only automationDetails and preserves the published bytes', async (t) => {
const opts = await fixture(t);
const original = JSON.parse(await readFile(join(opts.resultsDirectory, 'exports/results.sarif'), 'utf8'));
assert.ok(original.runs[0].automationDetails.id);
@@ -334,14 +334,6 @@ test('prepared SARIF lets the uploader set distinct workflow categories', async
const expected = structuredClone(original);
for (const run of expected.runs) delete run.automationDetails;
assert.deepEqual(sarif, expected);
- for (const category of ['security/frontend', 'security/backend']) {
- const upload = structuredClone(sarif);
- // Match upload-sarif's category precedence: it fills only missing automationDetails.
- for (const run of upload.runs) {
- if (run.automationDetails === undefined) run.automationDetails = {id: `${category}/`};
- }
- assert.ok(upload.runs.every((run: any) => run.automationDetails.id === `${category}/`));
- }
assert.deepEqual((await collectReports(result)).get('exports/results.sarif'), await readFile(result.paths.sarifPath));
assert.equal((await analyzeResults(opts)).sarifUploadReady, true, 'preparation is repeatable');
});
diff --git a/github-action/tests/targets.test.ts b/github-action/tests/targets.test.ts
index 5f3204a8a..3203bcf23 100644
--- a/github-action/tests/targets.test.ts
+++ b/github-action/tests/targets.test.ts
@@ -5,7 +5,19 @@ import { mkdtemp, realpath, writeFile, mkdir, rm, symlink } from 'node:fs/promis
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { parseInputs } from '../src/inputs.js';
-import { resolveTarget, validateEvent, git, gitEnvironment, type EventContext } from '../src/targets.js';
+import { resolveTarget, validateEvent, originMatches, git, gitEnvironment, type EventContext } from '../src/targets.js';
+
+function eventContext(head = 'a'.repeat(40), eventName = 'pull_request', base = head): EventContext {
+ const repository = 'example/repo';
+ const pr = eventName === 'pull_request';
+ return {
+ eventName, repository, sha:head, serverUrl:'https://github.com',
+ ref:pr ? 'refs/pull/7/merge' : 'refs/heads/main',
+ payload:pr ? {number:7, pull_request:{
+ number:7, head:{sha:head,repo:{full_name:repository}}, base:{sha:base,repo:{full_name:repository}},
+ }} : {},
+ };
+}
async function fixture(t: any) {
const path = await realpath(await mkdtemp(join(tmpdir(),'action-target-')));
@@ -18,9 +30,7 @@ async function fixture(t: any) {
const change = async (file = 'src/app.ts', text = 'export const n = 2;\n') => {
await writeFile(join(path,file),text); run('add','.'); run('commit','-m','change'); return run('rev-parse','HEAD');
};
- const event = (head: string, eventName = 'pull_request'): EventContext => ({eventName, repository:'example/repo',
- sha:head, ref:eventName === 'pull_request' ? 'refs/pull/7/merge' : 'refs/heads/main',serverUrl:'https://github.com',
- payload:{number:7,pull_request:{number:7,head:{sha:head,repo:{full_name:'example/repo'}},base:{sha:base,repo:{full_name:'example/repo'}}}}});
+ const event = (head: string, eventName?: string) => eventContext(head, eventName, base);
const inputs = (values: Record<string,string>={scope:'diff'}) => parseInputs(key=>values[key]??'',path);
return {path,run,base,change,event,inputs};
}
@@ -46,8 +56,7 @@ test('PR target resolves merge base and preserves exact upload head identity', a
assert.equal(target.diffBase,f.base); assert.equal(target.diffHead,head);
assert.equal(target.analysisRef,'refs/pull/7/head'); assert.equal(target.publishable,true);
});
-test('GitHub Cloud checkout origins match the event host for HTTPS and SSH', async t => {
- const f = await fixture(t); const head = await f.change();
+test('GitHub Cloud matches HTTPS, SSH and SCP origin spellings for each event host', () => {
for (const [host, sshUser] of [['github.com', 'git'], ['example-enterprise.ghe.com', 'example-enterprise']]) {
for (const origin of [
`https://${host}/example/repo.git`, `https://${host}:443/example/repo`,
@@ -55,58 +64,46 @@ test('GitHub Cloud checkout origins match the event host for HTTPS and SSH', asy
`${sshUser}@${host}:example/repo.git`, `${sshUser}@${host.toUpperCase()}:Example/Repo/`,
`ssh://${sshUser}@${host}/example/repo`, `ssh://${sshUser}@${host.toUpperCase()}:22/Example/Repo.git/`,
]) {
- f.run('remote', 'set-url', 'origin', origin);
- for (const eventName of ['schedule', 'pull_request']) {
- const event = {...f.event(head, eventName), serverUrl:`https://${host}`};
- const target = await resolveTarget(f.inputs({scope:eventName === 'schedule' ? 'repository' : 'diff'}), event);
- assert.equal(target.scannedSha, head); assert.equal(target.publishable, true);
- assert.equal(target.analysisRef, eventName === 'schedule' ? 'refs/heads/main' : 'refs/pull/7/head');
- assert.equal(target.diffBase, eventName === 'schedule' ? undefined : f.base);
- }
+ assert.equal(originMatches(origin, 'example/repo', `https://${host}`), true, origin);
}
}
});
-test('GitHub.com checkout origins support its SSH-over-443 endpoint', async t => {
- const f = await fixture(t);
- f.run('remote', 'set-url', 'origin', 'ssh://[email protected]:443/example/repo.git');
- const target = await resolveTarget(f.inputs({scope:'repository'}), f.event(f.base, 'schedule'));
- assert.equal(target.scannedSha, f.base);
- assert.equal(target.publishable, true);
+test('target resolution checks the configured SSH origin against the triggering host and repository', async t => {
+ const f = await fixture(t); const head = await f.change();
+ const enterpriseEvent = {...f.event(head), serverUrl:'https://example-enterprise.ghe.com'};
+ f.run('remote', 'set-url', 'origin', '[email protected]:example/repo.git');
+ const pr = await resolveTarget(f.inputs(), enterpriseEvent);
+ assert.equal(pr.scannedSha, head); assert.equal(pr.publishable, true);
+ assert.equal(pr.analysisRef, 'refs/pull/7/head'); assert.equal(pr.diffBase, f.base);
+ const schedule = f.event(head, 'schedule');
+ await assert.rejects(resolveTarget(f.inputs({scope:'repository'}), schedule), /origin does not match/);
- const enterpriseEvent = {...f.event(f.base), serverUrl:'https://example-enterprise.ghe.com'};
+ f.run('remote', 'set-url', 'origin', 'ssh://[email protected]:443/example/repo.git');
+ const target = await resolveTarget(f.inputs({scope:'repository'}), schedule);
+ assert.equal(target.scannedSha, head); assert.equal(target.publishable, true);
+ assert.equal(target.analysisRef, 'refs/heads/main'); assert.equal(target.diffBase, undefined);
await assert.rejects(resolveTarget(f.inputs(), enterpriseEvent), /origin does not match/);
- for (const origin of ['https://ssh.github.com/example/repo.git', 'ssh://[email protected]:22/example/repo.git',
- 'ssh://[email protected]:443/other/repo.git', 'ssh://[email protected]:443/example/repo.git']) {
- f.run('remote', 'set-url', 'origin', origin);
- await assert.rejects(resolveTarget(f.inputs(), f.event(f.base)), /origin does not match/);
- }
+ await assert.rejects(resolveTarget(f.inputs({scope:'repository'}), {...schedule, repository:'example/other'}), /origin does not match/);
});
-test('SCP checkout paths remain literal when compared with the triggering repository', async t => {
- const f = await fixture(t);
- for (const origin of ['[email protected]:example/repo.git#suffix', '[email protected]:example/repo.git?suffix',
+test('origins must match the event host, SSH endpoint and literal repository path', () => {
+ for (const origin of ['https://ssh.github.com/example/repo.git', 'ssh://[email protected]:22/example/repo.git',
+ 'ssh://[email protected]:443/other/repo.git', 'ssh://[email protected]:443/example/repo.git',
+ '[email protected]:example/repo.git#suffix', '[email protected]:example/repo.git?suffix',
'[email protected]:other/../example/repo.git']) {
- f.run('remote', 'set-url', 'origin', origin);
- await assert.rejects(resolveTarget(f.inputs(), f.event(f.base)), /origin does not match/);
+ assert.equal(originMatches(origin, 'example/repo', 'https://github.com'), false, origin);
}
-});
-test('unsupported server URLs still fail eligibility', async t => {
- const f = await fixture(t);
- for (const serverUrl of ['', 'https://github.example.invalid', 'http://example-enterprise.ghe.com',
- 'https://ghe.com', 'https://example-enterprise.ghe.com.example.invalid', 'https://example-enterprise-ghe.com']) {
- assert.throws(() => validateEvent({...f.event(f.base), serverUrl}), /supports GitHub.com and GitHub Enterprise Cloud/);
- }
-});
-test('enterprise checkout origins cannot select another host or repository', async t => {
- const f = await fixture(t);
- const event = {...f.event(f.base), serverUrl:'https://example-enterprise.ghe.com'};
for (const origin of ['https://github.com/example/repo.git', 'https://other-enterprise.ghe.com/example/repo.git',
'[email protected]:example/repo.git', 'https://example-enterprise.ghe.com/other/repo.git',
'[email protected]:other/repo.git']) {
- f.run('remote', 'set-url', 'origin', origin);
- await assert.rejects(resolveTarget(f.inputs(), event), /origin does not match/);
+ assert.equal(originMatches(origin, 'example/repo', 'https://example-enterprise.ghe.com'), false, origin);
+ }
+});
+test('unsupported server URLs fail eligibility', () => {
+ const event = eventContext();
+ for (const serverUrl of ['', 'https://github.example.invalid', 'http://example-enterprise.ghe.com',
+ 'https://ghe.com', 'https://example-enterprise.ghe.com.example.invalid', 'https://example-enterprise-ghe.com']) {
+ assert.throws(() => validateEvent({...event, serverUrl}), /supports GitHub.com and GitHub Enterprise Cloud/, serverUrl);
}
- f.run('remote', 'set-url', 'origin', 'https://example-enterprise.ghe.com/example/repo.git');
- await assert.rejects(resolveTarget(f.inputs(), f.event(f.base)), /origin does not match/);
});
test('manual diff scans resolve an explicit base against the checked-out head', async t => {
const f=await fixture(t);
@@ -129,7 +126,7 @@ test('large diffs are nonempty without collecting every changed pathname', async
f.run('reset', '--hard', head);
for (const eventName of ['pull_request', 'workflow_dispatch']) {
const event = f.event(head, eventName);
- event.payload.pull_request.base.sha = base;
+ if (eventName === 'pull_request') event.payload.pull_request.base.sha = base;
const inputs = f.inputs(eventName === 'pull_request' ? {scope:'diff'} : {scope:'diff', 'diff-base':base});
const target = await resolveTarget(inputs, event);
assert.equal(target.diffBase, base);
@@ -169,8 +166,8 @@ test('wrong checkout, local changes, missing diff base and option revisions are
await assert.rejects(resolveTarget(f.inputs(),f.event(head)),/local changes/);
await assert.rejects(resolveTarget(f.inputs({scope:'repository'}),f.event(head,'schedule')),/local changes/);
});
-test('forks and privileged events fail eligibility',async t=>{
- const f=await fixture(t); const event=f.event(f.base);
+test('forks and privileged events fail eligibility',()=>{
+ const event=eventContext();
event.payload.pull_request.head.repo.full_name='fork/repo'; assert.throws(()=>validateEvent(event),/Fork PR/);
for(const eventName of ['pull_request_target','workflow_run']) assert.throws(()=>validateEvent({...event,eventName}),/Unsupported event/);
});