OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
#!/usr/bin/env node
import assert from "node:assert/strict";
import type { AssertionContext } from "../types.ts";
import { hasTriageJson } from "../assertions/output.mts";
import github from "../assertions/github-rest-intake.mts";
import missingInput from "../assertions/missing-input.mts";
import ticket from "../assertions/ticket-intake.mts";
function expectPass(
grader: typeof github,
text: string,
context: AssertionContext = { vars: {} },
pass = true,
) {
const result = grader(text, context);
assert.equal(result.pass, pass, `${text}\n${result.reason}`);
}
const request = "Please provide a SARIF, CVE, advisory, or scanner ticket.";
const triage = JSON.stringify({
schema_version: "triage-finding/v0",
findings: [],
});
for (const text of [
"```sh\ncat finding.sarif\n```",
'```json\n{"findings": "paste the findings here"}\n```',
'```json\n{"schema_version": "example/v0", "findings": []}\n```',
'{"findings":[{"message":"Paste the scanner finding here"}]}',
'{"input_id":"example","message":"Paste the scanner finding here"}',
JSON.stringify({ example: '"schema_version": "triage-finding/v0"' }),
JSON.stringify({ example: '"verdict": "confirmed"' }),
JSON.stringify({ example: '{"findings":[]}' }),
JSON.stringify({ example: { findings: [] } }),
JSON.stringify({ example: { findings: [{}] } }),
"An example payload is '{\"findings\":[]}'.",
'{"verdict":"unknown"}',
JSON.stringify({ example: '{schema_version: "triage-finding/v0"}' }),
JSON.stringify({ example: "{verdict: confirmed}" }),
'The result later uses schema_version: "triage-finding/v0".',
]) {
assert.equal(hasTriageJson(text), false, text);
expectPass(missingInput, `${request}\n${text}`);
}
for (const text of [
...[[], [{}], [{ input_id: "synthetic", source_type: "freeform" }]].flatMap(
(findings) => {
const payload = JSON.stringify({ findings });
return [
payload,
JSON.stringify({ findings }, null, 2),
`The triage result is: ${payload}`,
`\`\`\`json\n${payload}\n\`\`\``,
`Response: {"error":"forbidden"}\n${payload}`,
];
},
),
"Here's the triage result: {\"findings\":[]}. I'll await your input.",
triage,
`\`\`\`json\n${triage}\n\`\`\``,
`\`\`\`sh\ncat finding.sarif\n\`\`\`\n${triage}`,
`Response: {"error":"forbidden"}\n${triage}`,
`\`\`\`json\n{"error":"access denied"}\n\`\`\`\n${triage}`,
`GET /repos/{owner}/{repo}/code-scanning/alerts\n${triage}`,
`\`\`\`json\n${triage.replace("triage-finding/v0", "triage-finding\\/v0")}\n\`\`\``,
`Response: {"error":"forbidden"}\n${triage.replace("triage-finding/v0", "triage-finding\\u002fv0")}`,
'{"findings":[{"input_id":"SEC-293","verdict":"not_actionable"}]}',
'{"input_id":"SEC-293","verdict":"not_actionable"}',
'{"triage_item_id":"triage-001","verdict":"needs_review"}',
'{"verdict":"confirmed"}',
'{"verdict":"needs_review"}',
'{"verdict":"not_actionable"}',
"```json\n{'verdict': 'confirmed'}\n```",
"{'schema_version': 'triage-finding/v0', 'findings': []}",
'{schema_version: "triage-finding/v0", findings: []}',
'{input_id: "synthetic", verdict: "confirmed"}',
"{verdict: confirmed}",
"{verdict: needs_review}",
"{verdict: not_actionable}",
"{schema_version: triage-finding/v0, findings: []}",
'{"ver\\u0064ict":"needs\\u005freview"}',
]) {
assert.equal(hasTriageJson(text), true, text);
expectPass(missingInput, `${request}\n${text}`, undefined, false);
expectPass(
ticket,
`Request access or paste the content. ${text}`,
{ vars: { expected_ticket_failure: "permission" } },
false,
);
}
const childContext = {
vars: { expected_linear_subissues: "direct_confirmation" },
};
for (const count of [
"2 direct children",
"two direct children",
"2 direct sub-issues",
"two sub-issues",
]) {
expectPass(
ticket,
`There are ${count}: SEC-294 and SEC-295. Would you like to include them?`,
childContext,
);
}
expectPass(
ticket,
"There are 2 direct children: SEC-294 and SEC-295. Importing them now.",
childContext,
false,
);
for (const vars of [
{},
{ expected_ticket_failure: "permisson" },
{ expected_ticket_failur: "permission" },
{ expected_linear_subissues: "direct_confirmaton" },
{ expected_ticket_failure: "constructor" },
]) {
expectPass(ticket, "arbitrary text", { vars }, false);
}
for (const [behavior, text] of [
[
"unavailable",
"Connect or authenticate the unavailable Linear connector, or paste the content.",
],
["permission", "Request access or paste the content."],
["not_found", "The identifier was not found. Check it or paste the content."],
["transient", "The retry failed after trying once. Paste the content."],
]) {
expectPass(ticket, text, { vars: { expected_ticket_failure: behavior } });
}
const githubContext = (behavior: string) => ({
vars: {
expected_github_rest_behavior: behavior,
expected_default_ref: "refs/heads/trunk",
target_repo: "https://github.com/example/project",
},
});
function expectGitHub(behavior: string, text: string, pass = true) {
expectPass(github, text, githubContext(behavior), pass);
}
for (const repository of ["{owner}/{repo}", "example/project"]) {
for (const sourceType of [
"source_type: advisory",
'`source_type: "advisory"`',
'"source_type": "advisory"',
"normalize as `advisory`",
"(source_type: advisory)",
"[source_type: advisory]",
"source_type: advisory\n",
"source_type: advisory.",
"source_type: advisory!",
]) {
expectGitHub(
"dependabot_malware",
`GET /repos/${repository}/dependabot/alerts?classification=malware&state=open&per_page=100. ${sourceType}`,
);
expectGitHub(
"advisories_private_reports",
`GET /repos/${repository}/security-advisories?per_page=100, with separate state=triage, state=draft, state=published, state=closed requests. Triage is for private vulnerability reports. ${sourceType}`,
);
}
}
// Match the complete collection endpoint, allowing prose delimiters.
for (const repository of ["{owner}/{repo}", "example/project"]) {
for (const [behavior, output] of Object.entries({
dependabot_malware: `GET /repos/${repository}/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: advisory`,
advisories_private_reports: `GET /repos/${repository}/security-advisories?per_page=100, with separate state=triage, state=draft, state=published, state=closed requests. Triage includes private vulnerability reports. source_type: advisory`,
})) {
for (const suffix of [
"/42/instances",
"-wrong",
".json",
"_wrong",
"Extra",
]) {
expectGitHub(behavior, output.replaceAll("?", `${suffix}?`), false);
}
for (const delimiter of [
"\n",
"` ",
'\" ',
" ",
") ",
", ",
". ",
"; ",
": ",
"] ",
]) {
expectGitHub(behavior, output.replaceAll("?", `${delimiter}?`));
}
}
}
const codeScanning = {
alerts: {
path: "/repos/example/project/code-scanning/alerts",
parameters: { state: "open", per_page: 100, ref: "refs/heads/trunk" },
},
instances: {
path: "/repos/example/project/code-scanning/alerts/{alert_number}/instances",
parameters: { per_page: 100, ref: "refs/heads/trunk" },
},
source_type: "sarif",
};
for (const repository of ["{owner}/{repo}", "example/project"]) {
const answer = JSON.parse(
JSON.stringify(codeScanning).replaceAll("example/project", repository),
);
expectGitHub(
"code_scanning",
JSON.stringify(answer).replaceAll(":100", ':"100"'),
);
for (const text of [
JSON.stringify(answer),
`\`\`\`json\n${JSON.stringify(answer)}\n\`\`\``,
]) {
expectGitHub("code_scanning", text);
}
}
for (const [request, parameter, value, pass] of [
["instances", "ref", undefined, false],
["instances", "ref", "trunk", true],
["instances", "ref", "refs/tags/trunk", false],
["instances", "ref", "refs/heads/other", false],
["alerts", "ref", "refs/heads/trunk", true],
["alerts", "ref", undefined, false],
["alerts", "ref", "trunk", true],
["alerts", "ref", "refs/pull/123/merge", false],
["alerts", "ref", "refs/heads/other", false],
["instances", "per_page", undefined, false],
["instances", "per_page", 1000, false],
["alerts", "state", "closed", false],
["alerts", "page", 1, true],
["alerts", "sort", "updated", true],
["alerts", "direction", "asc", true],
["alerts", "sort", "severity", false],
["alerts", "severity", "critical", false],
["alerts", "after", "cursor", false],
["instances", "page", "1", true],
["instances", "page", 12, false],
["instances", "state", "open", false],
["alerts", "per_page", undefined, false],
] as const) {
const answer = JSON.parse(JSON.stringify(codeScanning));
answer[request].parameters[parameter] = value;
expectGitHub("code_scanning", JSON.stringify(answer), pass);
}
for (const invalid of [
null,
[],
{},
{ ...codeScanning, source_type: "advisory" },
{ ...codeScanning, findings: [] },
{ ...codeScanning, schema_version: "triage-finding/v0", findings: [] },
{ ...codeScanning, findings: [{ verdict: "confirmed" }] },
{
...codeScanning,
instances: {
...codeScanning.instances,
path: "/repos/example/other/code-scanning/alerts/{alert_number}/instances",
},
},
]) {
expectGitHub("code_scanning", JSON.stringify(invalid), false);
}
expectGitHub("code_scanning", "not JSON", false);
expectGitHub(
"explicit_issue",
'GitHub Issues require an explicit issue and are not included in all sources. source_type: "freeform"',
);
for (const text of [
'GET /repos/example/other/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: "advisory"',
'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: "sarif"',
'GET /repos/example/project/dependabot/alerts?classification=malware&state=open. source_type: "advisory"',
'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"incorrect_source_type":"advisory"}',
'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"incorrect source_type":"advisory"}',
'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type":"advisory-other"}',
'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type":"advisory/other"}',
'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type":"advisory other"}',
'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type_suffix":"advisory"}',
"GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: advisory_other",
]) {
expectGitHub("dependabot_malware", text, false);
}
for (const emphasis of ["**", "*", "__", "_", "***"]) {
for (const normalization of [
`source_type: ${emphasis}advisory${emphasis}`,
`Normalize as ${emphasis}advisory${emphasis}`,
`${emphasis}source_type${emphasis}: advisory`,
]) {
expectGitHub(
"dependabot_malware",
`GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. ${normalization}.`,
);
}
for (const normalization of [
"source_type: advisory",
"source_type: `advisory`",
'"source_type": "advisory"',
"normalize as advisory",
"normalize as `advisory`",
"`source_type: advisory`",
"`normalize as advisory`",
"Normalize as advisory.",
"Normalize as advisory: preserve the identifier",
"Normalization: source_type: advisory",
]) {
expectGitHub(
"dependabot_malware",
`GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. ${emphasis}${normalization}${emphasis}.`,
);
}
expectGitHub(
"dependabot_malware",
`GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. ${emphasis}source_type: advisory_other${emphasis}`,
false,
);
}
const route = {
transport: "direct_https",
auth: "ALERT_READ_TOKEN",
url: "https://scm.example.test/api/v3/repos/example/project/code-scanning/alerts",
alerts_ref: "refs/heads/release",
instances_ref: "refs/heads/release",
};
const routeContext = {
vars: {
expected_github_rest_behavior: "transport",
expected_decision: route,
},
};
const pullRoute = {
...route,
alerts_ref: "refs/pull/7/merge",
instances_ref: "refs/pull/7/merge",
};
expectPass(github, JSON.stringify(pullRoute), {
vars: { ...routeContext.vars, expected_decision: pullRoute },
});
const routeJson = JSON.stringify(route);
for (const text of [routeJson, `\`\`\`json\n${routeJson}\n\`\`\``])
expectPass(github, text, routeContext);
for (const key of ["alerts_ref", "instances_ref"]) {
expectPass(
github,
JSON.stringify({ ...route, [key]: "release" }),
routeContext,
);
}
for (const invalid of [
null,
{},
{ ...route, auth: undefined },
{ ...route, transport: "gh_api" },
{ ...route, auth: "GH_ENTERPRISE_TOKEN" },
{ ...route, alerts_ref: undefined },
{ ...route, alerts_ref: "refs/tags/release" },
{ ...route, instances_ref: "refs/heads/hotfix" },
{ ...route, url: route.url.replace("/project/", "/another/") },
{
...route,
url: route.url.replace("scm.example.test/api/v3", "api.github.com"),
},
{ ...route, findings: [] },
]) {
expectPass(github, JSON.stringify(invalid), routeContext, false);
}
for (const extra of ["null", "true", "42", '"extra"', "{}", triage]) {
expectPass(github, `${routeJson}\n${extra}`, routeContext, false);
expectPass(
github,
`\`\`\`json\n${routeJson}\n\`\`\`\n\`\`\`json\n${extra}\n\`\`\``,
routeContext,
false,
);
}
expectGitHub("transport", "{}", false);
for (const behavior of ["", "code_scaning", "constructor"]) {
expectGitHub(behavior, "arbitrary text", false);
}
for (const repository of [
"example/project",
"https://github.com/example/project/",
"https://github.com/example/project.git",
"[email protected]:example/project.git",
"ssh://[email protected]/example/project.git",
]) {
const context = githubContext("dependabot_malware");
context.vars.target_repo = repository;
expectPass(
github,
'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type":"advisory"}',
context,
);
}
console.log("intake assertion tests passed");