OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
import type { createFixture } from "./fixtures.mts";
type Fixture = ReturnType<typeof createFixture>;
type Expected = Fixture["positives"][number];
export interface SourceLocation {
path: string;
startLine: number;
endLine?: number | null;
role?: string;
}
export interface CodeEvidence {
path: string;
startLine: number;
code?: string;
[key: string]: unknown;
}
export interface Finding {
taxonomy?: { category?: string; cwe?: string[] };
locations?: SourceLocation[];
codeEvidence?: CodeEvidence[];
[key: string]: unknown;
}
export interface Result {
findings?: Finding[];
[key: string]: unknown;
}
const explicitSecretTerms =
/secrets?|credentials?|passwords?|tokens?|(?:api|private)[-_ ]?keys?|hard[-_ ]?cod(?:e[ds]?|ing)?/i;
const secretTerms = `${explicitSecretTerms.source}|sensitive[-_ ]?(?:data|information)[-_ ]?exposures?`;
const explicitSecretCategory = new RegExp(
String.raw`\b(?:${explicitSecretTerms.source})\b`,
"i",
);
const secretCategory = new RegExp(String.raw`\b(?:${secretTerms})\b`, "i");
const negatedSecretCategory = new RegExp(
String.raw`^(?:(?:not(?:[-_\s]*a)?|non|no|without)[-_\s]*(?:${secretTerms})\b|(?:${secretTerms})[-_\s]*free\b)`,
"i",
);
const supportingLocation = (location: SourceLocation) =>
/^(?:supporting|support|context|consumer|expected_control)$/.test(
location.role ?? "",
);
function normalizeSourcePath(path: string | undefined) {
const normalized = path?.replaceAll("\\", "/").replace(/^(?:\.\/)+/u, "");
return process.platform === "win32" ? normalized?.toLowerCase() : normalized;
}
function sourceText(path: string, fixture: Fixture) {
return Object.entries(fixture.files).find(
([sourcePath]) =>
normalizeSourcePath(sourcePath) === normalizeSourcePath(path),
)?.[1];
}
function validSourceRange(location: SourceLocation, fixture: Fixture) {
const source = sourceText(location.path, fixture);
const end = location.endLine ?? location.startLine;
return (
typeof source === "string" &&
Number.isInteger(location.startLine) &&
Number.isInteger(end) &&
location.startLine > 0 &&
end >= location.startLine &&
end <= source.trimEnd().split("\n").length
);
}
function matchesLocation(location: SourceLocation, expected: Expected) {
const end = location.endLine ?? location.startLine;
return (
normalizeSourcePath(location.path) === normalizeSourcePath(expected.path) &&
!supportingLocation(location) &&
Number.isInteger(location.startLine) &&
Number.isInteger(end) &&
location.startLine > 0 &&
location.startLine <= expected.endLine &&
end >= location.startLine &&
end >= expected.line &&
end <= expected.lineCount
);
}
function matches(finding: Finding, expected: Expected) {
const category = (finding.taxonomy?.category ?? "")
.replaceAll("_", " ")
.replace(/([a-z])([A-Z])/g, "$1 $2")
.trim();
return (
!negatedSecretCategory.test(category) &&
secretCategory.test(category) &&
finding.taxonomy?.cwe?.some((value) => {
const cwe = value.trim().replace(/^cwe-0*(\d+)$/i, "CWE-$1");
return (
expected.cwes.includes(cwe) ||
(["CWE-200", "CWE-540"].includes(cwe) &&
explicitSecretCategory.test(category))
);
}) &&
finding.locations?.some((location) => matchesLocation(location, expected))
);
}
function matchesConsumer(location: SourceLocation, expected: Expected) {
return (
expected.consumerLine !== null &&
matchesLocation(location, {
...expected,
line: expected.consumerLine,
endLine: expected.consumerLine,
})
);
}
function validEvidence(finding: Finding, fixture: Fixture) {
const evidence = (finding.codeEvidence ?? []).map((entry) => {
const code = entry.code?.replaceAll("\r\n", "\n").replace(/\n$/u, "");
return {
...entry,
code,
endLine: entry.startLine + (code?.split("\n").length ?? 0) - 1,
};
});
return (
evidence.length > 0 &&
evidence.every(
(entry) =>
Boolean(entry.code?.trim()) &&
validSourceRange(entry, fixture) &&
entry.code ===
sourceText(entry.path, fixture)!
.replaceAll("\r\n", "\n")
.split("\n")
.slice(entry.startLine - 1, entry.endLine)
.join("\n"),
) &&
fixture.positives
.filter((expected) => matches(finding, expected))
.every((expected) =>
evidence.some(
(entry) =>
matchesLocation(entry, expected) ||
matchesConsumer(entry, expected),
),
)
);
}
/** Grade retained final findings, never keyword mentions or deferred candidates. */
export function gradeResult(result: Result | undefined, fixture: Fixture) {
const errors = [];
const findings = Array.isArray(result?.findings) ? result.findings : [];
if (!Array.isArray(result?.findings)) errors.push("missing findings array");
const cases = fixture.positives.map((expected) => ({
id: expected.id,
found: findings.some((finding) => matches(finding, expected)),
}));
const falsePositives = findings.flatMap<{
index: number;
unexpectedLocations?: string[];
reason?: string;
}>((finding, index) => {
const matchedCases = fixture.positives.filter((expected) =>
matches(finding, expected),
);
const unexpectedLocations = (finding.locations ?? [])
.filter(
(location) =>
!validSourceRange(location, fixture) ||
(!supportingLocation(location) &&
!matchedCases.some(
(expected) =>
matchesLocation(location, expected) ||
(location.role === "sink" &&
matchesConsumer(location, expected)),
)),
)
.map((location) => location.path);
if (unexpectedLocations.length) return [{ index, unexpectedLocations }];
const cwes = finding.taxonomy?.cwe ?? [];
if (
cwes.length === 0 ||
cwes.some(
(cwe) =>
!fixture.positives.some((expected) =>
matches(
{ ...finding, taxonomy: { ...finding.taxonomy, cwe: [cwe] } },
expected,
),
),
)
) {
return [{ index, reason: "no expected secret location and taxonomy" }];
}
return [];
});
const found = cases.filter((entry) => entry.found).length;
if (found !== cases.length) errors.push("missing retained secret findings");
if (falsePositives.length)
errors.push("false positives or incorrect taxonomy/locations");
if (findings.some((finding) => !validEvidence(finding, fixture)))
errors.push("missing or invalid code evidence");
return {
passed: errors.length === 0,
cases,
recall: found / cases.length,
falsePositiveCount: falsePositives.length,
falsePositives,
errors,
};
}