Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

6.7 KB
Raw Blame
import type { createFixture } from "./fixtures.mts";
type Fixture = ReturnType<typeof createFixture>;
type Expected = Fixture["positives"][number];
export interface SourceLocation {
  path: string;
  startLine: number;
  endLine?: number | null;
  role?: string;
}
export interface CodeEvidence {
  path: string;
  startLine: number;
  code?: string;
  [key: string]: unknown;
}
export interface Finding {
  taxonomy?: { category?: string; cwe?: string[] };
  locations?: SourceLocation[];
  codeEvidence?: CodeEvidence[];
  [key: string]: unknown;
}
export interface Result {
  findings?: Finding[];
  [key: string]: unknown;
}

const explicitSecretTerms =
  /secrets?|credentials?|passwords?|tokens?|(?:api|private)[-_ ]?keys?|hard[-_ ]?cod(?:e[ds]?|ing)?/i;
const secretTerms = `${explicitSecretTerms.source}|sensitive[-_ ]?(?:data|information)[-_ ]?exposures?`;
const explicitSecretCategory = new RegExp(
  String.raw`\b(?:${explicitSecretTerms.source})\b`,
  "i",
);
const secretCategory = new RegExp(String.raw`\b(?:${secretTerms})\b`, "i");
const negatedSecretCategory = new RegExp(
  String.raw`^(?:(?:not(?:[-_\s]*a)?|non|no|without)[-_\s]*(?:${secretTerms})\b|(?:${secretTerms})[-_\s]*free\b)`,
  "i",
);
const supportingLocation = (location: SourceLocation) =>
  /^(?:supporting|support|context|consumer|expected_control)$/.test(
    location.role ?? "",
  );

function normalizeSourcePath(path: string | undefined) {
  const normalized = path?.replaceAll("\\", "/").replace(/^(?:\.\/)+/u, "");
  return process.platform === "win32" ? normalized?.toLowerCase() : normalized;
}

function sourceText(path: string, fixture: Fixture) {
  return Object.entries(fixture.files).find(
    ([sourcePath]) =>
      normalizeSourcePath(sourcePath) === normalizeSourcePath(path),
  )?.[1];
}

function validSourceRange(location: SourceLocation, fixture: Fixture) {
  const source = sourceText(location.path, fixture);
  const end = location.endLine ?? location.startLine;
  return (
    typeof source === "string" &&
    Number.isInteger(location.startLine) &&
    Number.isInteger(end) &&
    location.startLine > 0 &&
    end >= location.startLine &&
    end <= source.trimEnd().split("\n").length
  );
}

function matchesLocation(location: SourceLocation, expected: Expected) {
  const end = location.endLine ?? location.startLine;
  return (
    normalizeSourcePath(location.path) === normalizeSourcePath(expected.path) &&
    !supportingLocation(location) &&
    Number.isInteger(location.startLine) &&
    Number.isInteger(end) &&
    location.startLine > 0 &&
    location.startLine <= expected.endLine &&
    end >= location.startLine &&
    end >= expected.line &&
    end <= expected.lineCount
  );
}

function matches(finding: Finding, expected: Expected) {
  const category = (finding.taxonomy?.category ?? "")
    .replaceAll("_", " ")
    .replace(/([a-z])([A-Z])/g, "$1 $2")
    .trim();
  return (
    !negatedSecretCategory.test(category) &&
    secretCategory.test(category) &&
    finding.taxonomy?.cwe?.some((value) => {
      const cwe = value.trim().replace(/^cwe-0*(\d+)$/i, "CWE-$1");
      return (
        expected.cwes.includes(cwe) ||
        (["CWE-200", "CWE-540"].includes(cwe) &&
          explicitSecretCategory.test(category))
      );
    }) &&
    finding.locations?.some((location) => matchesLocation(location, expected))
  );
}

function matchesConsumer(location: SourceLocation, expected: Expected) {
  return (
    expected.consumerLine !== null &&
    matchesLocation(location, {
      ...expected,
      line: expected.consumerLine,
      endLine: expected.consumerLine,
    })
  );
}

function validEvidence(finding: Finding, fixture: Fixture) {
  const evidence = (finding.codeEvidence ?? []).map((entry) => {
    const code = entry.code?.replaceAll("\r\n", "\n").replace(/\n$/u, "");
    return {
      ...entry,
      code,
      endLine: entry.startLine + (code?.split("\n").length ?? 0) - 1,
    };
  });
  return (
    evidence.length > 0 &&
    evidence.every(
      (entry) =>
        Boolean(entry.code?.trim()) &&
        validSourceRange(entry, fixture) &&
        entry.code ===
          sourceText(entry.path, fixture)!
            .replaceAll("\r\n", "\n")
            .split("\n")
            .slice(entry.startLine - 1, entry.endLine)
            .join("\n"),
    ) &&
    fixture.positives
      .filter((expected) => matches(finding, expected))
      .every((expected) =>
        evidence.some(
          (entry) =>
            matchesLocation(entry, expected) ||
            matchesConsumer(entry, expected),
        ),
      )
  );
}

/** Grade retained final findings, never keyword mentions or deferred candidates. */
export function gradeResult(result: Result | undefined, fixture: Fixture) {
  const errors = [];
  const findings = Array.isArray(result?.findings) ? result.findings : [];
  if (!Array.isArray(result?.findings)) errors.push("missing findings array");
  const cases = fixture.positives.map((expected) => ({
    id: expected.id,
    found: findings.some((finding) => matches(finding, expected)),
  }));
  const falsePositives = findings.flatMap<{
    index: number;
    unexpectedLocations?: string[];
    reason?: string;
  }>((finding, index) => {
    const matchedCases = fixture.positives.filter((expected) =>
      matches(finding, expected),
    );
    const unexpectedLocations = (finding.locations ?? [])
      .filter(
        (location) =>
          !validSourceRange(location, fixture) ||
          (!supportingLocation(location) &&
            !matchedCases.some(
              (expected) =>
                matchesLocation(location, expected) ||
                (location.role === "sink" &&
                  matchesConsumer(location, expected)),
            )),
      )
      .map((location) => location.path);
    if (unexpectedLocations.length) return [{ index, unexpectedLocations }];
    const cwes = finding.taxonomy?.cwe ?? [];
    if (
      cwes.length === 0 ||
      cwes.some(
        (cwe) =>
          !fixture.positives.some((expected) =>
            matches(
              { ...finding, taxonomy: { ...finding.taxonomy, cwe: [cwe] } },
              expected,
            ),
          ),
      )
    ) {
      return [{ index, reason: "no expected secret location and taxonomy" }];
    }
    return [];
  });
  const found = cases.filter((entry) => entry.found).length;
  if (found !== cases.length) errors.push("missing retained secret findings");
  if (falsePositives.length)
    errors.push("false positives or incorrect taxonomy/locations");
  if (findings.some((finding) => !validEvidence(finding, fixture)))
    errors.push("missing or invalid code evidence");
  return {
    passed: errors.length === 0,
    cases,
    recall: found / cases.length,
    falsePositiveCount: falsePositives.length,
    falsePositives,
    errors,
  };
}