Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

13.9 KB
Raw Blame
#!/usr/bin/env node
import type {
  SastBenchRecord,
  SastLocation,
  DatasetExpectations,
  SampleSpec,
} from "../../types.ts";

type SampleEntry = { record: SastBenchRecord; index: number };
import { hash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";

export const EVAL_ROOT = path.resolve(import.meta.dirname, "..", "..");
export const SASTBENCH_REPOSITORY_URL =
  "https://github.com/RivalSecurity/sastbench.git";
export const SASTBENCH_COMMIT = "b8d95b7720491b21d039b7504999577ba7ebb12b";
export const SASTBENCH_DATASET_RELATIVE_PATH = path.join(
  "data",
  "sastbench-v0.1-original.json",
);
export const SASTBENCH_DATASET_SHA256 =
  "57c765633bdefc2130d3725dd9648bc49b85456e4dd733c55f49d44d883c9487";
export const EXPECTED_CASE_COUNT = 2737;
export const EXPECTED_LABEL_COUNTS = Object.freeze({
  true_positive: 299,
  false_positive: 2438,
});
export const REPRESENTATIVE_SAMPLE_SPEC = Object.freeze({
  profile: "representative-100-v1",
  seed: "sastbench-representative-v1",
  labelCounts: Object.freeze({
    true_positive: 40,
    false_positive: 60,
  }),
});
export const DEFAULT_INSTALL_ROOT = path.join(
  EVAL_ROOT,
  "artifacts",
  "sastbench",
);
export const DEFAULT_DATASET_PATH = path.join(
  DEFAULT_INSTALL_ROOT,
  SASTBENCH_DATASET_RELATIVE_PATH,
);
export const DEFAULT_TARGET_ROOT = path.join(
  EVAL_ROOT,
  "artifacts",
  "sastbench-targets",
);
const ALLOWED_LABELS = new Set(Object.keys(EXPECTED_LABEL_COUNTS));
const REQUIRED_ANALYZER_FIELDS = [
  "vulnerability_type",
  "vulnerability_name",
  "description",
  "locations",
  "commit_context",
];

export function loadDataset(filename: string = DEFAULT_DATASET_PATH) {
  const records: SastBenchRecord[] = JSON.parse(
    fs.readFileSync(filename, "utf8"),
  );
  validateDataset(records);
  return records;
}

function requireNonEmptyString(
  value: unknown,
  field: string,
  index: number | string,
) {
  if (typeof value !== "string" || value.trim().length === 0) {
    throw new Error(`SastBench case ${index} requires non-empty ${field}`);
  }
}

function requireString(value: unknown, field: string, index: number | string) {
  if (typeof value !== "string") {
    throw new Error(`SastBench case ${index} requires string ${field}`);
  }
}

function validateLocation(
  location: SastLocation,
  caseIndex: number,
  locationIndex: number,
) {
  if (!location || typeof location !== "object" || Array.isArray(location)) {
    throw new Error(
      `SastBench case ${caseIndex} location ${locationIndex} must be an object`,
    );
  }
  requireNonEmptyString(
    location.file,
    `locations[${locationIndex}].file`,
    caseIndex,
  );
  // SastBench preserves scanner-native locations. One finding uses line 0 to
  // mean file-level scope, and some zero-width findings encode end = start - 1.
  if (!Number.isInteger(location.line_start) || location.line_start < 0) {
    throw new Error(
      `SastBench case ${caseIndex} location ${locationIndex} requires line_start`,
    );
  }
  if (!Number.isInteger(location.line_end) || location.line_end < 0) {
    throw new Error(
      `SastBench case ${caseIndex} location ${locationIndex} requires line_end`,
    );
  }
}

function validateRecord(record: SastBenchRecord, index: number) {
  if (!record || typeof record !== "object" || Array.isArray(record)) {
    throw new Error(`SastBench case ${index} must be an object`);
  }
  requireNonEmptyString(record.repo_name, "repo_name", index);
  requireNonEmptyString(record.repo_url, "repo_url", index);
  requireNonEmptyString(record.commit_hash, "commit_hash", index);
  if (!/^[a-f0-9]{40}$/i.test(record.commit_hash)) {
    throw new Error(
      `SastBench case ${index} commit_hash must be a 40-character Git SHA`,
    );
  }
  if (!ALLOWED_LABELS.has(record.ground_truth)) {
    throw new Error(
      `SastBench case ${index} has unsupported ground_truth: ${record.ground_truth}`,
    );
  }
  if (!record.to_analyzer || typeof record.to_analyzer !== "object") {
    throw new Error(`SastBench case ${index} requires to_analyzer`);
  }
  for (const field of REQUIRED_ANALYZER_FIELDS) {
    if (!(field in record.to_analyzer)) {
      throw new Error(`SastBench case ${index} requires to_analyzer.${field}`);
    }
  }
  requireNonEmptyString(
    record.to_analyzer.vulnerability_type,
    "to_analyzer.vulnerability_type",
    index,
  );
  for (const field of ["vulnerability_name", "description"]) {
    requireString(record.to_analyzer[field], `to_analyzer.${field}`, index);
  }
  if (
    !Array.isArray(record.to_analyzer.locations) ||
    record.to_analyzer.locations.length === 0
  ) {
    throw new Error(
      `SastBench case ${index} requires at least one to_analyzer location`,
    );
  }
  record.to_analyzer.locations.forEach(
    (location: SastLocation, locationIndex: number) =>
      validateLocation(location, index, locationIndex),
  );
  const context = record.to_analyzer.commit_context;
  if (!context || typeof context !== "object") {
    throw new Error(
      `SastBench case ${index} requires to_analyzer.commit_context`,
    );
  }
  if (context.commit !== record.commit_hash) {
    throw new Error(
      `SastBench case ${index} commit_context.commit does not match commit_hash`,
    );
  }
}

export function validateDataset(
  records: SastBenchRecord[],
  expectations: DatasetExpectations = {
    caseCount: EXPECTED_CASE_COUNT,
    labelCounts: EXPECTED_LABEL_COUNTS,
  },
) {
  if (!Array.isArray(records)) {
    throw new Error("SastBench dataset must be a JSON array");
  }
  records.forEach(validateRecord);
  const counts: Record<string, number> = {
    true_positive: 0,
    false_positive: 0,
  };
  for (const record of records) {
    counts[record.ground_truth] += 1;
  }
  if (records.length !== expectations.caseCount) {
    throw new Error(
      `SastBench dataset case count mismatch: expected ${expectations.caseCount}, got ${records.length}`,
    );
  }
  for (const label of ALLOWED_LABELS) {
    if (counts[label] !== expectations.labelCounts[label]) {
      throw new Error(
        `SastBench ${label} count mismatch: expected ${expectations.labelCounts[label]}, got ${counts[label]}`,
      );
    }
  }
  return { caseCount: records.length, labelCounts: counts };
}

export function repositoryStateId(
  record: Pick<SastBenchRecord, "repo_url" | "commit_hash">,
) {
  return `state-${hash("sha256", `${record.repo_url}\0${record.commit_hash}`).slice(0, 16)}`;
}

function formatLocation(location: SastLocation) {
  const functionName = location.function ? ` (${location.function})` : "";
  return `${location.file}:${location.line_start}-${location.line_end}${functionName}`;
}

function buildFindingInput(record: SastBenchRecord, caseId: string) {
  const analyzer = record.to_analyzer;
  const locations = analyzer.locations.map(
    (location: SastLocation) => `   - ${formatLocation(location)}`,
  );
  const title = [analyzer.vulnerability_type, analyzer.vulnerability_name]
    .filter((part) => part.trim().length > 0)
    .join(" ");
  const findingLines = [
    "Source type: scanner_ticket",
    `1. input_id: ${caseId}`,
    `   title: ${title}`,
    `   weakness: ${analyzer.vulnerability_type}`,
  ];
  if (analyzer.description.trim().length > 0) {
    findingLines.push(`   claim: ${analyzer.description}`);
  }
  findingLines.push(
    `   repository: ${analyzer.commit_context.repo}`,
    `   repository revision: ${analyzer.commit_context.commit}`,
    "   scanner locations:",
    ...locations,
  );
  return findingLines.join("\n");
}

export function buildPromptVars(
  record: SastBenchRecord,
  index: number,
  targetRoot: string | undefined = DEFAULT_TARGET_ROOT,
) {
  const caseId = `sastbench-${String(index).padStart(6, "0")}`;
  return {
    case_id: caseId,
    target_repo: path.join(path.resolve(targetRoot), repositoryStateId(record)),
    finding_input: buildFindingInput(record, caseId),
  };
}

function buildPromptfooTest<Metadata extends Record<string, unknown>>(
  record: SastBenchRecord,
  index: number,
  targetRoot: string | undefined,
  metadata: Metadata,
) {
  const vars = buildPromptVars(record, index, targetRoot);
  return {
    description: `SastBench case ${vars.case_id}`,
    metadata: {
      case_id: vars.case_id,
      ground_truth: record.ground_truth,
      ...metadata,
    },
    vars,
  };
}

export function generatePromptfooTests(
  records: SastBenchRecord[],
  targetRoot: string | undefined = DEFAULT_TARGET_ROOT,
) {
  return records.map((record: SastBenchRecord, index: number) =>
    buildPromptfooTest(record, index, targetRoot, {}),
  );
}

function evidenceSizeBucket(record: SastBenchRecord) {
  const locationCount = record.to_analyzer.locations.length;
  if (locationCount === 1) {
    return "1";
  }
  if (locationCount <= 3) {
    return "2-3";
  }
  if (locationCount <= 10) {
    return "4-10";
  }
  return "11+";
}

function allocateSampleCounts(
  strata: { name: string; entries: SampleEntry[] }[],
  sampleCount: number,
) {
  if (sampleCount < strata.length) {
    throw new Error(
      `Representative sample count ${sampleCount} cannot cover ${strata.length} evidence strata`,
    );
  }
  const populationCount = strata.reduce(
    (total, stratum) => total + stratum.entries.length,
    0,
  );
  const allocations = strata.map((stratum) => ({ ...stratum, count: 1 }));
  const deficit = (stratum: (typeof allocations)[number]) =>
    (stratum.entries.length * sampleCount) / populationCount - stratum.count;
  let remaining = sampleCount - strata.length;

  while (remaining > 0) {
    const candidates = allocations
      .filter((stratum) => stratum.count < stratum.entries.length)
      .sort(
        (left, right) =>
          deficit(right) - deficit(left) || compareText(left.name, right.name),
      );
    candidates[0].count += 1;
    remaining -= 1;
  }
  return allocations;
}

function compareText(left: string, right: string) {
  return left.localeCompare(right, "en-US");
}

function representativeSortKey(entry: SampleEntry) {
  const record = entry.record;
  return [
    record.repo_name,
    record.metadata.cwe_id,
    (record.metadata.languages || []).slice().sort().join(","),
    record.to_analyzer.locations[0].file,
    String(entry.index).padStart(6, "0"),
  ].join("\0");
}

function deterministicSystematicSample(
  entries: SampleEntry[],
  sampleCount: number,
  seed: string,
  stratumName: string,
) {
  if (sampleCount === entries.length) {
    return [...entries];
  }
  const sortedEntries = [...entries].sort((left, right) =>
    compareText(representativeSortKey(left), representativeSortKey(right)),
  );
  const interval = sortedEntries.length / sampleCount;
  const seedValue = hash(
    "sha256",
    `${seed}\0${stratumName}`,
    "buffer",
  ).readUInt32BE(0);
  const start = (seedValue / 2 ** 32) * interval;
  return Array.from(
    { length: sampleCount },
    (_, sampleIndex) =>
      sortedEntries[Math.floor(start + sampleIndex * interval)],
  );
}

/**
 * Select a deterministic, ground-truth-stratified sample. Evidence-size strata
 * preserve the benchmark's mix of small and large scanner tickets, while
 * systematic selection over repository/CWE/language ordering spreads each
 * stratum across the benchmark's main metadata dimensions.
 */
export function selectRepresentativeSample(
  records: SastBenchRecord[],
  sampleSpec: SampleSpec = REPRESENTATIVE_SAMPLE_SPEC,
) {
  requireNonEmptyString(sampleSpec.profile, "sample profile", "sample");
  requireNonEmptyString(sampleSpec.seed, "sample seed", "sample");
  const selectedEntries = [];

  for (const label of ALLOWED_LABELS) {
    const requestedCount = sampleSpec.labelCounts[label];
    if (!Number.isInteger(requestedCount) || requestedCount <= 0) {
      throw new Error(
        `Representative sample requires a positive integer count for ${label}`,
      );
    }
    const labelEntries = records
      .map((record: SastBenchRecord, index: number) => ({ record, index }))
      .filter((entry: SampleEntry) => entry.record.ground_truth === label);
    if (requestedCount > labelEntries.length) {
      throw new Error(
        `Representative sample requests ${requestedCount} ${label} cases from ${labelEntries.length}`,
      );
    }
    const entriesByBucket = Map.groupBy(labelEntries, (entry) =>
      evidenceSizeBucket(entry.record),
    );
    const strata = [...entriesByBucket.entries()]
      .map(([bucket, entries]) => ({ name: `${label}:${bucket}`, entries }))
      .sort((left, right) => compareText(left.name, right.name));
    const allocations = allocateSampleCounts(strata, requestedCount);

    for (const stratum of allocations) {
      const sampleWeight = stratum.entries.length / stratum.count;
      for (const entry of deterministicSystematicSample(
        stratum.entries,
        stratum.count,
        sampleSpec.seed,
        stratum.name,
      )) {
        selectedEntries.push({
          ...entry,
          sampleStratum: stratum.name,
          sampleWeight,
        });
      }
    }
  }

  return selectedEntries.sort((left, right) => left.index - right.index);
}

export function generateRepresentativeSampleTests(
  records: SastBenchRecord[],
  targetRoot: string | undefined = DEFAULT_TARGET_ROOT,
  sampleSpec: SampleSpec = REPRESENTATIVE_SAMPLE_SPEC,
) {
  return selectRepresentativeSample(records, sampleSpec).map((entry) =>
    buildPromptfooTest(entry.record, entry.index, targetRoot, {
      sample_profile: sampleSpec.profile,
      sample_stratum: entry.sampleStratum,
      sample_weight: entry.sampleWeight,
    }),
  );
}

interface GenerateOptions {
  dataset?: SastBenchRecord[];
  datasetPath?: string;
  targetRoot?: string;
  sampleSpec?: SampleSpec;
}

export function generateTests(options: GenerateOptions = {}) {
  return generatePromptfooTests(
    options.dataset || loadDataset(options.datasetPath || DEFAULT_DATASET_PATH),
    options.targetRoot || DEFAULT_TARGET_ROOT,
  );
}

export function generateSampleTests(options: GenerateOptions = {}) {
  return generateRepresentativeSampleTests(
    options.dataset || loadDataset(options.datasetPath || DEFAULT_DATASET_PATH),
    options.targetRoot || DEFAULT_TARGET_ROOT,
    options.sampleSpec,
  );
}