Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

10.9 KB
Raw Blame
#!/usr/bin/env node
import assert from "node:assert/strict";
import path from "node:path";
import { buildHydrationPlan, parseArgs } from "./hydrate-sastbench-repos.mts";

import {
  REPRESENTATIVE_SAMPLE_SPEC,
  buildPromptVars,
  generateTests,
  generateSampleTests,
  generatePromptfooTests,
  generateRepresentativeSampleTests,
  repositoryStateId,
  selectRepresentativeSample,
  validateDataset,
} from "./sastbench-lib.mts";

assert.deepEqual(REPRESENTATIVE_SAMPLE_SPEC, {
  profile: "representative-100-v1",
  seed: "sastbench-representative-v1",
  labelCounts: {
    true_positive: 40,
    false_positive: 60,
  },
});

function benchmarkRecord(overrides = {}) {
  return {
    finding_id: "upstream-label-bearing-id",
    repo_name: "example/project",
    repo_url: "https://github.com/example/project",
    commit_hash: "0123456789abcdef0123456789abcdef01234567",
    ground_truth: "true_positive",
    metadata: {
      source: "cve",
      cwe_id: "CWE-79",
    },
    to_analyzer: {
      vulnerability_type: "CWE-79",
      vulnerability_name: "Cross-site Scripting",
      description:
        "Untrusted text may be rendered without context-aware escaping.",
      locations: [
        {
          file: "src/render.js",
          function: "renderComment",
          line_start: 41,
          line_end: 47,
        },
      ],
      commit_context: {
        repo: "example/project",
        commit: "0123456789abcdef0123456789abcdef01234567",
      },
    },
    ...overrides,
  };
}

const positive = benchmarkRecord();
const negative = benchmarkRecord({
  finding_id: "another-label-bearing-id",
  ground_truth: "false_positive",
});
const targetRoot = path.resolve("/tmp/sastbench-targets");

validateDataset([positive, negative], {
  caseCount: 2,
  labelCounts: { true_positive: 1, false_positive: 1 },
});

assert.throws(
  () => validateDataset([positive, negative]),
  /case count mismatch/,
);
assert.throws(
  () =>
    validateDataset([positive, negative], {
      caseCount: 2,
      labelCounts: { true_positive: 2, false_positive: 0 },
    }),
  /true_positive count mismatch/,
);

const sparseScannerRecord = benchmarkRecord({
  to_analyzer: {
    ...positive.to_analyzer,
    vulnerability_name: "",
    description: "",
    locations: [
      {
        file: "src/file-level.js",
        function: "*global*",
        line_start: 0,
        line_end: 100,
      },
      {
        file: "src/zero-width.js",
        function: "construct",
        line_start: 54,
        line_end: 53,
      },
    ],
  },
});
validateDataset([sparseScannerRecord], {
  caseCount: 1,
  labelCounts: { true_positive: 1, false_positive: 0 },
});
const sparsePromptVars = buildPromptVars(sparseScannerRecord, 0, targetRoot);
assert.match(sparsePromptVars.finding_input, /title: CWE-79\n/);
assert.doesNotMatch(sparsePromptVars.finding_input, /claim:\s*\n/);
assert.match(sparsePromptVars.finding_input, /src\/file-level\.js:0-100/);
assert.match(sparsePromptVars.finding_input, /src\/zero-width\.js:54-53/);

assert.equal(
  buildPromptVars(positive, 2736, targetRoot).case_id,
  "sastbench-002736",
);

const stateId = repositoryStateId(positive);
assert.equal(repositoryStateId(negative), stateId);

assert.equal(parseArgs(["--", "--dry-run"]).dryRun, true);
const cacheRoot = path.resolve("/tmp/sastbench-git-cache");
const otherRevision = { ...positive, commit_hash: "2".repeat(40) };
const jobs = buildHydrationPlan([positive, negative, otherRevision], {
  cacheRoot,
  targetRoot,
});
assert.deepEqual(
  jobs.map((job) => job.caseCount),
  [2, 1],
);
for (const [index, record] of [positive, otherRevision].entries()) {
  const job = jobs[index]!;
  assert.equal(job.repoUrl, record.repo_url);
  assert.equal(job.commitHash, record.commit_hash);
  const target = repositoryStateId(record);
  assert.match(target, /^state-[a-f0-9]{16}$/);
  assert.equal(job.targetDir, path.join(targetRoot, target));
  assert.equal(job.cacheDir, path.join(cacheRoot, "repo-acb9cf7430784a93.git"));
}
assert.throws(
  () =>
    buildHydrationPlan([
      positive,
      { ...otherRevision, repo_url: "https://github.com/example/other" },
    ]),
  /maps to multiple repository URLs/,
);

const promptVars = buildPromptVars(positive, 0, targetRoot);
assert.equal(promptVars.case_id, "sastbench-000000");
assert.equal(promptVars.target_repo, path.join(targetRoot, stateId));
assert.match(promptVars.finding_input, /CWE-79/);
assert.match(promptVars.finding_input, /src\/render\.js:41-47/);
assert.equal("invocation_preamble" in promptVars, false);
assert.equal("eval_instructions" in promptVars, false);
assert.equal("source_type_under_test" in promptVars, false);
assert.equal("expected_input_id" in promptVars, false);

const renderedModelInput = JSON.stringify({
  case_id: promptVars.case_id,
  target_repo: promptVars.target_repo,
  finding_input: promptVars.finding_input,
});
assert.doesNotMatch(
  renderedModelInput,
  /true_positive|false_positive|ground_truth|upstream-label-bearing-id|"source":"cve"/,
);

const tests = generatePromptfooTests([positive, negative], targetRoot);
assert.equal(tests.length, 2);
assert.equal(tests[0].metadata.ground_truth, "true_positive");
assert.equal(tests[1].metadata.ground_truth, "false_positive");
assert.equal("expected_ground_truth" in tests[0].vars, false);
assert.equal(tests[0].metadata.case_id, "sastbench-000000");
assert.deepEqual(Object.keys(tests[0].vars).sort(), [
  "case_id",
  "finding_input",
  "target_repo",
]);

const generatedTests = generateTests({
  dataset: [positive, negative],
  targetRoot,
});
assert.deepEqual(generatedTests, tests);

const sampleDataset = Array.from({ length: 20 }, (_, index) => {
  const groundTruth = index % 5 < 2 ? "true_positive" : "false_positive";
  const locationCount = (index % 4) + 1;
  return benchmarkRecord({
    finding_id: `sample-${index}`,
    repo_name: `example/project-${index % 5}`,
    repo_url: `https://github.com/example/project-${index % 5}`,
    ground_truth: groundTruth,
    metadata: {
      source: groundTruth === "true_positive" ? "cve" : "semgrep",
      cwe_id: `CWE-${79 + (index % 3)}`,
      languages: [index % 2 === 0 ? "JavaScript" : "Python"],
    },
    to_analyzer: {
      ...positive.to_analyzer,
      locations: Array.from({ length: locationCount }, (_, locationIndex) => ({
        file: `src/file-${index}-${locationIndex}.js`,
        function: "handle",
        line_start: locationIndex + 1,
        line_end: locationIndex + 2,
      })),
    },
  });
});
const sampleSpec = {
  profile: "test-representative",
  seed: "test-seed",
  labelCounts: { true_positive: 4, false_positive: 6 },
};
// Keep the established v1 collation when comparing paired sample runs.
const collationRepositories = [
  "example/a-b",
  "example/a_b",
  "example/A",
  "example/a",
  "example/Z",
  "example/z",
];
const collationDataset = Array.from({ length: 48 }, (_, index) =>
  benchmarkRecord({
    repo_name: collationRepositories[index % collationRepositories.length],
    ground_truth: index < 24 ? "true_positive" : "false_positive",
    metadata: {
      source: "synthetic",
      cwe_id: "CWE-79",
      languages: ["JavaScript"],
    },
    to_analyzer: {
      ...positive.to_analyzer,
      locations: [
        { ...positive.to_analyzer.locations[0], file: `src/file-${index}.js` },
      ],
    },
  }),
);
assert.deepEqual(
  selectRepresentativeSample(collationDataset, {
    profile: "test-v1-collation",
    seed: "test-v1-collation",
    labelCounts: { true_positive: 6, false_positive: 6 },
  }).map(({ index }) => index),
  [0, 1, 10, 14, 21, 22, 32, 34, 36, 37, 44, 46],
);

const selectedSample = selectRepresentativeSample(sampleDataset, sampleSpec);
const selectedAgain = selectRepresentativeSample(sampleDataset, sampleSpec);

assert.equal(selectedSample.length, 10);
assert.deepEqual(
  selectedSample.map(({ index }) => index),
  selectedAgain.map(({ index }) => index),
);
assert.equal(
  new Set(selectedSample.map(({ index }) => index)).size,
  selectedSample.length,
);
assert.equal(
  selectedSample.filter(({ record }) => record.ground_truth === "true_positive")
    .length,
  4,
);
assert.equal(
  selectedSample.filter(
    ({ record }) => record.ground_truth === "false_positive",
  ).length,
  6,
);
for (const label of ["true_positive", "false_positive"]) {
  const populationCount = sampleDataset.filter(
    (record) => record.ground_truth === label,
  ).length;
  const representedPopulation = selectedSample
    .filter(({ record }) => record.ground_truth === label)
    .reduce((total, entry) => total + entry.sampleWeight, 0);
  assert.ok(Math.abs(representedPopulation - populationCount) < 1e-9);
}

const representativeTests = generateRepresentativeSampleTests(
  sampleDataset,
  targetRoot,
  sampleSpec,
);
assert.equal(representativeTests.length, 10);
assert.ok(
  representativeTests.some(
    (test) => Number(test.metadata.case_id.replace("sastbench-", "")) >= 10,
  ),
);
assert.ok(
  representativeTests.every(
    (test) => test.metadata.sample_profile === sampleSpec.profile,
  ),
);
assert.ok(representativeTests.every((test) => test.metadata.sample_weight > 0));
assert.ok(
  representativeTests.every((test) => "sample_stratum" in test.metadata),
);
assert.ok(representativeTests.every((test) => !("sample_weight" in test.vars)));

const generatedSampleTests = generateSampleTests({
  dataset: sampleDataset,
  targetRoot,
  sampleSpec,
});
assert.deepEqual(generatedSampleTests, representativeTests);

assert.throws(
  () =>
    validateDataset([benchmarkRecord({ ground_truth: "unknown" })], {
      caseCount: 1,
      labelCounts: { true_positive: 0, false_positive: 1 },
    }),
  /unsupported ground_truth/,
);
assert.throws(
  () =>
    validateDataset(
      [
        benchmarkRecord({
          to_analyzer: {
            ...positive.to_analyzer,
            commit_context: { repo: "example/project", commit: "different" },
          },
        }),
      ],
      { caseCount: 1, labelCounts: { true_positive: 1, false_positive: 0 } },
    ),
  /commit_context\.commit/,
);

console.log("sastbench dataset and prompt tests passed");

// A locale may reorder accents and punctuation; that must not change the sample.
import { execFileSync } from "node:child_process";
const localeRecords = ["true_positive", "false_positive"].flatMap(
  (ground_truth) =>
    ["a", "ä", "z", "Z", "_", "å"].map((repo_name) =>
      benchmarkRecord({ ground_truth, repo_name }),
    ),
);
const sampleProgram = `import { selectRepresentativeSample } from ${JSON.stringify(new URL("./sastbench-lib.mts", import.meta.url).href)}; console.log(JSON.stringify(selectRepresentativeSample(${JSON.stringify(localeRecords)}, {profile:'test',seed:'test',labelCounts:{true_positive:2,false_positive:2}}).map(entry => entry.index)));`;
const localeSample = (locale: string) =>
  execFileSync(
    process.execPath,
    [
      "--experimental-strip-types",
      "--input-type=module",
      "--eval",
      sampleProgram,
    ],
    { encoding: "utf8", env: { ...process.env, LANG: locale, LC_ALL: locale } },
  );
assert.equal(localeSample("en_US.UTF-8"), localeSample("sv_SE.UTF-8"));