OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(evals): verify calibration checkout roots before hydration (#1250)
mldangelo-oai400f485dd933
diff --git a/evals/triage-finding/scripts/hydrate-calibration-repos.mts b/evals/triage-finding/scripts/hydrate-calibration-repos.mtsindex f76856daf..da15c1ddf 100644--- a/evals/triage-finding/scripts/hydrate-calibration-repos.mts+++ b/evals/triage-finding/scripts/hydrate-calibration-repos.mts@@ -60,7 +60,10 @@ function ensureGitCheckout(job: (typeof jobs)[number]) {fs.mkdirSync(job.targetDir, { recursive: true });}- if (!fs.existsSync(path.join(job.targetDir, ".git"))) {+ if (+ !fs.existsSync(path.join(job.targetDir, ".git")) ||+ gitOutput(["rev-parse", "--show-prefix"], job.targetDir) !== ""+ ) {const entries = fs.readdirSync(job.targetDir);if (entries.length > 0) {throw new Error(diff --git a/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts b/evals/triage-finding/scripts/test-hydrate-calibration-repos.mtsindex 5393ca3ef..dcf6d6532 100644--- a/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts+++ b/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts@@ -2,6 +2,8 @@import assert from "node:assert/strict";import childProcess from "node:child_process";import path from "node:path";+import fs from "node:fs";+import os from "node:os";const evalDir = path.join(import.meta.dirname, "..");const scriptPath = path.join(@@ -41,4 +43,110 @@ assert.doesNotMatch(/oss-mantisbt-ghsa-73vx-49mv-v8w5\/vulnerable/,);-console.log("calibration hydration dry-run tests passed");+const temporary = fs.mkdtempSync(+ path.join(os.tmpdir(), "calibration-checkouts-"),+);+function git(directory: string, ...args: string[]) {+ return childProcess+ .execFileSync("git", args, {+ cwd: directory,+ encoding: "utf8",+ stdio: ["ignore", "pipe", "pipe"],+ })+ .trim();+}+function repository(name: string) {+ const directory = path.join(temporary, name);+ fs.mkdirSync(directory);+ git(directory, "init", "-b", "main");+ fs.writeFileSync(path.join(directory, "fixture.txt"), name);+ git(directory, "add", "fixture.txt");+ git(+ directory,+ "-c",+ "user.name=Synthetic",+ "-c",+ "[email protected]",+ "-c",+ "commit.gpgsign=false",+ "commit",+ "-m",+ "Fixture",+ );+ return directory;+}+try {+ const source = repository("source");+ const parent = repository("parent");+ git(+ parent,+ "remote",+ "add",+ "origin",+ "https://example.invalid/original.git",+ );+ const originalHead = git(parent, "rev-parse", "HEAD");+ const originalOrigin = git(parent, "remote", "get-url", "origin");+ const expectedHead = git(source, "rev-parse", "HEAD");+ const dataset = path.join(temporary, "dataset.json");+ fs.writeFileSync(+ dataset,+ JSON.stringify({+ cases: [+ {+ case_id: "case",+ repo: { url: source },+ variants: [{ variant_id: "variant", checkout_ref: expectedHead }],+ },+ ],+ }),+ );+ const repoRoot = path.join(parent, "checkouts");+ const target = path.join(repoRoot, "case", "variant");+ fs.mkdirSync(path.join(target, ".git"), { recursive: true });+ const rejected = childProcess.spawnSync(+ process.execPath,+ [+ "--experimental-strip-types",+ scriptPath,+ "--dataset",+ dataset,+ "--repo-root",+ repoRoot,+ ],+ { encoding: "utf8" },+ );+ assert.equal(git(parent, "remote", "get-url", "origin"), originalOrigin);+ assert.equal(git(parent, "rev-parse", "HEAD"), originalHead);+ assert.equal(git(parent, "symbolic-ref", "--short", "HEAD"), "main");+ assert.notEqual(rejected.status, 0);+ assert.match(rejected.stderr, /non-empty non-git directory/);++ fs.rmdirSync(path.join(target, ".git"));+ assert.match(+ runHydrator(["--dataset", dataset, "--repo-root", repoRoot]),+ /hydrated: case\/variant/,+ );+ assert.equal(git(target, "rev-parse", "HEAD"), expectedHead);+ assert.equal(git(parent, "remote", "get-url", "origin"), originalOrigin);+ assert.equal(git(parent, "rev-parse", "HEAD"), originalHead);+ assert.match(+ runHydrator(["--dataset", dataset, "--repo-root", repoRoot]),+ /already current/,+ );++ const alias = path.join(temporary, " checkout alias ");+ fs.symlinkSync(+ repoRoot,+ alias,+ process.platform === "win32" ? "junction" : "dir",+ );+ assert.match(+ runHydrator(["--dataset", dataset, "--repo-root", alias]),+ /already current/,+ );+} finally {+ fs.rmSync(temporary, { recursive: true, force: true });+}++console.log("calibration hydration tests passed");