Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(evals): verify calibration checkout roots before hydration (#1250)

mldangelo-oai400f485dd933
diff --git a/evals/triage-finding/scripts/hydrate-calibration-repos.mts b/evals/triage-finding/scripts/hydrate-calibration-repos.mts
index f76856daf..da15c1ddf 100644
--- a/evals/triage-finding/scripts/hydrate-calibration-repos.mts
+++ b/evals/triage-finding/scripts/hydrate-calibration-repos.mts
@@ -60,7 +60,10 @@ function ensureGitCheckout(job: (typeof jobs)[number]) {
fs.mkdirSync(job.targetDir, { recursive: true });
}
- if (!fs.existsSync(path.join(job.targetDir, ".git"))) {
+ if (
+ !fs.existsSync(path.join(job.targetDir, ".git")) ||
+ gitOutput(["rev-parse", "--show-prefix"], job.targetDir) !== ""
+ ) {
const entries = fs.readdirSync(job.targetDir);
if (entries.length > 0) {
throw new Error(
diff --git a/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts b/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts
index 5393ca3ef..dcf6d6532 100644
--- a/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts
+++ b/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts
@@ -2,6 +2,8 @@
import assert from "node:assert/strict";
import childProcess from "node:child_process";
import path from "node:path";
+import fs from "node:fs";
+import os from "node:os";
const evalDir = path.join(import.meta.dirname, "..");
const scriptPath = path.join(
@@ -41,4 +43,110 @@ assert.doesNotMatch(
/oss-mantisbt-ghsa-73vx-49mv-v8w5\/vulnerable/,
);
-console.log("calibration hydration dry-run tests passed");
+const temporary = fs.mkdtempSync(
+ path.join(os.tmpdir(), "calibration-checkouts-"),
+);
+function git(directory: string, ...args: string[]) {
+ return childProcess
+ .execFileSync("git", args, {
+ cwd: directory,
+ encoding: "utf8",
+ stdio: ["ignore", "pipe", "pipe"],
+ })
+ .trim();
+}
+function repository(name: string) {
+ const directory = path.join(temporary, name);
+ fs.mkdirSync(directory);
+ git(directory, "init", "-b", "main");
+ fs.writeFileSync(path.join(directory, "fixture.txt"), name);
+ git(directory, "add", "fixture.txt");
+ git(
+ directory,
+ "-c",
+ "user.name=Synthetic",
+ "-c",
+ "-c",
+ "commit.gpgsign=false",
+ "commit",
+ "-m",
+ "Fixture",
+ );
+ return directory;
+}
+try {
+ const source = repository("source");
+ const parent = repository("parent");
+ git(
+ parent,
+ "remote",
+ "add",
+ "origin",
+ "https://example.invalid/original.git",
+ );
+ const originalHead = git(parent, "rev-parse", "HEAD");
+ const originalOrigin = git(parent, "remote", "get-url", "origin");
+ const expectedHead = git(source, "rev-parse", "HEAD");
+ const dataset = path.join(temporary, "dataset.json");
+ fs.writeFileSync(
+ dataset,
+ JSON.stringify({
+ cases: [
+ {
+ case_id: "case",
+ repo: { url: source },
+ variants: [{ variant_id: "variant", checkout_ref: expectedHead }],
+ },
+ ],
+ }),
+ );
+ const repoRoot = path.join(parent, "checkouts");
+ const target = path.join(repoRoot, "case", "variant");
+ fs.mkdirSync(path.join(target, ".git"), { recursive: true });
+ const rejected = childProcess.spawnSync(
+ process.execPath,
+ [
+ "--experimental-strip-types",
+ scriptPath,
+ "--dataset",
+ dataset,
+ "--repo-root",
+ repoRoot,
+ ],
+ { encoding: "utf8" },
+ );
+ assert.equal(git(parent, "remote", "get-url", "origin"), originalOrigin);
+ assert.equal(git(parent, "rev-parse", "HEAD"), originalHead);
+ assert.equal(git(parent, "symbolic-ref", "--short", "HEAD"), "main");
+ assert.notEqual(rejected.status, 0);
+ assert.match(rejected.stderr, /non-empty non-git directory/);
+
+ fs.rmdirSync(path.join(target, ".git"));
+ assert.match(
+ runHydrator(["--dataset", dataset, "--repo-root", repoRoot]),
+ /hydrated: case\/variant/,
+ );
+ assert.equal(git(target, "rev-parse", "HEAD"), expectedHead);
+ assert.equal(git(parent, "remote", "get-url", "origin"), originalOrigin);
+ assert.equal(git(parent, "rev-parse", "HEAD"), originalHead);
+ assert.match(
+ runHydrator(["--dataset", dataset, "--repo-root", repoRoot]),
+ /already current/,
+ );
+
+ const alias = path.join(temporary, " checkout alias ");
+ fs.symlinkSync(
+ repoRoot,
+ alias,
+ process.platform === "win32" ? "junction" : "dir",
+ );
+ assert.match(
+ runHydrator(["--dataset", dataset, "--repo-root", alias]),
+ /already current/,
+ );
+} finally {
+ fs.rmSync(temporary, { recursive: true, force: true });
+}
+
+console.log("calibration hydration tests passed");