Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(evals): fetch calibration revisions as source refs (#2425)

Michael D'Angelo7bffa9552efb
diff --git a/evals/triage-finding/scripts/hydrate-calibration-repos.mts b/evals/triage-finding/scripts/hydrate-calibration-repos.mts
index 4f2f0b026..5b5d11557 100644
--- a/evals/triage-finding/scripts/hydrate-calibration-repos.mts
+++ b/evals/triage-finding/scripts/hydrate-calibration-repos.mts
@@ -96,7 +96,10 @@ function ensureGitCheckout(job: ReturnType<typeof plannedJobs>[number]) {
return "already current";
}
- runGit(["fetch", "--depth", "1", "origin", job.checkoutRef], job.targetDir);
+ runGit(
+ ["fetch", "--depth", "1", "--", "origin", `${job.checkoutRef}:`],
+ job.targetDir,
+ );
runGit(["checkout", "--detach", "FETCH_HEAD"], job.targetDir);
return "hydrated";
}
diff --git a/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts b/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts
index 429b3b6b1..d083d6e11 100644
--- a/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts
+++ b/evals/triage-finding/scripts/test-hydrate-calibration-repos.mts
@@ -21,6 +21,7 @@ function runHydrator(args: string[]) {
{
cwd: evalDir,
encoding: "utf8",
+ stdio: "pipe",
},
);
}
@@ -196,6 +197,26 @@ try {
gitfile,
);
assert.equal(git(worktreeTarget, "rev-parse", "HEAD"), expectedHead);
+
+ git(source, "update-ref", "refs/heads/--all", expectedHead);
+ fixture.cases[0].variants[0].checkout_ref = "--all";
+ fs.writeFileSync(dataset, JSON.stringify(fixture));
+ assert.match(
+ runHydrator(["--dataset", dataset, "--repo-root", repoRoot]),
+ /hydrated: case\/variant/,
+ );
+
+ fixture.cases[0].variants[0].checkout_ref = "HEAD:refs/tags/calibration-copy";
+ fs.writeFileSync(dataset, JSON.stringify(fixture));
+ const [job] = plannedJobs(fixture as Parameters<typeof plannedJobs>[0], {
+ repoRoot: worktreeRoot,
+ });
+ git(legacyTarget, "worktree", "add", "--detach", job.targetDir, expectedHead);
+ assert.throws(
+ () => runHydrator(["--dataset", dataset, "--repo-root", worktreeRoot]),
+ /invalid refspec/,
+ );
+ assert.equal(git(legacyTarget, "tag", "--list", "calibration-copy"), "");
} finally {
fs.rmSync(temporary, { recursive: true, force: true });
}