Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

ci: verify PR edits against an advanced base branch (#2024)

Michael D'Angelo795b0ba05de0
diff --git a/.github/scripts/select-ci-checks.mjs b/.github/scripts/select-ci-checks.mjs
index 35c3dd3c9..61e43dc0e 100644
--- a/.github/scripts/select-ci-checks.mjs
+++ b/.github/scripts/select-ci-checks.mjs
@@ -1,6 +1,7 @@
import { execFileSync } from "node:child_process";
import { appendFileSync, readFileSync } from "node:fs";
import { pathToFileURL } from "node:url";
+import { checkoutBase } from "./verify-pr-source.mjs";
const documentation =
/^(?:README\.md|CONTRIBUTING\.md|CODE_OF_CONDUCT\.md|docs\/.*\.md|github-action\/README\.md|docker\/README\.md)$/;
@@ -116,20 +117,16 @@ if (
const event = process.env.EVENT_NAME;
const baseChanged = process.env.BASE_CHANGED === "true";
if (event === "pull_request" && baseChanged) {
- const { base, head } = JSON.parse(
+ const { changes, pull_request: pr } = JSON.parse(
readFileSync(process.env.GITHUB_EVENT_PATH, "utf8"),
- ).pull_request;
- const parents = execFileSync("git", ["show", "-s", "--format=%P", "HEAD"], {
- encoding: "utf8",
- }).trim();
- const expected = `${base.sha} ${head.sha}`;
- console.log(
- `Requested base ${base.ref}. Expected merge parents ${expected}. Checked out ${parents}.`,
);
- if (parents !== expected) {
- throw new Error(
- "GitHub supplied a stale test merge. After the merge ref updates, reopen the pull request to start a fresh source run. Rerunning this attempt uses the same commit.",
- );
+ // Ordinary edits already proved their live target before falling back to full checks.
+ if (changes?.base) {
+ const base = checkoutBase(pr, process.env.GITHUB_SHA);
+ if (base !== pr.base.sha)
+ throw new Error(
+ `Stale test merge. Requested ${pr.base.ref} at ${pr.base.sha}, checked out base ${base} with head ${pr.head.sha}. Start a fresh pull request event after the merge ref updates.`,
+ );
}
}
const changes =
diff --git a/.github/scripts/test-verify-pr-source.mjs b/.github/scripts/test-verify-pr-source.mjs
index a49f1d101..cf13639a3 100644
--- a/.github/scripts/test-verify-pr-source.mjs
+++ b/.github/scripts/test-verify-pr-source.mjs
@@ -6,7 +6,7 @@ import { join } from "node:path";
import test from "node:test";
import { pathToFileURL } from "node:url";
import {
- checkCheckout,
+ checkoutBase,
confirmEdit,
realSource,
verifyEdit,
@@ -57,7 +57,14 @@ function evidence(id, selected = "success") {
}
function fixture(
history,
- { jobs = evidence, targets = [target], wait = false, runAttempt = 1 } = {},
+ {
+ jobs = evidence,
+ targets = [target],
+ wait = false,
+ runAttempt = 1,
+ base = pr.base.sha,
+ comparisons = [],
+ } = {},
) {
let current = [],
live = target;
@@ -67,6 +74,12 @@ function fixture(
if (live instanceof Error) throw live;
return live;
}
+ if (path.startsWith("compare/")) {
+ assert.equal(path, "compare/refs%2Fheads%2Fmain...base?per_page=1");
+ const result = comparisons.shift();
+ if (result instanceof Error) throw result;
+ return result;
+ }
const runId = path.match(/runs\/(\d+)\/jobs/);
if (runId) return jobs(Number(runId[1]));
current = history.shift() ?? current;
@@ -75,7 +88,7 @@ function fixture(
const runs = [edit(99, { run_attempt: runAttempt }), ...current];
return { total_count: runs.length, workflow_runs: runs };
};
- const input = { pr, sha, runId: 99, runAttempt, api };
+ const input = { pr, sha, base, runId: 99, runAttempt, api };
return {
check: () => verifyEdit(input, () => wait),
finish: (id, classifierAttempt = runAttempt) =>
@@ -220,6 +233,45 @@ test("even full fallback needs a current target when the CI gate finishes", asyn
);
});
+test("an edit can use a newer test-merge base only while its named branch still proves that advance", async () => {
+ const base = "new-base";
+ const current = { base_commit: { sha: base }, status: "behind" };
+ const changed = { base_commit: { sha: "moved" }, status: "behind" };
+ const check = (comparisons, history = [[run(2)]]) =>
+ fixture(history, { base, comparisons });
+ const updated = { ...target, base: { ...pr.base, sha: base } };
+ const source = run(2, { pull_requests: [updated] });
+ const caughtUp = fixture([[run(2)], [source]], {
+ base,
+ comparisons: [current, current, current],
+ targets: [target, updated],
+ });
+ assert.equal(await caughtUp.check(), 2);
+ await caughtUp.finish(2);
+ await assert.rejects(
+ fixture([[source]], {
+ base,
+ targets: [{ ...updated, base: { ...pr.base, sha: "unrelated" } }],
+ }).finish(2),
+ /target changed/,
+ );
+ assert.equal(await check([current, current], [[]]).check(), "full");
+ for (const invalid of [
+ changed,
+ { ...current, status: "ahead" },
+ { ...current, status: "diverged" },
+ { ...current, status: "identical" },
+ new Error("comparison unavailable"),
+ ]) {
+ const error =
+ invalid instanceof Error ? /comparison unavailable/ : /Stale test merge/;
+ await assert.rejects(check([invalid]).check(), error);
+ await assert.rejects(check([current, invalid]).check(), error);
+ await assert.rejects(check([invalid]).finish(2), error);
+ await assert.rejects(check([invalid]).finish(), error);
+ }
+});
+
test("the final gate reads both actual merge parents from a depth-one checkout", (t) => {
const root = mkdtempSync(join(tmpdir(), "ci-source-parents-"));
t.after(() => rmSync(root, { recursive: true, force: true }));
@@ -264,14 +316,10 @@ test("the final gate reads both actual merge parents from a depth-one checkout",
);
assert.equal(git(checkout, "rev-parse", "--is-shallow-repository"), "true");
const pull = { base: { sha: base }, head: { sha: head } };
- checkCheckout(pull, merge, checkout);
- assert.throws(
- () => checkCheckout({ ...pull, base: { sha: head } }, merge, checkout),
- /checked-out merge/,
- );
+ assert.equal(checkoutBase(pull, merge, checkout), base);
assert.throws(
- () => checkCheckout({ ...pull, head: { sha: base } }, merge, checkout),
- /checked-out merge/,
+ () => checkoutBase({ ...pull, head: { sha: base } }, merge, checkout),
+ /Stale test merge/,
);
- assert.throws(() => checkCheckout(pull, base, checkout), /checked-out merge/);
+ assert.throws(() => checkoutBase(pull, base, checkout), /Stale test merge/);
});
diff --git a/.github/scripts/verify-pr-source.mjs b/.github/scripts/verify-pr-source.mjs
index e2599f1e4..d494f7c70 100644
--- a/.github/scripts/verify-pr-source.mjs
+++ b/.github/scripts/verify-pr-source.mjs
@@ -7,43 +7,57 @@ const utf8 = { encoding: "utf8" };
const successful = (job) =>
job.status === "completed" && job.conclusion === "success";
-function samePull(candidate, pr) {
+function samePull(candidate, pr, base) {
return (
candidate?.number === pr.number &&
+ candidate.head?.sha === pr.head.sha &&
+ [pr.base.sha, base].includes(candidate.base?.sha) &&
["head", "base"].every(
(side) =>
candidate[side]?.ref === pr[side].ref &&
- candidate[side]?.sha === pr[side].sha &&
candidate[side].repo?.id === pr[side].repo.id,
)
);
}
-export function checkCheckout(pr, sha, cwd) {
- const git = (args) =>
- execFileSync("git", args, { encoding: "utf8", cwd }).trim();
- // The CI gate uses a depth-one checkout. Read its parent IDs from the raw object.
- const headers = git(["cat-file", "-p", "HEAD"])
+export function checkoutBase(pr, sha, cwd) {
+ // A depth-one checkout still contains the raw commit and its parent IDs.
+ const [object, ...headers] = execFileSync("git", ["cat-file", "--batch"], {
+ cwd,
+ input: "HEAD\n",
+ encoding: "utf8",
+ })
.split("\n\n", 1)[0]
.split("\n");
const parents = headers
- .filter((h) => h.startsWith("parent "))
- .map((h) => h.slice(7));
+ .filter((line) => line.startsWith("parent "))
+ .map((line) => line.slice(7));
if (
- git(["rev-parse", "HEAD"]) !== sha ||
- parents.join(" ") !== `${pr.base.sha} ${pr.head.sha}`
+ !object.startsWith(`${sha} commit `) ||
+ parents.length !== 2 ||
+ parents[1] !== pr.head.sha
)
throw new Error(
- "The checked-out merge does not match the pull request base and head. Reopen the pull request or push a commit to qualify its current target.",
+ `Stale test merge. Expected ${sha} with head ${pr.head.sha}, checked out ${object} with parents ${parents.join(" ")}. Start a fresh pull request event.`,
);
+ return parents[0];
}
-export async function confirmTarget(pr, sha, api) {
+async function confirmTarget({ pr, sha, base, api }) {
const live = await api(`pulls/${pr.number}`);
- if (!samePull(live, pr) || live.merge_commit_sha !== sha)
+ if (!samePull(live, pr, base) || live.merge_commit_sha !== sha)
throw new Error(
"The pull request target changed while its checks were running.",
);
+ if (base !== pr.base.sha) {
+ // PR metadata can lag its test merge. Compare the named branch itself, including its ancestry.
+ const ref = encodeURIComponent(`refs/heads/${pr.base.ref}`);
+ const comparison = await api(`compare/${ref}...${pr.base.sha}?per_page=1`);
+ if (comparison.base_commit.sha !== base || comparison.status !== "behind")
+ throw new Error(
+ `Stale test merge. Checked out base ${base}, but ${pr.base.ref} is at ${comparison.base_commit.sha} and event base ${pr.base.sha} is ${comparison.status}. Reopen the pull request or push a commit after its merge ref updates.`,
+ );
+ }
}
export function realSource(page, run) {
@@ -66,7 +80,7 @@ export function realSource(page, run) {
);
}
-function sourceCandidate(runs, { pr, sha }) {
+function sourceCandidate(runs, { pr, sha, base }) {
const target = `PR #${pr.number} → ${pr.base.ref}`;
const source = `Source CI · ${target}`;
const edit = `PR edit CI · ${target}`;
@@ -76,7 +90,7 @@ function sourceCandidate(runs, { pr, sha }) {
run.event === "pull_request" &&
run.head_sha === pr.head.sha &&
run.run_attempt === 1 &&
- run.pull_requests.some((p) => samePull(p, pr)) &&
+ run.pull_requests.some((p) => samePull(p, pr, base)) &&
refs?.length &&
refs.every(
(ref) => ref.sha === sha && ref.ref === `refs/pull/${pr.number}/merge`,
@@ -141,8 +155,7 @@ async function verifyHistory(input, wait) {
}
export async function verifyEdit(input, wait) {
- const { pr, sha, api } = input;
- await confirmTarget(pr, sha, api);
+ await confirmTarget(input);
let result;
try {
result = await verifyHistory(input, wait);
@@ -153,7 +166,7 @@ export async function verifyEdit(input, wait) {
result = "full";
}
// A history failure can select full CI only while the originating PR target is still current.
- await confirmTarget(pr, sha, api);
+ await confirmTarget(input);
return result;
}
@@ -164,7 +177,7 @@ export async function confirmEdit(input, expectedId, classifierAttempt) {
throw new Error(
"The selected source qualification changed before the CI gate finished.",
);
- await confirmTarget(input.pr, input.sha, input.api);
+ await confirmTarget(input);
}
if (
@@ -176,7 +189,6 @@ if (
readFileSync(env.GITHUB_EVENT_PATH, "utf8"),
);
const sha = env.GITHUB_SHA;
- checkCheckout(pr, sha);
function api(path) {
const endpoint = `repos/${env.GITHUB_REPOSITORY}/${path}`;
return JSON.parse(execFileSync("gh", ["api", endpoint], utf8));
@@ -184,6 +196,7 @@ if (
const input = {
pr,
sha,
+ base: checkoutBase(pr, sha),
runId: Number(env.GITHUB_RUN_ID),
runAttempt: Number(env.GITHUB_RUN_ATTEMPT),
api,
diff --git a/docs/ci.md b/docs/ci.md
index 99e547dc5..138dc095e 100644
--- a/docs/ci.md
+++ b/docs/ci.md
@@ -80,6 +80,10 @@ successful CI for the exact main commit before publication. Title and body edits
leave source CI running. On an edit, CI checks the source result for the same
pull request, base, head and test merge. It waits for a running source check and
reports a verified result, including static checks, through the required checks.
+If the PR event reports an older base, CI can verify the edit on a newer test
+merge whose first parent is the current tip of that same branch. A stale merge
+or a retarget with different parents needs a fresh PR event after the merge ref
+updates. Reopening the PR or pushing a commit starts one.
Only the original source run and an uninterrupted sequence of successful edits
can carry a green result. A cancelled or retried run, a changed target, missing
evidence or conflicting results require full qualification. Use **Re-run all
diff --git a/sdk/typescript/tests-ts/workflow-quality.test.ts b/sdk/typescript/tests-ts/workflow-quality.test.ts
index 8e79c867a..42fc2143a 100644
--- a/sdk/typescript/tests-ts/workflow-quality.test.ts
+++ b/sdk/typescript/tests-ts/workflow-quality.test.ts
@@ -243,6 +243,7 @@ test("classification uses the PR merge diff and validates retargeted merge paren
env: {
...process.env,
EVENT_NAME: "pull_request",
+ GITHUB_SHA: git("rev-parse", "HEAD").trim(),
BASE_CHANGED: "false",
GITHUB_OUTPUT: output,
...env,
@@ -265,14 +266,17 @@ test("classification uses the PR merge diff and validates retargeted merge paren
const head = git("rev-parse", "docs").trim();
const old = git("rev-parse", "HEAD^1^1").trim();
const eventPath = join(directory, "event.json");
- for (const [expectedBase, expectedHead, valid] of [
- [base, head, true],
- [old, head, false],
- [base, old, false],
+ const baseEdit = { base: { ref: { from: "previous" } } };
+ for (const [changes, expectedBase, expectedHead, valid] of [
+ [baseEdit, base, head, true],
+ [baseEdit, old, head, false],
+ [baseEdit, base, old, false],
+ [{ title: { from: "Previous title" } }, old, head, true],
] as const) {
await writeFile(
eventPath,
JSON.stringify({
+ changes,
pull_request: {
base: { ref: "main", sha: expectedBase },
head: { sha: expectedHead },
@@ -288,8 +292,9 @@ test("classification uses the PR merge diff and validates retargeted merge paren
const selection = await readFile(output, "utf8");
if (valid) expect(selection).toContain("workflow-quality=true");
else {
- expect(retarget.stdout).toContain(`${base} ${head}`);
- expect(retarget.stderr).toContain("stale test merge");
+ expect(retarget.stderr).toContain(base);
+ expect(retarget.stderr).toContain(head);
+ expect(retarget.stderr).toContain("Stale test merge");
expect(selection).toBe("");
}
}