Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

ci(release): request review when the updated PR head becomes visible (#1989)

Michael D'Angelo6a26b93c0899
diff --git a/RELEASING.md b/RELEASING.md
index 2a7becbb6..8a9f40155 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -110,7 +110,8 @@ the latest `main` and appends a commit; the updater never force-pushes.
A concurrent commit causes it to reread and
retry. It requests Codex review when the proposal files change. Updates that
only incorporate `main` keep CI current without repeating the same proposal
-review. New suggestions for human-owned notes still appear in a comment.
+review, even when the earlier request names an older commit. New suggestions
+for human-owned notes still appear in a comment.
Before merging the proposal, check CI and request a final Codex review
if the last review targets an older head.
@@ -129,7 +130,10 @@ changes; other workflow content comes from the latest `main`. After the release
PR merges, this example can reference the new version before npm publication
finishes. Wait for publication before using that pin.
-If a run reports that GitHub has not exposed the updated PR head, manually
+After updating an existing release branch and its title, the updater waits briefly
+if GitHub still shows the exact commit it just replaced. A different head or a
+newer `main` stops that review handoff; the next updater can use the current branches.
+If GitHub still has not exposed the updated PR head, manually
rerun the updater with **dry_run** disabled after the PR catches up. This
allows any deferred review request or note suggestions to be posted. Verify
review on the current head before merging the proposal.
@@ -227,8 +231,11 @@ job with read-only permissions for both scopes.
Review the resulting PR and select **Approve workflows to run** in its merge
box to start hosted CI. GitHub requires this approval for PRs created or updated
-with `GITHUB_TOKEN`. Check the Codex review on the current head as well, and
-request it manually if the automated request has not started a review.
+with `GITHUB_TOKEN`. Check that a Codex code review completed on the current
+commit; a passing security check alone does not confirm this. If the automated
+request does not start a review or the integration asks the automated account
+to connect, a maintainer with a connected account should request the code
+review manually and verify it completed on that commit.
For CI to start without this approval, optionally configure a GitHub App
installed on this repository with **Contents: write** and **Pull requests: write**.
diff --git a/sdk/typescript/scripts/release-pr.mjs b/sdk/typescript/scripts/release-pr.mjs
index 14dc7d38a..e450d93a6 100644
--- a/sdk/typescript/scripts/release-pr.mjs
+++ b/sdk/typescript/scripts/release-pr.mjs
@@ -458,18 +458,26 @@ async function branchHead(github, branch) {
}
}
-function pullHoldReason(pull, branch) {
+const closedPullReason =
+ "The release PR was closed or retargeted during the update. Review it before continuing.";
+const pendingHeadReason =
+ "GitHub has not exposed the updated PR head yet. Rerun the updater before final review.";
+const changedHeadReason =
+ "The release head changed. Rerun before final review.";
+
+function pullHoldReason(pull, branch, repository) {
if (
pull?.state !== "open" ||
pull.head.ref !== branch ||
+ pull.head.repo?.full_name?.toLowerCase() !== repository.toLowerCase() ||
pull.base.ref !== "main"
) {
- return "The release PR was closed or retargeted during the update. Review it before continuing.";
+ return closedPullReason;
}
return null;
}
-function releaseHoldReason(openPulls, pullNumber, branch, repository) {
+function conflictingReleaseReason(openPulls, pullNumber, branch, repository) {
const retargeted = openPulls.find(
(candidate) =>
candidate.head.ref === branch &&
@@ -486,10 +494,112 @@ function releaseHoldReason(openPulls, pullNumber, branch, repository) {
);
if (other)
return `Another release PR is open: #${other.number}. It has not been changed.`;
+ return null;
+}
+
+function releaseHoldReason(openPulls, pullNumber, branch, repository) {
+ const conflict = conflictingReleaseReason(
+ openPulls,
+ pullNumber,
+ branch,
+ repository,
+ );
+ if (conflict) return conflict;
const current = openPulls.find(
(candidate) => candidate.number === pullNumber,
);
- return pullNumber === undefined ? null : pullHoldReason(current, branch);
+ return pullNumber === undefined
+ ? null
+ : pullHoldReason(current, branch, repository);
+}
+
+function nextCursor({ hasNextPage, endCursor }) {
+ if (!hasNextPage) return null;
+ if (!endCursor)
+ throw new Error("GitHub did not return the next page cursor.");
+ return endCursor;
+}
+
+// Only wait for a commit this run actually replaced. Read the PR, refs,
+// competing proposals and review markers together on the final page.
+async function reviewComments(github, plan, current, head, previous, wait) {
+ const { number } = current;
+ if (current.head.sha === head)
+ return {
+ comments: await github.list(`issues/${number}/comments?per_page=100`),
+ };
+ if (!previous) return { reason: pendingHeadReason };
+ const held = (reason) => ({ action: "held", reason });
+ const foreign = (sha) => sha !== head && sha !== previous;
+ if (foreign(current.head.sha)) return held(changedHeadReason);
+ for (const ms of [0, 1000, 2000, 4000]) {
+ if (ms) await wait(ms);
+ const comments = [];
+ let after = null;
+ let openAfter = null;
+ for (;;) {
+ const page = await github.releaseReviewPage(
+ number,
+ plan.branch,
+ after,
+ openAfter,
+ );
+ const pr = page?.pullRequest;
+ if (
+ pr?.state !== "OPEN" ||
+ pr.baseRefName !== "main" ||
+ pr.headRefName !== plan.branch ||
+ pr.headRepository?.nameWithOwner.toLowerCase() !==
+ github.repository.toLowerCase()
+ )
+ return held(closedPullReason);
+ if (page.main?.target?.oid !== plan.mainSha)
+ return held(
+ "main advanced after the release branch was updated. Let the next updater run, or rerun it manually before final review.",
+ );
+ if (page.branch?.target?.oid !== head || foreign(pr.headRefOid))
+ return held(changedHeadReason);
+ const open = page.open.nodes.map((p) => ({
+ number: p.number,
+ title: p.title,
+ head: {
+ ref: p.headRefName,
+ sha: p.headRefOid,
+ repo: { full_name: p.headRepository?.nameWithOwner },
+ },
+ base: { ref: p.baseRefName },
+ }));
+ const hold = conflictingReleaseReason(
+ open,
+ number,
+ plan.branch,
+ github.repository,
+ );
+ if (hold) return held(hold);
+ if (open.some((p) => p.number === number && foreign(p.head.sha)))
+ return held(changedHeadReason);
+ if (pr.headRefOid === previous) break;
+ const nextComments = nextCursor(pr.comments.pageInfo);
+ const nextOpen = nextCursor(page.open.pageInfo);
+ // Keep reading each connection's last page until both are complete.
+ // Newly opened or updated PRs are at the end of the open connection.
+ if (!nextComments && !nextOpen) {
+ comments.push(...pr.comments.nodes);
+ // Match the REST issue-comment order used for proposal marker selection.
+ return {
+ comments: comments.sort((a, b) =>
+ Number(BigInt(a.id) - BigInt(b.id)),
+ ),
+ };
+ }
+ if (nextComments) {
+ comments.push(...pr.comments.nodes);
+ after = nextComments;
+ }
+ if (nextOpen) openAfter = nextOpen;
+ }
+ }
+ return { reason: pendingHeadReason };
}
async function ensurePullRequest(
@@ -499,6 +609,8 @@ async function ensurePullRequest(
template,
headSha,
currentPulls,
+ previousSha,
+ waitForHead,
) {
const holdReason = releaseHoldReason(
currentPulls ?? (await github.list("pulls?state=open&per_page=100")),
@@ -510,7 +622,7 @@ async function ensurePullRequest(
return { action: "held", reason: holdReason, reviewRequested: false };
if (pull) {
pull = await github.request("GET", `pulls/${pull.number}`);
- const reason = pullHoldReason(pull, plan.branch);
+ const reason = pullHoldReason(pull, plan.branch, github.repository);
if (reason)
return {
action: "held",
@@ -533,7 +645,7 @@ async function ensurePullRequest(
});
}
const current = await github.request("GET", `pulls/${pull.number}`);
- const reason = pullHoldReason(current, plan.branch);
+ const reason = pullHoldReason(current, plan.branch, github.repository);
if (reason)
return {
action: "held",
@@ -541,20 +653,19 @@ async function ensurePullRequest(
pull: current.number,
reviewRequested: false,
};
- if (current.head.sha !== headSha) {
- return {
- pull: current.number,
- reviewRequested: false,
- reason:
- "GitHub has not exposed the updated PR head yet. Rerun the updater before final review.",
- };
- }
+ const { comments, ...stopped } = await reviewComments(
+ github,
+ plan,
+ current,
+ headSha,
+ previousSha,
+ waitForHead,
+ );
+ if (!comments)
+ return { pull: current.number, reviewRequested: false, ...stopped };
const marker = `<!-- release-pr-head: ${headSha} -->`;
const proposalMarker = `<!-- release-pr-proposal: ${hash("sha256", JSON.stringify(plan.files))} -->`;
const suggestionsMarker = `<!-- release-pr-suggestions: ${hash("sha256", JSON.stringify(plan.generated))} -->`;
- const comments = await github.list(
- `issues/${current.number}/comments?per_page=100`,
- );
const previous = comments.findLast((comment) =>
comment.body?.startsWith("<!-- release-pr-head: "),
);
@@ -585,6 +696,7 @@ export async function reconcileReleasePullRequest({
repo,
github,
dryRun = true,
+ waitForHead = delay,
}) {
// Rebuild the plan only when main or the proposal actually moves.
for (;;) {
@@ -698,6 +810,8 @@ export async function reconcileReleasePullRequest({
repo.readFile(mainSha, templatePath),
commitSha,
pull ? currentPulls : undefined,
+ pull && branchChanged ? headSha : null,
+ waitForHead,
);
return {
action: !pull ? "created" : changed ? "updated" : "unchanged",
@@ -709,6 +823,24 @@ export async function reconcileReleasePullRequest({
}
}
+const releaseReviewQuery = `query ReleaseReviewPage($owner: String!, $name: String!, $number: Int!, $ref: String!, $after: String, $openAfter: String) {
+ repository(owner: $owner, name: $name) {
+ branch: ref(qualifiedName: $ref) { target { oid } }
+ main: ref(qualifiedName: "refs/heads/main") { target { oid } }
+ open: pullRequests(states: OPEN, first: 100, after: $openAfter, orderBy: { field: UPDATED_AT, direction: ASC }) {
+ nodes { number title baseRefName headRefName headRefOid headRepository { nameWithOwner } }
+ pageInfo { hasNextPage endCursor }
+ }
+ pullRequest(number: $number) {
+ state baseRefName headRefName headRefOid headRepository { nameWithOwner }
+ comments(first: 100, after: $after) {
+ nodes { body id: fullDatabaseId }
+ pageInfo { hasNextPage endCursor }
+ }
+ }
+ }
+}`;
+
export function createGitHubClient(
repository,
token,
@@ -716,11 +848,15 @@ export function createGitHubClient(
sleep = delay,
) {
const apiRoot = `https://api.github.com/repos/${repository}/`;
- async function responseFor(method, path, body) {
+ async function responseFor(method, path, body, reviewQuery = false) {
+ const read = method === "GET" || reviewQuery;
+ const url = reviewQuery
+ ? "https://api.github.com/graphql"
+ : `${apiRoot}${path}`;
for (let retry = 0; ; retry += 1) {
let response;
try {
- response = await fetcher(`${apiRoot}${path}`, {
+ response = await fetcher(url, {
method,
headers: {
Accept: "application/vnd.github+json",
@@ -734,16 +870,11 @@ export function createGitHubClient(
return { data: await response.json(), headers: response.headers };
}
} catch (error) {
- if (method !== "GET" || retry >= 2 || error instanceof SyntaxError)
- throw error;
+ if (!read || retry >= 2 || error instanceof SyntaxError) throw error;
await sleep(1000 * 2 ** retry);
continue;
}
- if (
- method === "GET" &&
- retry < 2 &&
- [500, 502, 503, 504].includes(response.status)
- ) {
+ if (read && retry < 2 && [500, 502, 503, 504].includes(response.status)) {
await response.body?.cancel().catch(() => undefined);
await sleep(1000 * 2 ** retry);
continue;
@@ -762,6 +893,30 @@ export function createGitHubClient(
async request(method, path, body) {
return (await responseFor(method, path, body)).data;
},
+ async releaseReviewPage(number, branch, after, openAfter) {
+ const [owner, name] = repository.split("/");
+ const { data: result } = await responseFor(
+ "POST",
+ "graphql",
+ {
+ query: releaseReviewQuery,
+ variables: {
+ owner,
+ name,
+ number,
+ ref: `refs/heads/${branch}`,
+ after,
+ openAfter,
+ },
+ },
+ true,
+ );
+ if (result.errors?.length)
+ throw new Error(
+ `GitHub release review query failed: ${result.errors.map((e) => e.message).join("; ")}`,
+ );
+ return result.data.repository;
+ },
async list(path) {
const items = [];
for (let page = 1; ; page += 1) {
diff --git a/sdk/typescript/tests-ts/release-pr.test.ts b/sdk/typescript/tests-ts/release-pr.test.ts
index 8e820b7d7..8d7386480 100644
--- a/sdk/typescript/tests-ts/release-pr.test.ts
+++ b/sdk/typescript/tests-ts/release-pr.test.ts
@@ -54,6 +54,12 @@ type GitHubClient = {
repositoryUrl: string;
request: (method: string, path: string, body?: Body) => Promise<unknown>;
list: (path: string) => Promise<unknown[]>;
+ releaseReviewPage: (
+ number: number,
+ branch: string,
+ after: string | null,
+ openAfter: string | null,
+ ) => Promise<unknown>;
};
type Fetcher = (url: string, options: RequestInit) => Promise<Response>;
type Pull = {
@@ -121,6 +127,7 @@ const {
repo: GitRepository;
github: GitHubClient;
dryRun?: boolean;
+ waitForHead?: (ms: number) => Promise<void>;
}) => Promise<{
action: string;
reason?: string;
@@ -377,11 +384,12 @@ class Fixture {
return sha;
}
- run(dryRun = false) {
+ run(dryRun = false, waitForHead?: (ms: number) => Promise<void>) {
return reconcileReleasePullRequest({
repo: this.repo,
github: this.github,
dryRun,
+ ...(waitForHead && { waitForHead }),
});
}
}
@@ -394,6 +402,9 @@ class FakeGitHub {
readonly commitPulls = new Map<string, Pull[]>();
readonly comments = new Map<number, { body: string }[]>();
readonly writes: { method: string; path: string; body: Body }[] = [];
+ releaseReviewPage: GitHubClient["releaseReviewPage"] = async () => {
+ throw new Error("Unexpected release head recovery");
+ };
beforeRefWrite: (() => void) | undefined;
afterCommit: (() => void) | undefined;
failPullCreation = false;
@@ -542,6 +553,53 @@ class FakeGitHub {
}
}
+// The REST listing still shows the original pull head after the fake ref moves.
+async function laggingPull(title: string) {
+ const { fixture, first, pull } = await Fixture.withInitialPull();
+ const main = fixture.merge(title);
+ const { github } = fixture;
+ const comments = github.comments.get(pull.number)!;
+ const original = github.request.bind(github);
+ github.request = async (method, path, body) => {
+ if (method === "GET" && path === `pulls/${pull.number}`)
+ return structuredClone(pull);
+ return original(method, path, body);
+ };
+ const page = () => {
+ const branch = fixture.head(first.plan.branch)!;
+ const last = { hasNextPage: false, endCursor: null as string | null };
+ return {
+ main: { target: { oid: main } },
+ branch: { target: { oid: branch } as { oid: string } | null },
+ open: {
+ nodes: github.pulls
+ .filter((p) => p.state === "open")
+ .map((p) => ({
+ number: p.number,
+ title: p.title,
+ baseRefName: p.base.ref,
+ headRefName: p.head.ref,
+ headRefOid: p.head.sha,
+ headRepository: { nameWithOwner: p.head.repo.full_name },
+ })),
+ pageInfo: last,
+ },
+ pullRequest: {
+ state: "OPEN",
+ baseRefName: pull.base.ref,
+ headRefName: pull.head.ref,
+ headRefOid: branch,
+ headRepository: { nameWithOwner: github.repository },
+ comments: {
+ nodes: comments.map(({ body }, i) => ({ id: String(i + 1), body })),
+ pageInfo: last,
+ },
+ },
+ };
+ };
+ return { fixture, first, pull, comments, page, original };
+}
+
describe("GitHub request transport", () => {
function retryClient(fetcher: Fetcher) {
const delays: number[] = [];
@@ -556,6 +614,45 @@ describe("GitHub request transport", () => {
return { github, delays };
}
+ test("retries the same read-only review query and rejects a partial error response", async () => {
+ const requests: unknown[] = [];
+ const repository = { pullRequest: null };
+ const { github, delays } = retryClient(async (url, options) => {
+ expect(url).toBe("https://api.github.com/graphql");
+ expect(options.method).toBe("POST");
+ const { query, variables } = JSON.parse(String(options.body));
+ expect(query).toMatch(/^\s*query\b/u);
+ requests.push(variables);
+ if (requests.length === 1) return new Response(null, { status: 503 });
+ return Response.json({
+ data: { repository },
+ ...(requests.length === 3 && {
+ errors: [{ message: "Review page unavailable" }],
+ }),
+ });
+ });
+ const variables = {
+ owner: "example",
+ name: "release-fixture",
+ number: 42,
+ ref: "refs/heads/release/next",
+ after: "page-2",
+ openAfter: "open-3",
+ };
+ expect(
+ await github.releaseReviewPage(42, "release/next", "page-2", "open-3"),
+ ).toEqual(repository);
+ await expect(
+ github.releaseReviewPage(42, "release/next", null, null),
+ ).rejects.toThrow("Review page unavailable");
+ expect(requests).toEqual([
+ variables,
+ variables,
+ { ...variables, after: null, openAfter: null },
+ ]);
+ expect(delays).toEqual([1000]);
+ });
+
const interruptedJsonResponse = (error: Error) =>
new Response(
new ReadableStream({
@@ -1586,29 +1683,158 @@ describe("rolling release reconciliation", () => {
});
test("recovers review on a manual rerun after GitHub exposes the updated head", async () => {
- const { fixture, first } = await Fixture.withInitialPull();
- fixture.merge("fix: later fix");
- const request = fixture.github.request.bind(fixture.github);
- let headLagged = true;
- fixture.github.request = async (method, path, body) => {
- const result = structuredClone(await request(method, path, body));
- if (headLagged && method === "GET" && path === `pulls/${first.pull}`) {
- (result as Pull).head.sha = first.headSha!;
- }
- return result;
+ const { fixture, first, pull, page, comments, original } =
+ await laggingPull("fix!: later fix");
+ fixture.github.releaseReviewPage = async () => {
+ const snapshot = page();
+ snapshot.pullRequest.headRefOid = first.headSha!;
+ return snapshot;
};
- const pending = await fixture.run();
+ const delays: number[] = [];
+ const pending = await fixture.run(false, async (ms) => {
+ expect(pull.title).toBe("chore(release): 0.2.0");
+ delays.push(ms);
+ });
expect(pending.reviewRequested).toBe(false);
expect(pending.reason).toContain("Rerun");
- expect(fixture.github.comments.get(first.pull!)).toHaveLength(1);
- headLagged = false;
+ expect(delays).toEqual([1000, 2000, 4000]);
+ expect(comments).toHaveLength(1);
+ const count = fixture.github.writes.length;
+ expect((await fixture.run()).reviewRequested).toBe(false); // No new ref write to wait for.
+ expect(fixture.github.writes).toHaveLength(count);
+ fixture.github.request = original;
const recovered = await fixture.run();
expect(recovered.action).toBe("unchanged");
expect(recovered.headSha).toBe(pending.headSha);
expect(recovered.reviewRequested).toBe(true);
- expect(fixture.github.comments.get(first.pull!)).toHaveLength(2);
+ expect(comments).toHaveLength(2);
});
+ test.each([
+ "changed plan",
+ "proposal",
+ "suggestions",
+ "same",
+ "release moved",
+ "main moved",
+ "retargeted",
+ "competing PR",
+ "competing head",
+ ])(
+ "checks the latest target and marker when the release PR head catches up: %s",
+ async (change) => {
+ const { fixture, first, pull, comments, page } = await laggingPull(
+ change === "changed plan"
+ ? "fix!: change the public interface"
+ : "test: more coverage",
+ );
+ const initial = comments[0]!.body;
+ const digest = change === "competing PR" ? "proposal" : change;
+ const newer = initial.replace(
+ new RegExp(`release-pr-${digest}: [a-f0-9]+`),
+ `release-pr-${digest}: ${"0".repeat(64)}`,
+ );
+ let visible = false;
+ const cursors: [string | null, string | null][] = [];
+ const writes = fixture.github.writes.length;
+ fixture.github.releaseReviewPage = async (
+ number,
+ branch,
+ after,
+ openAfter,
+ ) => {
+ expect([number, branch]).toEqual([pull.number, first.plan.branch]);
+ if (visible && after && change === "competing PR")
+ fixture.github.addPull(
+ "manual",
+ "release: alternate",
+ first.headSha!,
+ );
+ const snapshot = page();
+ const pr = snapshot.pullRequest;
+ if (!visible) {
+ pr.headRefOid = first.headSha!;
+ return snapshot;
+ }
+ cursors.push([after, openAfter]);
+ // Comments need two pages; a long open-PR list needs three. A new
+ // marker can arrive as the last comments page is read again.
+ if (change === "proposal" && openAfter !== "last") {
+ snapshot.open.nodes = [];
+ snapshot.open.pageInfo = {
+ hasNextPage: true,
+ endCursor: openAfter ? "last" : "more",
+ };
+ }
+ pr.comments.nodes = after
+ ? [{ id: "9007199254740992", body: initial }]
+ : [
+ {
+ id: "9007199254740993",
+ body: change === "proposal" ? initial : newer,
+ },
+ ];
+ if (openAfter === "last")
+ pr.comments.nodes.push({ id: "9007199254740994", body: newer });
+ if (!after)
+ pr.comments.pageInfo = { hasNextPage: true, endCursor: "older" };
+ else if (change === "release moved") snapshot.branch.target = null;
+ else if (change === "main moved")
+ snapshot.main.target.oid = first.headSha!;
+ else if (change === "retargeted") pr.baseRefName = "maintenance";
+ return snapshot;
+ };
+ const result = await fixture.run(false, async () => {
+ visible = true;
+ if (change === "changed plan")
+ expect(pull.title).toBe("chore(release): 0.2.0");
+ if (change === "competing head") pull.head.sha = "c".repeat(40);
+ });
+ expect(visible).toBe(true);
+ expect(
+ fixture.github.writes
+ .slice(writes)
+ .filter((w) => w.path.startsWith("git/refs")),
+ ).toHaveLength(1);
+ expect(cursors).toEqual(
+ change === "proposal"
+ ? [
+ [null, null],
+ ["older", "more"],
+ ["older", "last"],
+ ]
+ : change === "competing head"
+ ? [[null, null]]
+ : [
+ [null, null],
+ ["older", null],
+ ],
+ );
+ if (
+ ["changed plan", "proposal", "suggestions", "same"].includes(change)
+ ) {
+ const eligible = change === "proposal" || change === "changed plan";
+ expect(result.action).toBe("updated");
+ expect(result.reviewRequested).toBe(eligible);
+ expect(comments).toHaveLength(change === "same" ? 1 : 2);
+ if (change !== "same") {
+ expect(comments[1]!.body).toContain(
+ `<!-- release-pr-head: ${result.headSha} -->`,
+ );
+ expect(
+ comments[1]!.body.includes(
+ `@codex review the current head ${result.headSha}.`,
+ ),
+ ).toBe(eligible);
+ }
+ } else {
+ expect(result.action).toBe("held");
+ expect(result.reviewRequested).toBe(false);
+ expect(comments).toHaveLength(1);
+ }
+ },
+ );
+
test("requests review when the proposal changes back to previously reviewed content", async () => {
const fixture = new Fixture();
const merged = fixture.merge("feat: initial feature");