OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { delimiter, join } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { afterEach, describe, expect, spyOn, test } from "bun:test";
import { publishGitHubRelease } from "../scripts/publish-github-release.mjs";
import { createTemporaryDirectoriesSync } from "./support/temporary-directories.js";
import { bashCommand, runCommand, writeSource } from "./support/shell.js";
import { workflow } from "./support/workflow.js";
import {
type ReleaseMetadata,
parseReviewedReleaseNotes,
extractHistoricalReleaseSummary,
resolveReleaseSummary,
composeReleaseNotes,
releaseVersion,
releaseTagVersion,
compareReleaseVersions,
requireReleaseIncrease,
publishedReleaseMode,
requirePublishedReleaseIncrease,
releaseHistory,
verifyPublishedRelease,
verifyGitHubPublishedRelease,
verifySignatureAudit,
verifyRecoveredSignatureAudit,
verifyGitHubRelease,
} from "../scripts/release-automation.mjs";
const automationScript = new URL(
"../scripts/release-automation.mjs",
import.meta.url,
);
const releaseCommit = "1e03c89ad22d2df5ae65b146be1483b3608572a9";
const releaseRun = "30481596229";
const releaseRepository = "openai/codex-security";
const combinedPublisher = "Publish npm and container images";
const recoveryPublisher = "Verify existing npm release";
const releaseWorkflowTimeout = process.platform === "win32" ? 20_000 : 10_000;
const directories = createTemporaryDirectoriesSync();
afterEach(directories.cleanup);
const bash = bashCommand();
function runShell(
script: string,
env: NodeJS.ProcessEnv,
options: { cwd?: string; timeout?: number } = {},
) {
return runCommand(bash, ["-c", script], {
...options,
env: { ...process.env, ...env },
});
}
function runPublisher({
existingLookup = "existing",
existingNotes = "Generated release notes",
generatedNotes = "Generated release notes\n",
generateNotesResult = {
status: 0,
stdout: JSON.stringify({ body: generatedNotes }),
stderr: "",
},
latestTag = "npm-v0.1.3",
makeLatest = false,
releaseConfiguration = true,
releaseSummary = "__missing__",
tagType = "commit",
tagObject = releaseCommit,
peeledCommit = "",
downloadedArchive = archive,
} = {}) {
const workspace = mkdtempSync(join(tmpdir(), "github release "));
const archivePath = join(workspace, "verified archive.tgz");
writeFileSync(archivePath, archive);
const calls: string[][] = [];
let stdout = "",
stderr = "",
status = 0;
const logs = spyOn(console, "log").mockImplementation((text) => {
stdout += `${text}\n`;
});
const errors = spyOn(console, "error").mockImplementation((text) => {
stderr += `${text}\n`;
});
const response = (code: number, body: unknown) => ({
status: code === 200 ? 0 : 1,
stdout: `HTTP/2.0 ${code} Response\r\ncontent-type: application/json\r\n\r\n${JSON.stringify(body)}\n`,
});
const lookup = (kind: string) => {
if (kind === "network" || kind === "__network__")
return { status: 1, stdout: "" };
if (kind === "malformed" || kind === "__malformed_missing__")
return { status: 1, stdout: "HTTP/2.0 404 Not Found\n\nnot-json" };
return response(
kind === "unauthorized"
? 401
: kind === "unavailable" || kind === "__error__"
? 500
: 404,
{ message: "Unavailable" },
);
};
try {
publishGitHubRelease(
{
GITHUB_REPOSITORY: "test/codex-security",
RELEASE_TAG: "npm-v0.1.2",
RELEASE_VERSION: "0.1.2",
RELEASE_SHA: releaseCommit,
RELEASE_ARCHIVE: archivePath,
},
workspace,
(program: string, args: string[]) => {
calls.push([program, ...args]);
if (program === "npm") {
expect(args).toEqual([
"view",
"@openai/codex-security",
"versions",
"--json",
"--registry=https://registry.npmjs.org/",
]);
return {
status: 0,
stdout: JSON.stringify(makeLatest ? ["0.1.2"] : ["0.1.2", "0.1.3"]),
};
}
if (program === "git" && args[0] === "rev-parse")
return { status: 0, stdout: "" };
if (program === "git" && args[0] === "tag") {
expect(args).toEqual([
"tag",
"--merged",
`${releaseCommit}^`,
"--list",
"npm-v*",
"--sort=-version:refname",
]);
return { status: 0, stdout: "npm-v0.1.1\nnpm-v0.1.0\n" };
}
const endpoint = args.find((argument) => argument.startsWith("repos/"));
if (program === "gh" && args[0] === "api") {
switch (endpoint) {
case "repos/test/codex-security/releases?per_page=100":
expect(args).toContain("--paginate");
expect(args).toContain(
".[] | select(.draft == false and .prerelease == false) | .tag_name",
);
return { status: 0, stdout: "npm-v0.1.1\n" };
case "repos/test/codex-security/releases/tags/npm-v0.1.2":
return existingLookup !== "existing"
? lookup(existingLookup)
: response(200, {
tag_name: "npm-v0.1.2",
draft: false,
prerelease: false,
body: existingNotes,
assets: [{ name: "verified archive.tgz", digest }],
});
case "repos/test/codex-security/releases/latest":
return latestTag.startsWith("__")
? lookup(latestTag)
: response(200, { tag_name: latestTag });
case "repos/test/codex-security/git/ref/tags/npm-v0.1.2":
return {
status: tagType === "missing" ? 1 : 0,
stdout: `${tagType}\t${tagObject}\n`,
};
case `repos/test/codex-security/git/tags/${tagObject}`:
return { status: 0, stdout: `${peeledCommit}\n` };
case "repos/test/codex-security/releases/generate-notes":
expect(args).toContain("previous_tag_name=npm-v0.1.1");
expect(
args.includes("configuration_file_path=.github/release.yml"),
).toBe(releaseConfiguration);
if (generateNotesResult.stderr)
console.error(generateNotesResult.stderr);
return generateNotesResult;
}
}
if (program === "git" && args[0] === "cat-file")
return { status: releaseConfiguration ? 0 : 1, stdout: "" };
if (program === "git" && args[0] === "show")
return {
status: releaseSummary === "__missing__" ? 1 : 0,
stdout: `${releaseSummary}\n`,
};
if (program === "gh" && args[0] === "release") {
const value = (flag: string) => args[args.indexOf(flag) + 1]!;
if (args[1] === "download")
writeFileSync(
join(value("--dir"), value("--pattern")),
downloadedArchive,
);
else if (args[1] === "edit" || args[1] === "create") {
expect(args).toContain(`--latest=${makeLatest}`);
expect(args.includes("--verify-tag")).toBe(args[1] === "create");
if (args[1] === "create")
console.log("created verified GitHub release");
else {
console.log(`updated latest: ${makeLatest}`);
if (args.includes("--notes-file"))
console.log(
`updated release notes: ${readFileSync(value("--notes-file"), "utf8")}`,
);
}
} else throw new Error(`Unexpected release command: ${args}`);
return { status: 0, stdout: "" };
}
throw new Error(`Unexpected publication command: ${program} ${args}`);
},
);
} catch (error) {
if (error instanceof Error) {
status = 1;
stderr += error.message;
} else status = (error as { status: number }).status;
} finally {
logs.mockRestore();
errors.mockRestore();
rmSync(workspace, { recursive: true, force: true });
}
return { status, stdout, stderr, calls };
}
function releaseLabelGhMock(handlers: readonly string[]) {
return [
"gh() {",
' if [[ "$1" != "api" ]]; then return 64; fi',
" shift",
" local method=GET",
' if [[ "${1:-}" == "--method" ]]; then',
' method="$2"',
" shift 2",
" fi",
' local endpoint="$1"',
" shift",
' case "$method $endpoint" in',
' "GET repos/test/codex-security/issues/17")',
' [[ $# == 2 && "$1" == --jq ]] || return 64',
` jq ${process.platform === "win32" ? "--binary " : ""}-r "$2" <<< "$MOCK_ISSUE"`,
" ;;",
...handlers,
' "POST repos/test/codex-security/issues/17/labels")',
" printf '%s\\n' \"$@\"",
" ;;",
" *) return 65 ;;",
" esac",
"}",
].join("\n");
}
const jqMock = [
"jq() {",
' node -e \'const fs=require("node:fs");const filter=process.argv.at(-1);const value=JSON.parse(fs.readFileSync(0,"utf8"));if(filter==="[.object.type, .object.sha] | @tsv"){const fields=[value.object?.type,value.object?.sha];if(fields.some((field)=>typeof field!=="string"))process.exit(1);process.stdout.write(fields.join("\\t")+"\\n");}else if(filter===".status // empty"){if(value.status!=null)process.stdout.write(String(value.status)+"\\n");}else process.exit(64);\' -- "$@"',
"}",
].join("\n");
const releaseSigningCertificate =
"MIIHOjCCBr+gAwIBAgIUDDD6xE6tccKRAzn6GcB6Ajvw2+swCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3Rv" +
"cmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYwNzI5MTg1MTA1WhcNMjYwNzI5MTkw" +
"MTA1WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEkin3vstve90HTjDjvA07JK3um96wz1+wu9IbeAOgqPZW" +
"7Ijzx++FAEBWmRp9dJiLOa3xlHM1ZEqjZc3FlbBQrqOCBd4wggXaMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAK" +
"BggrBgEFBQcDAzAdBgNVHQ4EFgQUn/ffAFOuPTzNV/Vezq93CcIME+wwHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjp" +
"KFWixi4YZD8wbgYDVR0RAQH/BGQwYoZgaHR0cHM6Ly9naXRodWIuY29tL29wZW5haS9jb2RleC1zZWN1cml0eS8u" +
"Z2l0aHViL3dvcmtmbG93cy9ub2RlLXJlbGVhc2UueW1sQHJlZnMvdGFncy9ucG0tdjAuMS4yMDkGCisGAQQBg78w" +
"AQEEK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wEgYKKwYBBAGDvzABAgQEcHVz" +
"aDA2BgorBgEEAYO/MAEDBCgxZTAzYzg5YWQyMmQyZGY1YWU2NWIxNDZiZTE0ODNiMzYwODU3MmE5MBoGCisGAQQB" +
"g78wAQQEDG5vZGUtcmVsZWFzZTAjBgorBgEEAYO/MAEFBBVvcGVuYWkvY29kZXgtc2VjdXJpdHkwIgYKKwYBBAGD" +
"vzABBgQUcmVmcy90YWdzL25wbS12MC4xLjIwOwYKKwYBBAGDvzABCAQtDCtodHRwczovL3Rva2VuLmFjdGlvbnMu" +
"Z2l0aHVidXNlcmNvbnRlbnQuY29tMHAGCisGAQQBg78wAQkEYgxgaHR0cHM6Ly9naXRodWIuY29tL29wZW5haS9j" +
"b2RleC1zZWN1cml0eS8uZ2l0aHViL3dvcmtmbG93cy9ub2RlLXJlbGVhc2UueW1sQHJlZnMvdGFncy9ucG0tdjAu" +
"MS4yMDgGCisGAQQBg78wAQoEKgwoMWUwM2M4OWFkMjJkMmRmNWFlNjViMTQ2YmUxNDgzYjM2MDg1NzJhOTAdBgor" +
"BgEEAYO/MAELBA8MDWdpdGh1Yi1ob3N0ZWQwOAYKKwYBBAGDvzABDAQqDChodHRwczovL2dpdGh1Yi5jb20vb3Bl" +
"bmFpL2NvZGV4LXNlY3VyaXR5MDgGCisGAQQBg78wAQ0EKgwoMWUwM2M4OWFkMjJkMmRmNWFlNjViMTQ2YmUxNDgz" +
"YjM2MDg1NzJhOTAkBgorBgEEAYO/MAEOBBYMFHJlZnMvdGFncy9ucG0tdjAuMS4yMBoGCisGAQQBg78wAQ8EDAwK" +
"MTI5OTc2OTIyMDApBgorBgEEAYO/MAEQBBsMGWh0dHBzOi8vZ2l0aHViLmNvbS9vcGVuYWkwGAYKKwYBBAGDvzAB" +
"EQQKDAgxNDk1NzA4MjBwBgorBgEEAYO/MAESBGIMYGh0dHBzOi8vZ2l0aHViLmNvbS9vcGVuYWkvY29kZXgtc2Vj" +
"dXJpdHkvLmdpdGh1Yi93b3JrZmxvd3Mvbm9kZS1yZWxlYXNlLnltbEByZWZzL3RhZ3MvbnBtLXYwLjEuMjA4Bgor" +
"BgEEAYO/MAETBCoMKDFlMDNjODlhZDIyZDJkZjVhZTY1YjE0NmJlMTQ4M2IzNjA4NTcyYTkwFAYKKwYBBAGDvzAB" +
"FAQGDARwdXNoMFwGCisGAQQBg78wARUETgxMaHR0cHM6Ly9naXRodWIuY29tL29wZW5haS9jb2RleC1zZWN1cml0" +
"eS9hY3Rpb25zL3J1bnMvMzA0ODE1OTYyMjkvYXR0ZW1wdHMvMTAWBgorBgEEAYO/MAEWBAgMBnB1YmxpYzATBgor" +
"BgEEAYO/MAEXBAUMA25wbTA6BgorBgEEAYO/MAEYBCwMKnJlcG86b3BlbmFpL2NvZGV4LXNlY3VyaXR5OmVudmly" +
"b25tZW50Om5wbTCBigYKKwYBBAHWeQIEAgR8BHoAeAB2AN09MGrGxxEyYxkeHJlnNwKiSl643jyt/4eKcoAvKe6O" +
"AAABn683UUoAAAQDAEcwRQIhAIByLL09iV3Tt78+V79VHOAcTGiwBe8ZQJO2YfWKr5CcAiAYJQpfxgVSyerx1dTC" +
"SivSEzQt/ABuKvEHEFavAb+qCzAKBggqhkjOPQQDAwNpADBmAjEA09chNJjy7FhYVY6n7ioITfLzDBs9oaHuGFrD" +
"HXYnMKbVXVlkt2fVM8WjllMQitjhAjEA2m9qBcOod9M8uMCw76eVJk3YloyAhcTDZfMTtMWaM5DpNG4v/vEZ+MIt" +
"zIiMHK/0";
const archive = Buffer.from("verified codex security release artifact");
const integrity =
"sha512-" + createHash("sha512").update(archive).digest("base64");
const sha512 = createHash("sha512").update(archive).digest("hex");
const digest = "sha256:" + createHash("sha256").update(archive).digest("hex");
const protectedReleaseWorkflow = readFileSync(
new URL("../../../.github/workflows/node-release.yml", import.meta.url),
"utf8",
);
const darwinSigningWorkflow = readFileSync(
new URL(
"../../../.github/workflows/sign-darwin-release.yml",
import.meta.url,
),
"utf8",
);
const releaseCutWorkflow = readFileSync(
new URL("../../../.github/workflows/node-release-cut.yml", import.meta.url),
"utf8",
);
const githubReleaseWorkflow = readFileSync(
new URL(
"../../../.github/workflows/node-github-release.yml",
import.meta.url,
),
"utf8",
);
const releaseLabelsWorkflow = readFileSync(
new URL(
"../../../.github/workflows/node-release-labels.yml",
import.meta.url,
),
"utf8",
);
const nodeCiWorkflow = readFileSync(
new URL("../../../.github/workflows/node-ci.yml", import.meta.url),
"utf8",
);
const titleWorkflow = readFileSync(
new URL("../../../.github/workflows/validate-pr-title.yml", import.meta.url),
"utf8",
);
function publishedMetadata(): ReleaseMetadata {
return {
name: "@openai/codex-security",
version: "0.1.2",
gitHead: releaseCommit,
"dist.integrity": integrity,
"dist.attestations": {
provenance: {
predicateType: "https://slsa.dev/provenance/v1",
},
},
};
}
function githubRelease(): ReleaseMetadata {
return {
tag_name: "npm-v0.1.2",
draft: false,
prerelease: false,
assets: [{ name: "openai-codex-security-0.1.2.tgz", digest }],
};
}
type SignatureAuditFixture = {
name?: string;
version?: string;
registry?: string;
invalid?: unknown[];
missing?: unknown[];
includeVerified?: boolean;
includeProvenance?: boolean;
includeBundle?: boolean;
signingCertificate?: string | null;
certificateLocation?: "certificate" | "chain";
subjectName?: string;
subjectDigest?: string;
repository?: string;
workflowPath?: string;
workflowRef?: string;
sourceCommit?: string;
runId?: string;
attempt?: string;
builder?: string;
};
function signatureAudit(options: SignatureAuditFixture = {}): ReleaseMetadata {
const version = options.version ?? "0.1.2";
const repository = options.repository ?? releaseRepository;
const releaseRef = options.workflowRef ?? `refs/tags/npm-v${version}`;
const signingCertificate =
options.signingCertificate === undefined
? releaseSigningCertificate
: options.signingCertificate;
const verificationMaterial =
options.certificateLocation === "chain"
? {
x509CertificateChain: {
certificates: [{ rawBytes: signingCertificate }],
},
}
: { certificate: { rawBytes: signingCertificate } };
const statement = {
_type: "https://in-toto.io/Statement/v1",
subject: [
{
name:
options.subjectName ?? `pkg:npm/%40openai/codex-security@${version}`,
digest: { sha512: options.subjectDigest ?? sha512 },
},
],
predicateType: "https://slsa.dev/provenance/v1",
predicate: {
buildDefinition: {
externalParameters: {
workflow: {
repository: `https://github.com/${repository}`,
ref: releaseRef,
path: options.workflowPath ?? ".github/workflows/node-release.yml",
},
},
resolvedDependencies: [
{
uri: `git+https://github.com/${repository}@${releaseRef}`,
digest: { gitCommit: options.sourceCommit ?? releaseCommit },
},
],
},
runDetails: {
builder: {
id:
options.builder ??
"https://github.com/actions/runner/github-hosted",
},
metadata: {
invocationId:
`https://github.com/${repository}/actions/runs/` +
`${options.runId ?? releaseRun}/attempts/${options.attempt ?? "1"}`,
},
},
},
};
return {
invalid: options.invalid ?? [],
missing: options.missing ?? [],
verified:
options.includeVerified === false
? []
: [
{
name: options.name ?? "@openai/codex-security",
version,
registry: options.registry ?? "https://registry.npmjs.org/",
attestations:
options.includeProvenance === false
? {}
: {
provenance: {
predicateType: "https://slsa.dev/provenance/v1",
},
},
attestationBundles:
options.includeBundle === false
? []
: [
{
predicateType: "https://slsa.dev/provenance/v1",
bundle: {
verificationMaterial,
dsseEnvelope: {
payload: Buffer.from(
JSON.stringify(statement),
).toString("base64"),
},
},
},
],
},
],
};
}
function signingCertificateWithReplacement(
original: string,
replacement: string,
): string {
const needle = Buffer.from(original);
const substitute = Buffer.from(replacement);
if (needle.length !== substitute.length) {
throw new Error(
"Signing certificate replacements must preserve DER lengths.",
);
}
const certificate = Buffer.from(releaseSigningCertificate, "base64");
let offset = certificate.indexOf(needle);
if (offset === -1) {
throw new Error(
"Signing certificate does not contain the requested value.",
);
}
while (offset !== -1) {
substitute.copy(certificate, offset);
offset = certificate.indexOf(needle, offset + substitute.length);
}
return certificate.toString("base64");
}
function signatureExpected() {
return {
version: "0.1.2",
gitHead: releaseCommit,
repository: releaseRepository,
runId: releaseRun,
};
}
function verifySignatureFixture(options: SignatureAuditFixture = {}) {
return verifySignatureAudit(
signatureAudit(options),
archive,
signatureExpected(),
);
}
type WorkflowContract = {
on?: { workflow_call?: { secrets?: Record<string, { required?: boolean }> } };
concurrency?: Record<string, unknown>;
jobs: Record<
string,
{
name?: string;
uses?: string;
with?: Record<string, unknown>;
secrets?: Record<string, string>;
"cache-mode"?: string;
outputs?: Record<string, string>;
permissions?: Record<string, string>;
env?: Record<string, string>;
needs?: string | string[];
environment?: string;
if?: string;
strategy?: { matrix: Record<string, unknown>; "fail-fast"?: boolean };
concurrency?: Record<string, unknown>;
"continue-on-error"?: boolean;
steps?: {
name?: string;
uses?: string;
run?: string;
if?: string;
"continue-on-error"?: boolean;
with?: Record<string, unknown>;
env?: Record<string, string>;
}[];
}
>;
};
function workflowStepShell(workflow: string, stepName: string): string {
const parsed = Bun.YAML.parse(workflow) as {
jobs: Record<string, { steps?: { name?: string; run?: string }[] }>;
};
const step = Object.values(parsed.jobs)
.flatMap((job) => job.steps ?? [])
.find((step) => step.name === stepName);
if (step === undefined) {
throw new Error(`Workflow step is missing: ${stepName}`);
}
if (typeof step.run !== "string") {
throw new Error(`Workflow step has no shell script: ${stepName}`);
}
return step.run;
}
function resolveProtectedRelease(env: Record<string, string>) {
const script = workflowStepShell(
githubReleaseWorkflow,
"Resolve the successful protected release",
);
return runShell(
String.raw`
sleep() {
[[ $# == 1 && -n "$MOCK_SLEEP_LOG" ]] || return 99
printf '%s\n' "$1" >> "$MOCK_SLEEP_LOG"
MOCK_RUN_STATUS=$MOCK_NEXT_STATUS
}
git() {
[[ $# == 3 && "$1" == rev-parse && "$2" == --verify && "$3" == "$MOCK_TAG_OBJECT^{commit}" ]] || return 66
[[ "$MOCK_TAG_TYPE" == commit ]] || return 67
printf '%s\n' "$MOCK_TAG_OBJECT"
}
gh() {
printf '%s\n' "$*" >> "$MOCK_GH_LOG"
[[ $# == 4 && "$1" == api && "$3" == --jq ]] || return 64
case "$2" in
"repos/test/codex-security/git/ref/tags/npm-v0.1.2")
[[ -n "$MOCK_TAG_TYPE" ]] || return 1
[[ "$4" == '[.object.type, .object.sha] | @tsv' ]] || return 65
printf '%s\t%s\n' "$MOCK_TAG_TYPE" "$MOCK_TAG_OBJECT"
;;
"repos/test/codex-security/git/tags/$MOCK_TAG_OBJECT")
[[ "$MOCK_TAG_TYPE" == tag && "$4" == 'if .object.type == "commit" then .object.sha else empty end' ]] || return 65
printf '%s\n' '${releaseCommit}'
;;
"repos/test/codex-security/commits/npm-v0.1.2")
printf '%s\n' "$MOCK_BRANCH_SHA"
;;
"repos/test/codex-security/actions/runs/$INPUT_RUN_ID")
[[ "$4" == '[.path, .event, .status, (.conclusion // "pending"), .head_sha, .head_branch] | @tsv' ]] || return 65
local conclusion=pending
if [[ "$MOCK_RUN_STATUS" == completed ]]; then conclusion=success; fi
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$MOCK_WORKFLOW_PATH" "$MOCK_RUN_EVENT" "$MOCK_RUN_STATUS" "$conclusion" "$MOCK_RUN_SHA" npm-v0.1.2
;;
*) return 65 ;;
esac
}
${script}`,
{
GITHUB_OUTPUT: "/dev/null",
GITHUB_REPOSITORY: "test/codex-security",
INPUT_RUN_ID: releaseRun,
INPUT_TAG: "npm-v0.1.2",
MOCK_TAG_TYPE: "commit",
MOCK_TAG_OBJECT: releaseCommit,
MOCK_RUN_STATUS: "completed",
MOCK_RUN_EVENT: "workflow_dispatch",
MOCK_NEXT_STATUS: "completed",
MOCK_SLEEP_LOG: "",
MOCK_RUN_SHA: releaseCommit,
MOCK_WORKFLOW_PATH:
".github/workflows/node-release.yml@refs/tags/npm-v0.1.2",
MOCK_BRANCH_SHA: releaseCommit,
MOCK_GH_LOG: "/dev/null",
...env,
},
);
}
function runReleaseLabels(
title: string,
routes: readonly string[],
setup: readonly string[] = [],
) {
const mock = [...setup, releaseLabelGhMock(routes)].join("\n");
const script = workflowStepShell(
releaseLabelsWorkflow,
"Categorize pull request without checking out its code",
);
return runShell(`${mock}\n${script}`, {
GITHUB_REPOSITORY: "test/codex-security",
MOCK_ISSUE: JSON.stringify({ title }),
PR_NUMBER: "17",
});
}
async function evaluateWorkflowCondition(
condition: string,
values: Record<string, string>,
): Promise<boolean> {
return (await evaluateWorkflowConditions([{ condition, values }]))[0]!;
}
async function evaluateWorkflowConditions(
conditions: Array<{ condition: string; values: Record<string, string> }>,
): Promise<boolean[]> {
const expressions = conditions.map(({ condition, values }) => {
let expression = condition
.replace(/^\$\{\{\s*/u, "")
.replace(/\s*\}\}$/u, "");
for (const [identifier, value] of Object.entries(values).sort(
([left], [right]) => right.length - left.length,
)) {
if (!/^[a-z0-9_/-]+$/u.test(value)) {
throw new Error(`Unsafe workflow test value: ${value}`);
}
expression = expression.replaceAll(identifier, `'${value}'`);
}
if (!/^[\s()'/a-z0-9_!=&|-]+$/u.test(expression)) {
throw new Error(`Unsupported workflow condition: ${condition}`);
}
return `[[ ${expression} ]]
result=$?
if [[ $result != 0 && $result != 1 ]]; then exit "$result"; fi
printf '%s\n' "$result"`;
});
const result = await runCommand(bash, [], {
input: expressions.join("\n"),
timeout: 30_000,
});
if (result.status !== 0) {
throw new Error(
`Could not evaluate workflow conditions (status ${result.status}, signal ${result.signal}):\n${result.stderr}`,
{ cause: result.error },
);
}
return result.stdout
.trimEnd()
.split("\n")
.map((status) => status === "0");
}
describe("reviewed release note helpers", () => {
test.each([
{
name: "one-line summary",
notes: "<!-- release-version: 1.2.3 -->\nReviewed summary\n",
summary: "Reviewed summary",
},
{
name: "multiline summary with trailing newlines",
notes: "<!-- release-version: 1.2.3 -->\n## Highlights\n\nDetails\n\n",
summary: "## Highlights\n\nDetails",
},
])("parses $name", ({ notes, summary }) => {
expect(parseReviewedReleaseNotes("1.2.3", notes)).toBe(summary);
});
test.each([
["missing header", "Reviewed summary"],
["mismatched version", "<!-- release-version: 1.2.2 -->\nSummary"],
["missing body", "<!-- release-version: 1.2.3 -->"],
["blank body", "<!-- release-version: 1.2.3 -->\n \t\n"],
["NUL-only body", "<!-- release-version: 1.2.3 -->\n\0"],
["NUL-bearing body", "<!-- release-version: 1.2.3 -->\nReviewed\0summary"],
])("rejects %s", (_name, notes) => {
expect(() => parseReviewedReleaseNotes("1.2.3", notes)).toThrow(
"Release notes must start with <!-- release-version: 1.2.3 --> and include a reviewed summary.",
);
});
test("rejects Unicode whitespace-only reviewed summaries", () => {
for (const space of ["\u00a0", "\u2007", "\u202f", "\ufeff"]) {
expect(() =>
parseReviewedReleaseNotes(
"1.2.3",
`<!-- release-version: 1.2.3 -->\n${space}`,
),
).toThrow("include a reviewed summary");
}
});
test.each([
{
name: "canonical prefix marker pair",
notes:
"<!-- codex-security-release-summary:start -->\nHistorical summary\n<!-- codex-security-release-summary:end -->\n\nGenerated",
summary: "Historical summary",
},
{
name: "CRLF standalone marker pair",
notes:
"<!-- codex-security-release-summary:start -->\r\nHistorical summary\r\n<!-- codex-security-release-summary:end -->\r\n\r\nGenerated",
summary: "Historical summary",
},
{
name: "marker-like inline substrings",
notes:
"<!-- codex-security-release-summary:start -->\nHistorical <!-- codex-security-release-summary:start --> summary\n<!-- codex-security-release-summary:end -->\n\nGenerated <!-- codex-security-release-summary:end -->",
summary:
"Historical <!-- codex-security-release-summary:start --> summary",
},
{
name: "inline markers in generated notes",
notes:
"* fix: <!-- codex-security-release-summary:start -->Unreviewed<!-- codex-security-release-summary:end --> by @contributor",
summary: null,
},
{
name: "standalone markers outside the canonical prefix",
notes:
"* fix: generated title\n<!-- codex-security-release-summary:start -->\nUnreviewed\n<!-- codex-security-release-summary:end -->",
summary: null,
},
{
name: "no marker pair",
notes: "Generated only",
summary: null,
},
])("extracts $name", ({ notes, summary }) => {
expect(extractHistoricalReleaseSummary(notes)).toBe(summary);
});
test.each([
[
"missing end marker",
"<!-- codex-security-release-summary:start -->\nSummary",
"Existing release summary markers are malformed.",
],
[
"duplicate marker pair after a blank line",
"<!-- codex-security-release-summary:start -->\nFirst\n<!-- codex-security-release-summary:end -->\n\n<!-- codex-security-release-summary:start -->\nSecond\n<!-- codex-security-release-summary:end -->",
"Existing release summary markers are malformed.",
],
[
"extra standalone marker after the canonical pair",
"<!-- codex-security-release-summary:start -->\nSummary\n<!-- codex-security-release-summary:end -->\n\nGenerated\n<!-- codex-security-release-summary:start -->",
"Existing release summary markers are malformed.",
],
[
"generated notes without a blank separator",
"<!-- codex-security-release-summary:start -->\nSummary\n<!-- codex-security-release-summary:end -->\nGenerated",
"Existing release summary markers are malformed.",
],
[
"blank historical summary",
"<!-- codex-security-release-summary:start -->\n \t\n<!-- codex-security-release-summary:end -->",
"Existing release summary is empty.",
],
[
"NUL-only historical summary",
"<!-- codex-security-release-summary:start -->\n\0\n<!-- codex-security-release-summary:end -->",
"Existing release summary is empty.",
],
[
"NUL-bearing historical summary",
"<!-- codex-security-release-summary:start -->\nReviewed\0summary\n<!-- codex-security-release-summary:end -->",
"Existing release summary is empty.",
],
])("rejects %s", (_name, notes, message) => {
expect(() => extractHistoricalReleaseSummary(notes)).toThrow(message);
});
test("prefers a tagged summary over malformed historical markers", () => {
expect(
resolveReleaseSummary(
"1.2.3",
"<!-- release-version: 1.2.3 -->\nTagged summary\n",
"<!-- codex-security-release-summary:start -->\nIncomplete",
),
).toBe("Tagged summary");
});
test("does not fall back when a tagged summary is invalid", () => {
expect(() =>
resolveReleaseSummary(
"1.2.3",
"<!-- release-version: 1.2.2 -->\nWrong version",
"<!-- codex-security-release-summary:start -->\nHistorical summary\n<!-- codex-security-release-summary:end -->",
),
).toThrow("Release notes must start with <!-- release-version: 1.2.3 -->");
});
test("composes reviewed and generated notes deterministically", () => {
const summary = "## Highlights\n\nReviewed summary";
const generated = [
"## What's Changed",
"",
"- A change ([#1](https://github.com/example/project/pull/1))",
"",
"## New Contributors",
"",
"- @contributor",
].join("\n");
const notes = composeReleaseNotes(generated, summary);
expect(extractHistoricalReleaseSummary(notes)).toBe(summary);
expect(notes.indexOf(summary)).toBeLessThan(notes.indexOf("<details>"));
expect(notes).toContain(`</summary>\n\n${generated}\n\n</details>`);
expect(notes.match(/<details>/gu)).toHaveLength(1);
expect(
composeReleaseNotes(generated, extractHistoricalReleaseSummary(notes)),
).toBe(notes);
expect(composeReleaseNotes("Generated notes", null)).toBe(
"Generated notes",
);
});
});
describe("stable npm release versions", () => {
test("accepts the official stable package and version", () => {
expect(
releaseVersion({
name: "@openai/codex-security",
version: "0.1.2",
}),
).toBe("0.1.2");
});
test("rejects another package", () => {
expect(() =>
releaseVersion({ name: "codex-security", version: "0.1.2" }),
).toThrow("Release package must be @openai/codex-security.");
});
test.each(["0.1.2-beta.1", "01.1.2", "0.1", "latest", ""])(
"rejects unstable or malformed version %s",
(version) => {
expect(() =>
releaseVersion({ name: "@openai/codex-security", version }),
).toThrow("Release package must have a stable X.Y.Z version.");
},
);
});
describe("protected Git release refs", () => {
const packageJson = {
name: "@openai/codex-security",
version: "0.1.2",
};
test("accepts the exact stable Git tag and package", () => {
expect(
releaseTagVersion(
"tag",
"refs/tags/npm-v0.1.2",
"npm-v0.1.2",
packageJson,
),
).toBe("0.1.2");
});
test.each([
[
"rejects a release-shaped branch",
"branch",
"refs/heads/npm-v0.1.2",
"npm-v0.1.2",
"npm releases must be dispatched from a real Git tag.",
],
[
"rejects a mismatched full Git ref",
"tag",
"refs/tags/npm-v0.1.3",
"npm-v0.1.2",
"npm releases must be dispatched from a real Git tag.",
],
[
"rejects an unstable release tag",
"tag",
"refs/tags/npm-v0.1.2-beta.1",
"npm-v0.1.2-beta.1",
"Release tags must identify a stable npm-vX.Y.Z version.",
],
[
"rejects a Git tag for a different package version",
"tag",
"refs/tags/npm-v0.1.3",
"npm-v0.1.3",
"npm release tag must match the stable package version.",
],
] as const)("%s", (_name, refType, ref, refName, message) => {
expect(() => releaseTagVersion(refType, ref, refName, packageJson)).toThrow(
message,
);
});
});
describe("monotonic stable release versions", () => {
test("compares semantic version components numerically", () => {
expect(compareReleaseVersions("0.1.10", "0.1.9")).toBe(1);
expect(compareReleaseVersions("1.0.0", "0.99.99")).toBe(1);
expect(compareReleaseVersions("0.1.2", "0.1.2")).toBe(0);
expect(compareReleaseVersions("0.1.2", "0.1.3")).toBe(-1);
});
test("compares components without unsafe JavaScript number rounding", () => {
expect(
compareReleaseVersions("9007199254740993.0.0", "9007199254740992.0.0"),
).toBe(1);
});
test("accepts a strictly increasing release", () => {
expect(requireReleaseIncrease("0.1.3", "0.1.2")).toBe("0.1.3");
});
test("rejects a downgrade or repeated release", () => {
expect(() => requireReleaseIncrease("0.1.1", "0.1.2")).toThrow(
"Release version must be greater than the previous stable version.",
);
expect(() => requireReleaseIncrease("0.1.2", "0.1.2")).toThrow(
"Release version must be greater than the previous stable version.",
);
});
test("rejects malformed and prerelease versions", () => {
expect(() => compareReleaseVersions("0.1.3-beta.1", "0.1.2")).toThrow(
"Release versions must use stable X.Y.Z versions.",
);
});
test("requires every release to exceed every published stable version", () => {
expect(
requirePublishedReleaseIncrease("0.1.11", [
"0.1.9",
"0.1.10",
"0.1.3-beta.1",
]),
).toBe("0.1.11");
});
test("identifies a strictly increasing version as a new publication", () => {
expect(publishedReleaseMode("0.1.3", ["0.1.0", "0.1.2"])).toBe("publish");
});
test("identifies the current published version as recoverable", () => {
expect(publishedReleaseMode("0.1.2", ["0.1.0", "0.1.2"])).toBe("recover");
});
test("never recovers a published version below a newer stable release", () => {
expect(() => publishedReleaseMode("0.1.2", ["0.1.2", "0.1.3"])).toThrow(
"Release version must be greater than every published stable version.",
);
});
test("ignores prerelease versions when determining recovery", () => {
expect(publishedReleaseMode("0.1.2", ["0.1.2", "0.1.3-beta.1"])).toBe(
"recover",
);
});
test("fails safely when recovery history is malformed", () => {
for (const publishedVersions of [null, {}, "0.1.2"]) {
expect(() => publishedReleaseMode("0.1.2", publishedVersions)).toThrow(
"Published npm release versions must be an array.",
);
}
});
test("rejects a manual release below the highest published version", () => {
expect(() =>
requirePublishedReleaseIncrease("1.9.9", ["1.9.8", "2.0.0"]),
).toThrow(
"Release version must be greater than every published stable version.",
);
});
test("rejects a previously published release version", () => {
expect(() =>
requirePublishedReleaseIncrease("0.1.2", ["0.1.0", "0.1.2"]),
).toThrow(
"Release version must be greater than every published stable version.",
);
});
test("fails safely when published release history is malformed", () => {
for (const publishedVersions of [null, {}, "0.1.2"]) {
expect(() =>
requirePublishedReleaseIncrease("0.1.3", publishedVersions),
).toThrow("Published npm release versions must be an array.");
}
});
});
describe("published GitHub and npm release history", () => {
test("marks the first verified GitHub release as latest", () => {
expect(
releaseHistory("npm-v0.1.2", {
registryVersions: ["0.1.0", "0.1.1", "0.1.2"],
githubReleaseTags: [],
reachableTags: ["npm-v0.1.1", "npm-v0.1.0"],
}),
).toEqual({ previousTag: "npm-v0.1.1", makeLatest: true });
});
test("keeps an already-published newest GitHub release marked latest", () => {
expect(
releaseHistory("npm-v0.1.2", {
registryVersions: ["0.1.1", "0.1.2"],
githubReleaseTags: ["npm-v0.1.1", "npm-v0.1.2"],
reachableTags: ["npm-v0.1.1", "npm-v0.1.2"],
}),
).toEqual({ previousTag: "npm-v0.1.1", makeLatest: true });
});
test("never marks a historical backfill as latest", () => {
expect(
releaseHistory("npm-v0.1.2", {
registryVersions: ["0.1.1", "0.1.2"],
githubReleaseTags: ["npm-v0.1.3"],
reachableTags: ["npm-v0.1.1"],
}),
).toEqual({ previousTag: "npm-v0.1.1", makeLatest: false });
});
test("never marks a backfill latest when a newer npm release lacks GitHub notes", () => {
expect(
releaseHistory("npm-v0.1.2", {
registryVersions: ["0.1.1", "0.1.2", "0.1.3"],
githubReleaseTags: [],
reachableTags: ["npm-v0.1.1"],
}),
).toEqual({ previousTag: "npm-v0.1.1", makeLatest: false });
});
test("starts notes from the newest actually published version", () => {
expect(
releaseHistory("npm-v0.1.4", {
registryVersions: ["0.1.1", "0.1.4"],
githubReleaseTags: ["npm-v0.1.2"],
reachableTags: ["npm-v0.1.3", "npm-v0.1.2", "npm-v0.1.1"],
}),
).toEqual({ previousTag: "npm-v0.1.2", makeLatest: true });
});
test("compares published versions numerically rather than lexically", () => {
expect(
releaseHistory("npm-v0.1.11", {
registryVersions: ["0.1.9", "0.1.10", "0.1.11"],
githubReleaseTags: ["npm-v0.1.9", "npm-v0.1.10"],
reachableTags: ["npm-v0.1.9", "npm-v0.1.10"],
}),
).toEqual({ previousTag: "npm-v0.1.10", makeLatest: true });
});
test("ignores unrelated, unstable, and unreachable release tags", () => {
expect(
releaseHistory("npm-v0.1.2", {
registryVersions: ["0.1.1", "0.1.2", "0.1.3-beta.1"],
githubReleaseTags: ["container-v99.0.0", "npm-v0.1.3-beta.1"],
reachableTags: ["container-v99.0.0", "npm-v0.1.3-beta.1", "npm-v0.1.1"],
}),
).toEqual({ previousTag: "npm-v0.1.1", makeLatest: true });
});
test("rejects invalid release history inputs", () => {
expect(() =>
releaseHistory("0.1.2", {
registryVersions: [],
githubReleaseTags: [],
reachableTags: [],
}),
).toThrow("Release tags must identify a stable npm-vX.Y.Z version.");
});
});
describe("published npm release verification", () => {
test("verifies source commit, tarball integrity, and SLSA provenance", () => {
expect(
verifyPublishedRelease(publishedMetadata(), archive, {
version: "0.1.2",
gitHead: releaseCommit,
}),
).toEqual({
version: "0.1.2",
gitHead: releaseCommit,
integrity,
sha256: digest.slice("sha256:".length),
});
});
test("supports nested public npm metadata", () => {
expect(
verifyPublishedRelease(
{
name: "@openai/codex-security",
version: "0.1.2",
gitHead: releaseCommit,
dist: {
integrity,
attestations: {
provenance: {
predicateType: "https://slsa.dev/provenance/v1",
},
},
},
},
archive,
{ version: "0.1.2", gitHead: releaseCommit },
).integrity,
).toBe(integrity);
});
test("rejects a different published version", () => {
expect(() =>
verifyPublishedRelease(publishedMetadata(), archive, {
version: "0.1.3",
gitHead: releaseCommit,
}),
).toThrow("Published npm package must match the release version.");
});
test("rejects a missing or mismatched release commit", () => {
expect(() =>
verifyPublishedRelease(
{ ...publishedMetadata(), gitHead: undefined },
archive,
{ version: "0.1.2", gitHead: releaseCommit },
),
).toThrow("npm package gitHead must match release commit");
});
test("rejects a tarball that differs from the published npm artifact", () => {
expect(() =>
verifyPublishedRelease(
publishedMetadata(),
Buffer.from("different release artifact"),
{ version: "0.1.2", gitHead: releaseCommit },
),
).toThrow(
"Published npm integrity must match the verified release artifact.",
);
});
test("rejects missing or unexpected provenance", () => {
expect(() =>
verifyPublishedRelease(
{ ...publishedMetadata(), "dist.attestations": undefined },
archive,
{ version: "0.1.2", gitHead: releaseCommit },
),
).toThrow("Published npm package must have SLSA v1 provenance.");
});
});
describe("GitHub release publication verification", () => {
const expected = {
version: "0.1.2",
gitHead: releaseCommit,
repository: releaseRepository,
runId: releaseRun,
};
const provenance = {
...expected,
runAttempt: "1",
sha512,
};
test("retains strict source and artifact verification for current releases", () => {
expect(
verifyGitHubPublishedRelease(
publishedMetadata(),
archive,
expected,
provenance,
),
).toEqual({
version: "0.1.2",
gitHead: releaseCommit,
integrity,
sha256: digest.slice("sha256:".length),
});
});
test.each(["0.1.0", "0.1.1"])(
"recovers the missing gitHead only from verified provenance for %s",
(version) => {
const metadata: ReleaseMetadata = { ...publishedMetadata(), version };
delete metadata["gitHead"];
expect(
verifyGitHubPublishedRelease(
metadata,
archive,
{ ...expected, version },
{ ...provenance, version },
),
).toEqual({
version,
gitHead: releaseCommit,
integrity,
sha256: digest.slice("sha256:".length),
});
expect(() =>
verifyPublishedRelease(metadata, archive, { ...expected, version }),
).toThrow("npm package gitHead must match release commit");
},
);
test("rejects a missing gitHead on all later npm releases", () => {
const metadata: ReleaseMetadata = { ...publishedMetadata() };
delete metadata["gitHead"];
expect(() =>
verifyGitHubPublishedRelease(metadata, archive, expected, provenance),
).toThrow("Only npm releases 0.1.0 and 0.1.1 may omit gitHead.");
});
test("rejects a mismatched historical npm gitHead", () => {
expect(() =>
verifyGitHubPublishedRelease(
{
...publishedMetadata(),
version: "0.1.0",
gitHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
},
archive,
{ ...expected, version: "0.1.0" },
{ ...provenance, version: "0.1.0" },
),
).toThrow("npm package gitHead must match release commit");
});
test.each([
{ version: "0.1.1" },
{ gitHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
{ repository: "different/codex-security" },
{ runId: "30481596228" },
{ sha512: "0".repeat(128) },
])("rejects signed provenance that does not match %j", (mismatch) => {
expect(() =>
verifyGitHubPublishedRelease(publishedMetadata(), archive, expected, {
...provenance,
...mismatch,
}),
).toThrow("Verified signed npm provenance must match the GitHub release.");
});
test("rejects missing verified provenance", () => {
expect(() =>
verifyGitHubPublishedRelease(
publishedMetadata(),
archive,
expected,
undefined as unknown as typeof provenance,
),
).toThrow("Verified signed npm provenance must match the GitHub release.");
});
});
describe("cryptographically verified npm provenance", () => {
test("binds the verified bundle to the exact archive, source, and run", () => {
expect(verifySignatureFixture()).toEqual({
version: "0.1.2",
gitHead: releaseCommit,
repository: releaseRepository,
runId: releaseRun,
runAttempt: "1",
sha512,
});
});
test("recovers a published archive using its authenticated original run", () => {
expect(
verifyRecoveredSignatureAudit(signatureAudit(), archive, {
version: "0.1.2",
gitHead: releaseCommit,
repository: releaseRepository,
}),
).toEqual({
version: "0.1.2",
gitHead: releaseCommit,
repository: releaseRepository,
runId: releaseRun,
runAttempt: "1",
sha512,
});
});
test.each(["", "0", "01", "not-a-number", "1/extra", "1?attempt=2"])(
"rejects the malformed protected workflow run attempt %j",
(attempt) => {
expect(() => verifySignatureFixture({ attempt })).toThrow(
"Verified SLSA provenance must identify the successful release run.",
);
expect(() =>
verifyRecoveredSignatureAudit(signatureAudit({ attempt }), archive, {
version: "0.1.2",
gitHead: releaseCommit,
repository: releaseRepository,
}),
).toThrow(
"Verified SLSA provenance must identify the protected release run.",
);
},
);
test("rejects a recovered archive whose certificate identifies another run", () => {
expect(() =>
verifyRecoveredSignatureAudit(
signatureAudit({ runId: "30481596228" }),
archive,
{
version: "0.1.2",
gitHead: releaseCommit,
repository: releaseRepository,
},
),
).toThrow("The Fulcio certificate must identify the exact release run.");
});
test("rejects an invocation for a different attempt of the signing run", () => {
expect(() => verifySignatureFixture({ attempt: "2" })).toThrow(
"The Fulcio certificate must identify the exact release run.",
);
});
test("returns the attempt number authenticated by the statement and certificate", () => {
expect(
verifySignatureFixture({
attempt: "3",
signingCertificate: signingCertificateWithReplacement(
"/attempts/1",
"/attempts/3",
),
}).runAttempt,
).toBe("3");
});
test("rejects a recovered archive signed for a different source commit", () => {
expect(() =>
verifyRecoveredSignatureAudit(signatureAudit(), archive, {
version: "0.1.2",
gitHead: "0000000000000000000000000000000000000000",
repository: releaseRepository,
}),
).toThrow("npm package gitHead must match release commit");
});
test("rejects a recovery without an authentic provenance bundle", () => {
expect(() =>
verifyRecoveredSignatureAudit(
signatureAudit({ includeBundle: false }),
archive,
{
version: "0.1.2",
gitHead: releaseCommit,
repository: releaseRepository,
},
),
).toThrow("The verified SLSA provenance bundle is missing.");
});
test("accepts the real Fulcio certificate in the legacy chain format", () => {
expect(verifySignatureFixture({ certificateLocation: "chain" })).toEqual({
version: "0.1.2",
gitHead: releaseCommit,
repository: releaseRepository,
runId: releaseRun,
runAttempt: "1",
sha512,
});
});
test("rejects missing and malformed Fulcio signing certificates", () => {
for (const signingCertificate of [null, "not-base64"]) {
expect(() => verifySignatureFixture({ signingCertificate })).toThrow(
"The verified Fulcio signing certificate is invalid.",
);
}
});
test("rejects empty and noncanonical DER signing certificates", () => {
const invalidCertificates = [
Buffer.from([0x30, 0x00]),
Buffer.from([0x30, 0x02, 0x30, 0x00]),
Buffer.from([0x30, 0x80, 0x00, 0x00]),
Buffer.from([0x30, 0x81, 0x00]),
Buffer.from([0x30, 0x82, 0x00, 0x01, 0x00]),
];
for (const certificate of invalidCertificates) {
expect(() =>
verifySignatureFixture({
signingCertificate: certificate.toString("base64"),
}),
).toThrow("The verified Fulcio signing certificate is invalid.");
}
});
test.each([
{
name: "rejects a certificate from another OIDC issuer",
options: () => ({
signingCertificate: signingCertificateWithReplacement(
"token.actions.githubusercontent.com",
"token.actions.githabusercontent.com",
),
}),
message:
"The Fulcio certificate must use the GitHub Actions OIDC issuer.",
},
{
name: "rejects a certificate for another GitHub Actions workflow",
options: () => ({
signingCertificate: signingCertificateWithReplacement(
"node-release.yml",
"fake-release.yml",
),
}),
message:
"The Fulcio certificate must identify the protected release workflow.",
},
{
name: "rejects a certificate for a different release commit",
options: () => ({
signingCertificate: signingCertificateWithReplacement(
releaseCommit,
"2e03c89ad22d2df5ae65b146be1483b3608572a9",
),
}),
message: "The Fulcio certificate must identify the exact release commit.",
},
{
name: "rejects a certificate for another release run",
options: () => ({
signingCertificate: signingCertificateWithReplacement(
releaseRun,
"30481596228",
),
}),
message: "The Fulcio certificate must identify the exact release run.",
},
{
name: "rejects a certificate outside the protected npm environment",
options: () => ({
signingCertificate: signingCertificateWithReplacement(
"environment:npm",
"environment:dev",
),
}),
message:
"The Fulcio certificate must identify the protected npm environment.",
},
{
name: "rejects invalid or missing registry signatures: invalid",
options: () => ({ invalid: [{ name: "@openai/codex-security" }] }),
message: "npm registry signatures and attestations must verify.",
},
{
name: "rejects invalid or missing registry signatures: missing",
options: () => ({ missing: [{ name: "@openai/codex-security" }] }),
message: "npm registry signatures and attestations must verify.",
},
{
name: "requires the exact published package in the verified audit: includeVerified",
options: () => ({ includeVerified: false }),
message:
"The published package must have a cryptographically verified attestation.",
},
{
name: "requires the exact published package in the verified audit: name",
options: () => ({ name: "@openai/codex" }),
message:
"The published package must have a cryptographically verified attestation.",
},
{
name: "requires the signed public npm registry and SLSA bundle: registry",
options: () => ({ registry: "https://registry.example/" }),
message: "Verified provenance must come from the public npm registry.",
},
{
name: "requires the signed public npm registry and SLSA bundle: includeProvenance",
options: () => ({ includeProvenance: false }),
message: "The verified npm package must have SLSA v1 provenance.",
},
{
name: "requires the signed public npm registry and SLSA bundle: includeBundle",
options: () => ({ includeBundle: false }),
message: "The verified SLSA provenance bundle is missing.",
},
{
name: "rejects provenance for a different package subject or archive: subjectName",
options: () => ({ subjectName: "pkg:npm/[email protected]" }),
message:
"Verified SLSA provenance must identify the exact published tarball.",
},
{
name: "rejects provenance for a different package subject or archive: subjectDigest",
options: () => ({ subjectDigest: "incorrect" }),
message:
"Verified SLSA provenance must identify the exact published tarball.",
},
{
name: "rejects another repository, workflow, or release tag: repository",
options: () => ({ repository: "attacker/codex-security" }),
message:
"Verified SLSA provenance must identify the protected release workflow.",
},
{
name: "rejects another repository, workflow, or release tag: workflowPath",
options: () => ({
workflowPath: ".github/workflows/untrusted-release.yml",
}),
message:
"Verified SLSA provenance must identify the protected release workflow.",
},
{
name: "rejects another repository, workflow, or release tag: workflowRef",
options: () => ({ workflowRef: "refs/heads/npm-v0.1.2" }),
message:
"Verified SLSA provenance must identify the protected release workflow.",
},
{
name: "rejects a source commit outside the protected release",
options: () => ({
sourceCommit: "0000000000000000000000000000000000000000",
}),
message: "npm package gitHead must match release commit",
},
{
name: "rejects provenance from another release run or builder: runId",
options: () => ({ runId: "12345" }),
message:
"Verified SLSA provenance must identify the successful release run.",
},
{
name: "rejects provenance from another release run or builder: builder",
options: () => ({ builder: "https://example.com/untrusted-runner" }),
message:
"Verified SLSA provenance must use a GitHub-hosted release runner.",
},
])("$name", ({ options, message }) => {
expect(() => verifySignatureFixture(options())).toThrow(message);
});
test("rejects malformed attestation bundle collections safely", () => {
const report = signatureAudit();
const [verified] = report["verified"] as ReleaseMetadata[];
for (const attestationBundles of [null, {}, "invalid", 42, [null]]) {
expect(() =>
verifySignatureAudit(
{
...report,
verified: [{ ...verified, attestationBundles }],
},
archive,
signatureExpected(),
),
).toThrow("The verified SLSA provenance bundle is missing.");
}
});
});
describe("idempotent GitHub release verification", () => {
function verifyGitHubReleaseFixture(
release: ReleaseMetadata,
downloadedArchive?: Uint8Array,
) {
return verifyGitHubRelease(
release,
archive,
"npm-v0.1.2",
"openai-codex-security-0.1.2.tgz",
downloadedArchive,
);
}
test("accepts the already-published exact verified release asset", () => {
expect(verifyGitHubReleaseFixture(githubRelease())).toEqual({
tag: "npm-v0.1.2",
asset: "openai-codex-security-0.1.2.tgz",
digest,
});
});
test("verifies downloaded release bytes when GitHub omits the asset digest", () => {
const release = githubRelease();
expect(
verifyGitHubReleaseFixture(
{
...release,
assets: [{ name: "openai-codex-security-0.1.2.tgz" }],
},
archive,
),
).toEqual({
tag: "npm-v0.1.2",
asset: "openai-codex-security-0.1.2.tgz",
digest,
});
});
test("rejects a digestless release asset without its downloaded bytes", () => {
expect(() =>
verifyGitHubReleaseFixture({
...githubRelease(),
assets: [{ name: "openai-codex-security-0.1.2.tgz" }],
}),
).toThrow(
"Existing GitHub Release asset must match the verified npm artifact.",
);
});
test("rejects downloaded release bytes that differ from the npm archive", () => {
expect(() =>
verifyGitHubReleaseFixture(
{
...githubRelease(),
assets: [{ name: "openai-codex-security-0.1.2.tgz" }],
},
Buffer.from("different downloaded GitHub release"),
),
).toThrow(
"Existing GitHub Release asset must match the verified npm artifact.",
);
});
test("rejects a GitHub release for another tag", () => {
expect(() =>
verifyGitHubReleaseFixture({
...githubRelease(),
tag_name: "npm-v0.1.1",
}),
).toThrow("Existing GitHub Release must match the release tag.");
});
test("rejects a draft or prerelease", () => {
expect(() =>
verifyGitHubReleaseFixture({ ...githubRelease(), draft: true }),
).toThrow("Existing GitHub Release must be published and stable.");
});
test("rejects a missing or different GitHub release artifact", () => {
expect(() =>
verifyGitHubReleaseFixture({
...githubRelease(),
assets: [
{
name: "openai-codex-security-0.1.2.tgz",
digest: "sha256:incorrect",
},
],
}),
).toThrow(
"Existing GitHub Release asset must match the verified npm artifact.",
);
});
test("rejects malformed GitHub release assets safely", () => {
for (const assets of [null, {}, [null]]) {
expect(() =>
verifyGitHubReleaseFixture({ ...githubRelease(), assets }),
).toThrow(
"Existing GitHub Release asset must match the verified npm artifact.",
);
}
});
test("ignores malformed assets before the exact verified release asset", () => {
const release = githubRelease();
expect(
verifyGitHubReleaseFixture({
...release,
assets: [null, ...(release["assets"] as ReleaseMetadata[])],
}),
).toEqual({
tag: "npm-v0.1.2",
asset: "openai-codex-security-0.1.2.tgz",
digest,
});
});
});
describe("GitHub release workflow safeguards", () => {
const checkedOutVersion = releaseVersion(
JSON.parse(
readFileSync(new URL("../package.json", import.meta.url), "utf8"),
) as ReleaseMetadata,
);
const checkedOutTag = `npm-v${checkedOutVersion}`;
test("stops reading inherited pipes when a command times out", async () => {
const result = await runCommand(
bash,
[
"-c",
"(sleep 1; printf 'late stdout'; printf 'late stderr' >&2) & wait",
],
{ timeout: 100 },
);
expect(result.status).toBeNull();
expect(result.stdout).not.toContain("late stdout");
expect(result.stderr).not.toContain("late stderr");
});
test("requires a real tag for protected npm publication", () => {
expect(protectedReleaseWorkflow).toContain("release-tag");
expect(protectedReleaseWorkflow).toContain('"$GITHUB_REF_TYPE"');
expect(protectedReleaseWorkflow).toContain('"$GITHUB_REF"');
});
test("requires reviewed notes only for new npm publications", () => {
expect(protectedReleaseWorkflow).toContain(
[
" - name: Validate reviewed release notes",
" if: steps.release.outputs.mode == 'publish'",
].join("\n"),
);
});
test.each([
{
scenario: "a missing reviewed summary",
summary: "__missing__",
status: 1,
message: "The tagged commit must include .github/release-notes.md.",
},
{
scenario: "a summary for another version",
summary: "<!-- release-version: 0.1.5 -->\nReviewed summary",
status: 1,
message: "Release notes must start with <!-- release-version: 0.1.6 -->",
},
{
scenario: "a U+00A0-only reviewed summary",
summary: "<!-- release-version: 0.1.6 -->\n\u00a0",
status: 1,
message: "Release notes must start with <!-- release-version: 0.1.6 -->",
},
{
scenario: "a matching reviewed summary",
summary: "<!-- release-version: 0.1.6 -->\nReviewed summary",
status: 0,
message: "",
},
])(
"validates $scenario before protected npm publication",
async ({ summary, status, message }) => {
const script = workflowStepShell(
protectedReleaseWorkflow,
"Validate reviewed release notes",
);
const mock = [
"git() {",
' if [[ "$MOCK_RELEASE_SUMMARY" == "__missing__" ]]; then return 1; fi',
" printf '%s\\n' \"$MOCK_RELEASE_SUMMARY\"",
"}",
].join("\n");
const result = await runShell(
`${mock}\n${script}`,
{
GITHUB_SHA: releaseCommit,
MOCK_RELEASE_SUMMARY: summary,
RELEASE_VERSION: "0.1.6",
},
{ cwd: fileURLToPath(new URL("../../../", import.meta.url)) },
);
expect(result.status).toBe(status);
if (message !== "") {
expect(result.stderr).toContain(message);
}
},
);
test("pins the supported-minimum release verification runtime", () => {
expect(protectedReleaseWorkflow).toContain('node-version: "22.13.0"');
});
test("restricts release cuts to main and increasing stable versions", () => {
expect(releaseCutWorkflow).toMatch(
/- name: Set up Node\.js\n(?:[^\n]*\n)*?\s+node-version: "24\.15\.0"/u,
);
expect(releaseCutWorkflow).toContain(
'if [[ "$GITHUB_REF" != "refs/heads/main" ]]; then',
);
expect(releaseCutWorkflow).toContain(
'git merge-base --is-ancestor "$GITHUB_SHA" origin/main',
);
expect(releaseCutWorkflow).toContain("require-increase");
expect(releaseCutWorkflow).toContain("require-published-increase");
expect(releaseCutWorkflow).toContain(
"npm view @openai/codex-security versions",
);
});
test.each([
{
name: "successful push on main",
eventName: "workflow_run",
sourceEvent: "push",
conclusion: "success",
headBranch: "main",
expected: true,
},
{
name: "pull request from a branch named main",
eventName: "workflow_run",
sourceEvent: "pull_request",
conclusion: "success",
headBranch: "main",
expected: false,
},
{
name: "failed push on main",
eventName: "workflow_run",
sourceEvent: "push",
conclusion: "failure",
headBranch: "main",
expected: false,
},
{
name: "successful push on another branch",
eventName: "workflow_run",
sourceEvent: "push",
conclusion: "success",
headBranch: "release",
expected: false,
},
{
name: "manual dispatch",
eventName: "workflow_dispatch",
sourceEvent: "none",
conclusion: "none",
headBranch: "none",
expected: true,
},
])("gates release cutting for $name", async (scenario) => {
const workflow = Bun.YAML.parse(releaseCutWorkflow) as {
jobs: { cut: { if: string } };
};
expect(
await evaluateWorkflowCondition(workflow.jobs.cut.if, {
"github.event.workflow_run.conclusion": scenario.conclusion,
"github.event.workflow_run.head_branch": scenario.headBranch,
"github.event.workflow_run.event": scenario.sourceEvent,
"github.event_name": scenario.eventName,
"github.repository": "openai/codex-security",
}),
).toBe(scenario.expected);
});
async function createReleaseAncestryFixture() {
const cwd = directories.create("release-ancestry-");
// A maintainer may be running with GIT_DIR, a custom template or global hooks.
const env = {
PATH: process.env["PATH"],
SYSTEMROOT: process.env["SYSTEMROOT"],
HOME: cwd,
GIT_CONFIG_NOSYSTEM: "1",
};
const setup = await runCommand(
bash,
[
"--noprofile",
"--norc",
"-c",
`
set -eu
git init --quiet --bare --template= -b main .
git init --quiet --bare --template= -b main remote.git
git config --local user.name "Release fixture"
git config --local user.email "[email protected]"
tree=$(git mktree </dev/null)
qualified=$(git commit-tree "$tree" -m qualified)
main=$(git commit-tree "$tree" -p "$qualified" -m main)
candidate=$(git commit-tree "$tree" -p "$qualified" -m off-main)
git update-ref refs/heads/main "$main"
git push --quiet ./remote.git "$main:refs/heads/main"
git remote add origin ./remote.git
printf '%s\\n%s\\n%s\\n' "$qualified" "$main" "$candidate"
`,
],
{ cwd, env },
);
expect(setup.status, setup.stderr).toBe(0);
const [qualified, main, candidate] = setup.stdout.trim().split(/\r?\n/u);
return { cwd, env, qualified, main, candidate };
}
test("only consults release metadata for a CI commit on main", async () => {
const { cwd, env, qualified, main, candidate } =
await createReleaseAncestryFixture();
const script = `
# actions/checkout detaches HEAD at the selected release SHA.
git update-ref --no-deref HEAD "$RELEASE_SHA" || exit 1
node() { echo "release metadata lookup" >&2; return 64; }
npm() { echo "unexpected registry lookup" >&2; return 65; }
${workflowStepShell(releaseCutWorkflow, "Resolve the stable package version")}
`;
for (const [workflowSha, releaseSha, expectedStatus, expectedMessage] of [
[main, candidate, 1, "Release tags can only be cut from commits on main"],
[
candidate,
qualified,
1,
"Release tags can only be cut from commits on main",
],
[main, qualified, 64, "release metadata lookup"],
] as const) {
const result = await runCommand(
bash,
["--noprofile", "--norc", "-c", script],
{
cwd,
env: {
...env,
GITHUB_REF: "refs/heads/main",
GITHUB_EVENT_NAME: "workflow_run",
GITHUB_OUTPUT: "/dev/null",
GITHUB_SHA: workflowSha,
RELEASE_SHA: releaseSha,
},
},
);
expect(result.status, result.stderr).toBe(expectedStatus);
expect(result.stderr).toContain(expectedMessage);
expect(result.stderr).not.toContain("unexpected registry lookup");
}
});
test("only builds a release tag whose exact commit is on protected main", async () => {
const { cwd, env, qualified, candidate } =
await createReleaseAncestryFixture();
const { jobs } = Bun.YAML.parse(darwinSigningWorkflow) as WorkflowContract;
const validation = jobs["validate"]!.steps!;
expect(validation[0]?.with?.["ref"]).toBe("refs/heads/main");
const validate = workflowStepShell(
darwinSigningWorkflow,
"Verify the release tag is on main",
);
for (const [tag, commit, eventSha, allowed] of [
["npm-v1", qualified, qualified, true],
["npm-v2", candidate, candidate, false],
["npm-v3", candidate, qualified, false],
] as const) {
const result = await runCommand(
bash,
[
"--noprofile",
"--norc",
"-c",
`set -eu
git tag -a "$GITHUB_REF_NAME" "$TAG_COMMIT" -m fixture
${validate}`,
],
{
cwd,
env: {
...env,
GITHUB_SHA: eventSha,
GITHUB_REF_TYPE: "tag",
GITHUB_REF_NAME: tag,
GITHUB_REF: `refs/tags/${tag}`,
GITHUB_OUTPUT: "/dev/null",
TAG_COMMIT: commit,
},
},
);
expect(result.status, result.stderr).toBe(allowed ? 0 : 1);
}
});
test("executes the manual release cut against all published versions", async () => {
const script = workflowStepShell(
releaseCutWorkflow,
"Resolve the stable package version",
);
const mocks = [
"git() {",
' if [[ "$1" == "show" && "$2" == *":.github/release-notes.md" ]]; then',
` printf '%s\\n%s\\n' '<!-- release-version: ${checkedOutVersion} -->' 'Reviewed summary'`,
" fi",
" return 0",
"}",
"npm() { printf '%s\\n' '[\"0.1.1\",\"999999999999999999999999.0.0\"]'; }",
].join("\n");
const result = await runShell(
`${mocks}\n${script}`,
{
BEFORE_SHA: "",
GITHUB_EVENT_NAME: "workflow_dispatch",
GITHUB_OUTPUT: "/dev/null",
GITHUB_REF: "refs/heads/main",
GITHUB_SHA: releaseCommit,
},
{ cwd: fileURLToPath(new URL("../../../", import.meta.url)) },
);
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"Release version must be greater than every published stable version.",
);
});
test.each([
{
scenario: "a version-increase commit with a conflicting existing tag",
previousVersion: "0.1.5",
publishedVersions: ["0.1.5"],
taggedSha: "cccccccccccccccccccccccccccccccccccccccc",
taggedVersion: "0.1.6",
changed: true,
},
{
scenario: "an unavailable blob for the previous package manifest",
previousVersion: null,
changed: false,
},
{
scenario: "a successful fix after the version-increase commit failed CI",
changed: true,
},
{
scenario:
"an unchanged unpublished version already tagged on an ancestor",
taggedSha: "cccccccccccccccccccccccccccccccccccccccc",
taggedVersion: "0.1.6",
changed: false,
},
{
scenario: "a retry at the exact tagged unpublished commit",
taggedSha: releaseCommit,
taggedVersion: "0.1.6",
changed: true,
},
{
scenario: "an unchanged version with a tag outside its ancestry",
taggedSha: "cccccccccccccccccccccccccccccccccccccccc",
taggedVersion: "0.1.6",
taggedAncestor: false,
changed: true,
},
{
scenario: "an unchanged version with a tag for another package version",
taggedSha: "cccccccccccccccccccccccccccccccccccccccc",
taggedVersion: "0.1.5",
changed: true,
},
{
scenario: "an unchanged version that has already been published",
previousVersion: "0.1.5",
currentVersion: "0.1.5",
changed: false,
},
{
scenario: "a manually dispatched unpublished version",
event: "workflow_dispatch",
previousVersion: "0.1.5",
changed: true,
},
{
scenario: "a manual dispatch after an unpublished version was tagged",
event: "workflow_dispatch",
taggedSha: "cccccccccccccccccccccccccccccccccccccccc",
taggedVersion: "0.1.6",
changed: true,
},
])(
"resolves $scenario against its published npm history",
async ({
event = "workflow_run",
previousVersion = "0.1.6",
currentVersion = "0.1.6",
publishedVersions = ["0.1.5"],
taggedSha = "",
taggedVersion = "",
taggedAncestor = true,
changed,
}) => {
const script = workflowStepShell(
releaseCutWorkflow,
"Resolve the stable package version",
);
const mocks = [
"git() {",
' case "$1" in',
" fetch) return 0 ;;",
" merge-base)",
' if [[ "$3" == "$MOCK_TAGGED_SHA" && "$4" == "$RELEASE_SHA" ]]; then',
' [[ "$MOCK_TAGGED_ANCESTOR" == "true" ]]',
" else",
" return 0",
" fi",
" ;;",
" rev-parse)",
' if [[ "$2" == "--verify" && "$3" == "$RELEASE_SHA^:sdk/typescript/package.json" ]]; then',
" echo previous-manifest",
' elif [[ "$2" == "--verify" && "$3" == "refs/tags/npm-v$MOCK_RELEASE_VERSION^{commit}" ]]; then',
' [[ -n "$MOCK_TAGGED_SHA" ]] || return 1',
" printf '%s\\n' \"$MOCK_TAGGED_SHA\"",
" else",
" return 64",
" fi",
" ;;",
" show)",
' if [[ "$2" == *":.github/release-notes.md" ]]; then',
" printf '%s\\n%s\\n' \"<!-- release-version: $MOCK_RELEASE_VERSION -->\" 'Reviewed summary'",
' elif [[ "$2" == "$MOCK_TAGGED_SHA:sdk/typescript/package.json" ]]; then',
' printf \'{"version":"%s"}\\n\' "$MOCK_TAGGED_VERSION"',
" else",
" return 64",
" fi",
" ;;",
" cat-file)",
' [[ "$2 $3" == "blob previous-manifest" ]] || return 64',
' [[ -n "$MOCK_PREVIOUS_VERSION" ]] || { echo "unavailable manifest blob" >&2; return 1; }',
' printf \'{"version":"%s"}\\n\' "$MOCK_PREVIOUS_VERSION"',
" ;;",
" *) return 64 ;;",
" esac",
"}",
"node() {",
' if [[ "${2:-}" == "version" ]]; then',
" printf '%s\\n' \"$MOCK_RELEASE_VERSION\"",
" return 0",
" fi",
' command node "$@"',
"}",
"npm() { printf '%s\\n' \"$MOCK_PUBLISHED_VERSIONS\"; }",
].join("\n");
const workspace = directories.create("codex-security-release-cut-");
const outputPath = join(workspace, "outputs");
const result = await runShell(
`${mocks}\n${script}`,
{
GITHUB_EVENT_NAME: event,
GITHUB_OUTPUT: outputPath,
GITHUB_REF: "refs/heads/main",
GITHUB_SHA: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
MOCK_PREVIOUS_VERSION: previousVersion ?? "",
MOCK_PUBLISHED_VERSIONS: JSON.stringify(publishedVersions),
MOCK_RELEASE_VERSION: currentVersion,
MOCK_TAGGED_SHA: taggedSha,
MOCK_TAGGED_VERSION: taggedVersion,
MOCK_TAGGED_ANCESTOR: String(taggedAncestor),
RELEASE_SHA: releaseCommit,
},
{ cwd: fileURLToPath(new URL("../../../", import.meta.url)) },
);
if (previousVersion === null) {
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("unavailable manifest blob");
expect(existsSync(outputPath)).toBe(false);
return;
}
expect(result.stderr).toBe("");
expect(result.status).toBe(0);
const outputs = readFileSync(outputPath, "utf8");
expect(outputs).toContain(`changed=${changed}`);
if (changed) {
expect(outputs.split("\n")).toContain(`tag=npm-v${currentVersion}`);
} else {
expect(outputs).not.toContain("tag=");
}
},
);
test.each([
{
scenario: "a missing reviewed summary",
summary: "__missing__",
message: "The tagged commit must include .github/release-notes.md.",
},
{
scenario: "a summary for another version",
summary: "<!-- release-version: 0.1.5 -->\nReviewed summary",
message: "Release notes must start with <!-- release-version: 0.1.6 -->",
},
{
scenario: "an empty reviewed summary",
summary: "<!-- release-version: 0.1.6 -->\n ",
message: "Release notes must start with <!-- release-version: 0.1.6 -->",
},
])("rejects $scenario before cutting a tag", async ({ summary, message }) => {
const script = workflowStepShell(
releaseCutWorkflow,
"Resolve the stable package version",
);
const mocks = [
"git() {",
' case "$1" in',
" fetch|merge-base) return 0 ;;",
" show)",
' if [[ "$MOCK_RELEASE_SUMMARY" == "__missing__" ]]; then return 1; fi',
" printf '%s\\n' \"$MOCK_RELEASE_SUMMARY\"",
" ;;",
" *) return 64 ;;",
" esac",
"}",
"node() {",
' if [[ "${2:-}" == "version" ]]; then printf \'%s\\n\' 0.1.6; return 0; fi',
' command node "$@"',
"}",
"npm() { printf '%s\\n' 'npm history must not be queried' >&2; return 70; }",
].join("\n");
const result = await runShell(
`${mocks}\n${script}`,
{
GITHUB_EVENT_NAME: "workflow_dispatch",
GITHUB_OUTPUT: "/dev/null",
GITHUB_REF: "refs/heads/main",
GITHUB_SHA: releaseCommit,
MOCK_RELEASE_SUMMARY: summary,
RELEASE_SHA: releaseCommit,
},
{ cwd: fileURLToPath(new URL("../../../", import.meta.url)) },
);
expect(result.status).toBe(1);
expect(result.stderr).toContain(message);
expect(result.stderr).not.toContain("npm history must not be queried");
});
test.each([
{
workflow: "release cut",
version: "0.1.0",
errorCode: "E404",
},
{
workflow: "protected publisher",
version: "0.1.0",
errorCode: "E404",
},
{
workflow: "release cut",
version: "0.1.1",
errorCode: "E404",
},
{
workflow: "protected publisher",
version: "0.1.1",
errorCode: "E404",
},
{
workflow: "release cut",
version: "0.1.0",
errorCode: "ETIMEDOUT",
},
{
workflow: "protected publisher",
version: "0.1.0",
errorCode: "E403",
},
])(
"stops the $workflow when npm history for $version fails with $errorCode",
async ({ workflow, version, errorCode }) => {
const cutting = workflow === "release cut";
const script = workflowStepShell(
cutting ? releaseCutWorkflow : protectedReleaseWorkflow,
cutting ? "Resolve the stable package version" : "Validate release tag",
);
const mocks = [
"git() {",
' if [[ "$1" == "show" && "$2" == *":.github/release-notes.md" ]]; then',
" printf '%s\\n%s\\n' \"<!-- release-version: $MOCK_RELEASE_VERSION -->\" 'Reviewed summary'",
" fi",
" return 0",
"}",
"node() {",
' if [[ "${2:-}" == "version" || "${2:-}" == "release-tag" ]]; then',
" printf '%s\\n' \"$MOCK_RELEASE_VERSION\"",
" return 0",
" fi",
' command node "$@"',
"}",
"npm() {",
" printf '%s\\n' \"$MOCK_REGISTRY_RESPONSE\"",
" return 1",
"}",
"sfw() {",
" printf '%s\\n' \"$MOCK_REGISTRY_RESPONSE\"",
" return 1",
"}",
].join("\n");
const result = await runShell(
`${mocks}\n${script}`,
{
BEFORE_SHA: "",
GITHUB_EVENT_NAME: "workflow_dispatch",
GITHUB_OUTPUT: "/dev/null",
GITHUB_REF: cutting ? "refs/heads/main" : `refs/tags/npm-v${version}`,
GITHUB_REF_NAME: `npm-v${version}`,
GITHUB_REF_TYPE: "tag",
GITHUB_SHA: releaseCommit,
MOCK_REGISTRY_RESPONSE: JSON.stringify({
error: { code: errorCode },
}),
MOCK_RELEASE_VERSION: version,
},
{ cwd: fileURLToPath(new URL("../../../", import.meta.url)) },
);
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"Unable to verify published npm release history.",
);
},
);
test("creates the release tag at the successful CI commit", async () => {
const script = workflowStepShell(
releaseCutWorkflow,
"Create the exact merged release tag",
);
const mock = [
jqMock,
"gh() {",
' if [[ "$1" != "api" ]]; then return 64; fi',
" shift",
' if [[ "$1" == "--include" ]]; then',
" printf 'HTTP/2.0 404 Mock\\nContent-Type: application/json\\n\\n'",
' printf \'%s\\n\' \'{"message":"Not Found","status":"404"}\'',
" return 1",
" fi",
' if [[ "$1" != "--method" || "$2" != "POST" ||',
' "$3" != "repos/test/codex-security/git/refs" ||',
' "$4" != "-f" || "$5" != "ref=refs/tags/npm-v0.1.2" ||',
' "$6" != "-f" || "$7" != "sha=$RELEASE_SHA" ||',
' "$8" != "--silent" ]]; then return 65; fi',
" printf 'created tag at %s\\n' \"$RELEASE_SHA\"",
"}",
].join("\n");
const result = await runShell(
`${mock}\n${script}`,
{
GITHUB_REPOSITORY: "test/codex-security",
GITHUB_SHA: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
RELEASE_SHA: releaseCommit,
RELEASE_TAG: "npm-v0.1.2",
},
{ timeout: releaseWorkflowTimeout },
);
expect(result.status).toBe(0);
expect(result.stdout).toContain(`created tag at ${releaseCommit}`);
});
test.each([
{
kind: "missing tag with GitHub's 404 error body",
lookupResponse: JSON.stringify({ message: "Not Found", status: "404" }),
lookupHttpStatus: "404",
status: 0,
},
{
kind: "forbidden tag lookup",
lookupResponse: JSON.stringify({ message: "Forbidden", status: "403" }),
lookupHttpStatus: "403",
status: 1,
},
{
kind: "malformed missing-tag response",
lookupResponse: "not JSON",
lookupHttpStatus: "404",
status: 1,
},
{
kind: "unavailable tag lookup",
lookupResponse: "",
lookupHttpStatus: "",
status: 1,
},
])(
"handles $kind safely before cutting a release",
async ({ lookupResponse, lookupHttpStatus, status }) => {
const script = workflowStepShell(
releaseCutWorkflow,
"Create the exact merged release tag",
);
const mock = [
jqMock,
"gh() {",
' if [[ "$1" != "api" ]]; then return 64; fi',
" shift",
' if [[ "$1" == "--include" &&',
' "$2" == "repos/test/codex-security/git/ref/tags/npm-v0.1.2" ]]; then',
' if [[ -n "$MOCK_LOOKUP_HTTP_STATUS" ]]; then',
" printf 'HTTP/2.0 %s Mock\\nContent-Type: application/json\\n\\n%s\\n' \\",
' "$MOCK_LOOKUP_HTTP_STATUS" "$MOCK_LOOKUP_RESPONSE"',
" fi",
" return 1",
" fi",
' if [[ "$1" != "--method" || "$2" != "POST" ||',
' "$3" != "repos/test/codex-security/git/refs" ||',
' "$4" != "-f" || "$5" != "ref=refs/tags/npm-v0.1.2" ||',
' "$6" != "-f" || "$7" != "sha=$GITHUB_SHA" ||',
' "$8" != "--silent" ]]; then return 65; fi',
" printf 'created exact release tag\\n'",
"}",
].join("\n");
const result = await runShell(
`${mock}\n${script}`,
{
GITHUB_REPOSITORY: "test/codex-security",
GITHUB_SHA: releaseCommit,
MOCK_LOOKUP_HTTP_STATUS: lookupHttpStatus,
MOCK_LOOKUP_RESPONSE: lookupResponse,
RELEASE_TAG: "npm-v0.1.2",
},
{ timeout: releaseWorkflowTimeout },
);
expect(result.status).toBe(status);
if (status === 0) {
expect(result.stdout).toContain("created exact release tag");
} else {
expect(result.stderr).toContain(
"Unable to query release tag npm-v0.1.2.",
);
expect(result.stdout).not.toContain("created exact release tag");
}
},
);
test.each([
{
kind: "existing lightweight tag",
tagType: "commit",
tagObject: releaseCommit,
peeledCommit: "",
status: 0,
},
{
kind: "existing annotated tag",
tagType: "tag",
tagObject: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
peeledCommit: releaseCommit,
status: 0,
},
{
kind: "retargeted annotated tag",
tagType: "tag",
tagObject: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
peeledCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
status: 1,
},
])(
"resolves an $kind to its exact commit before cutting a release",
async ({ tagType, tagObject, peeledCommit, status }) => {
const script = workflowStepShell(
releaseCutWorkflow,
"Create the exact merged release tag",
);
const mock = [
jqMock,
"gh() {",
' if [[ "$1" != "api" ]]; then return 64; fi',
" shift",
' if [[ "$1" == "--include" ]]; then',
" shift",
' if [[ "$1" != "repos/test/codex-security/git/ref/tags/npm-v0.1.2" ]]; then',
" return 65",
" fi",
" printf 'HTTP/2.0 200 OK\\nContent-Type: application/json\\n\\n'",
' printf \'{"object":{"type":"%s","sha":"%s"}}\\n\' \\',
' "$MOCK_TAG_TYPE" "$MOCK_TAG_OBJECT"',
" return 0",
" fi",
' case "$1" in',
" repos/test/codex-security/git/tags/*)",
" printf '%s\\n' \"$MOCK_PEELED_COMMIT\"",
" ;;",
" *) return 65 ;;",
" esac",
"}",
].join("\n");
const result = await runShell(
`${mock}\n${script}`,
{
GITHUB_REPOSITORY: "test/codex-security",
GITHUB_SHA: releaseCommit,
MOCK_PEELED_COMMIT: peeledCommit,
MOCK_TAG_OBJECT: tagObject,
MOCK_TAG_TYPE: tagType,
RELEASE_TAG: "npm-v0.1.2",
},
{ timeout: releaseWorkflowTimeout },
);
expect(result.status).toBe(status);
if (status === 0) {
expect(result.stdout).toContain(
`Release tag npm-v0.1.2 already points to ${releaseCommit}.`,
);
} else {
expect(result.stderr).toContain(
"Release tag npm-v0.1.2 already points to a different commit.",
);
}
},
);
test("enforces increasing versions inside the protected npm publisher", () => {
expect(protectedReleaseWorkflow).toContain(
"sfw npm view @openai/codex-security versions",
);
expect(protectedReleaseWorkflow).toContain("release-mode");
});
test.each([
{
kind: "deleted release tag",
tagType: "missing",
tagObject: "",
status: 1,
},
{
kind: "retargeted lightweight tag",
tagType: "commit",
tagObject: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
status: 1,
},
{
kind: "retargeted annotated tag",
tagType: "tag",
tagObject: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
peeledCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
status: 1,
},
{
kind: "verified lightweight tag",
tagType: "commit",
tagObject: releaseCommit,
status: 0,
},
{
kind: "verified annotated tag",
tagType: "tag",
tagObject: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
peeledCommit: releaseCommit,
status: 0,
},
])(
"revalidates the authoritative $kind before signing and publication",
async ({ tagType, tagObject, peeledCommit = "", status }) => {
const action = Bun.YAML.parse(
readFileSync(
new URL(
"../../../.github/actions/revalidate-release-tag/action.yml",
import.meta.url,
),
"utf8",
),
) as { runs: { steps: { run: string }[] } };
const script = action.runs.steps[0]!.run;
const mock = [
"gh() {",
' if [[ "$1" != "api" ]]; then return 64; fi',
" shift",
' case "$1" in',
` "repos/test/codex-security/git/ref/tags/${checkedOutTag}")`,
' if [[ "$MOCK_TAG_TYPE" == "missing" ]]; then return 1; fi',
' printf \'%s\\t%s\\n\' "$MOCK_TAG_TYPE" "$MOCK_TAG_OBJECT"',
" ;;",
" repos/test/codex-security/git/tags/*)",
" printf '%s\\n' \"$MOCK_PEELED_COMMIT\"",
" ;;",
" *) return 65 ;;",
" esac",
"}",
].join("\n");
const result = await runShell(`${mock}\n${script}`, {
GITHUB_REF: `refs/tags/${checkedOutTag}`,
GITHUB_REF_NAME: checkedOutTag,
GITHUB_REF_TYPE: "tag",
GITHUB_REPOSITORY: "test/codex-security",
GITHUB_SHA: releaseCommit,
MOCK_PEELED_COMMIT: peeledCommit,
MOCK_TAG_OBJECT: tagObject,
MOCK_TAG_TYPE: tagType,
});
expect(result.status).toBe(status);
if (status === 0) {
expect(result.stdout).toContain(
"Protected npm release tag matches the verified commit.",
);
} else {
expect(result.stderr).toContain(
"Protected npm release tag must still point to the verified commit.",
);
}
},
);
test("binds signing to both verified Darwin builds and packaging to the signed artifact", () => {
const { jobs } = Bun.YAML.parse(
protectedReleaseWorkflow,
) as WorkflowContract;
const signing = Bun.YAML.parse(darwinSigningWorkflow) as WorkflowContract;
expect(jobs["sign-darwin"]?.needs).toBe("eligibility");
expect(jobs["sign-darwin"]?.if).toBe(
"needs.eligibility.outputs.mode == 'publish'",
);
expect(jobs["recover-native"]).toMatchObject({
needs: "eligibility",
if: "needs.eligibility.outputs.mode == 'recover'",
uses: "./.github/workflows/native-artifacts.yml",
});
expect(jobs["sign-darwin"]?.uses).toBe(
"openai/codex-security/.github/workflows/sign-darwin-release.yml@main",
);
const sign = signing.jobs["sign"]!;
expect(signing.jobs["build"]).toMatchObject({
needs: "validate",
uses: "./.github/workflows/native-artifacts.yml",
"cache-mode": "none",
with: { ref: "${{ github.sha }}", "reuse-verified": false },
});
expect(sign.needs).toBe("build");
expect(sign.environment).toBe("codesigning");
const signer = sign.steps!;
const usedSecrets = [
...JSON.stringify(signer).matchAll(/\$\{\{ secrets\.(\w+) \}\}/gu),
].map(([reference, name]) => [name!, reference] as const);
expect(jobs["sign-darwin"]?.secrets).toEqual(
Object.fromEntries(usedSecrets),
);
for (const [name] of usedSecrets) {
expect(signing.on?.workflow_call?.secrets?.[name]?.required).toBe(true);
}
const requireIds = signer.find(
(step) => step.name === "Require both build artifact IDs",
)!;
const digestCheck = signer.find(
(step) => step.name === "Verify both addon digests",
);
for (const arch of ["arm64", "x64"]) {
const download = signer.find(
(step) => step.name === `Download the ${arch} build`,
);
const artifactId = `\${{ needs.build.outputs.darwin-${arch}-artifact-id }}`;
expect(requireIds.env?.[arch.toUpperCase() + "_ID"]).toBe(artifactId);
expect(signer.indexOf(requireIds)).toBeLessThan(
signer.indexOf(download!),
);
expect(download?.with).toMatchObject({
"artifact-ids": artifactId,
"digest-mismatch": "error",
path: `${arch}-build`,
});
expect(digestCheck?.env?.[arch.toUpperCase() + "_SHA256"]).toBe(
`\${{ needs.build.outputs.darwin-${arch}-sha256 }}`,
);
}
const verify = jobs["verify"]!.steps!;
expect(
verify.find((step) => step.name === "Download verified native payload")
?.with,
).toMatchObject({
"artifact-ids": "${{ needs.native.outputs.artifact-id }}",
"digest-mismatch": "error",
});
expect(
verify.find(
(step) => step.name === "Overlay signed Darwin addons for publication",
)?.with,
).toMatchObject({
"artifact-ids": "${{ needs.native.outputs.signed-darwin-id }}",
"digest-mismatch": "error",
});
const setup = signer.findIndex(
(step) => step.name === "Set up Developer ID signing",
);
expect(signer[setup]?.with?.["setup-mode"]).toBe("tools-only");
expect(signer[setup - 1]).toMatchObject({
name: "Revalidate tag before Developer ID signing",
uses: "$/.github/actions/revalidate-release-tag",
});
const publisher = jobs["publish"]!.steps!;
const publishIndex = publisher.findIndex(
(step) => step.name === "Publish to npm using trusted publishing",
);
expect(publisher[publishIndex - 1]).toMatchObject({
name: "Revalidate protected release tag",
if: "needs.verify.outputs.mode == 'publish'",
uses: "$/.github/actions/revalidate-release-tag",
});
});
test("publication and recovery select only their own successful native build", async () => {
const { jobs } = Bun.YAML.parse(
protectedReleaseWorkflow,
) as WorkflowContract;
const native = jobs["native"]!;
const select = native.steps!.find(
(step) =>
step.name === "Select the native artifact IDs for this release mode",
)!;
expect(select.env).toMatchObject({
PUBLISH_NATIVE_ID: "${{ needs.sign-darwin.outputs.native-artifact-id }}",
SIGNED_ID: "${{ needs.sign-darwin.outputs.artifact-id }}",
RECOVER_NATIVE_ID: "${{ needs.recover-native.outputs.artifact-id }}",
});
for (const mode of ["publish", "recover"]) {
const publish = mode === "publish";
const context = {
"!cancelled()": "true",
"needs.eligibility.result": "success",
"needs.eligibility.outputs.mode": mode,
"needs.sign-darwin.result": publish ? "success" : "skipped",
"needs.recover-native.result": publish ? "skipped" : "success",
};
expect(await evaluateWorkflowCondition(native.if!, context)).toBe(true);
const requiredJob = publish ? "sign-darwin" : "recover-native";
expect(
await evaluateWorkflowCondition(native.if!, {
...context,
[`needs.${requiredJob}.result`]: "failure",
}),
).toBe(false);
const output = join(
directories.create("release-native-"),
"outputs",
).replaceAll("\\", "/");
const result = await runShell(select.run!, {
RELEASE_MODE: mode,
PUBLISH_NATIVE_ID: "101",
SIGNED_ID: "102",
RECOVER_NATIVE_ID: "201",
GITHUB_OUTPUT: output,
});
expect(result.status, result.stderr).toBe(0);
expect(readFileSync(output, "utf8").trim().split(/\r?\n/u)).toEqual(
publish ? ["native-id=101", "signed-id=102"] : ["native-id=201"],
);
}
});
test("requires the protected signing workflow before signing and consuming addons from this attempt", async () => {
const { jobs } = Bun.YAML.parse(
protectedReleaseWorkflow,
) as WorkflowContract;
const steps = jobs["native"]!.steps!;
const select = steps.findIndex(
(step) =>
step.name === "Select the native artifact IDs for this release mode",
);
const guard = steps[select - 1]!;
expect(guard.name).toBe("Verify reusable signing workflow ref");
expect(guard.if).toBe("needs.eligibility.outputs.mode == 'publish'");
const eligibility = jobs["eligibility"]!;
const early = eligibility.steps!.find(
(step) =>
step.name === "Verify reusable signing workflow ref before signing",
);
expect(early).toMatchObject({
if: "steps.release.outputs.mode == 'publish'",
run: guard.run,
});
expect(eligibility.permissions).toEqual({
actions: "read",
contents: "read",
});
expect(jobs["sign-darwin"]?.needs).toBe("eligibility");
expect(jobs["verify"]?.needs).toContain("native");
expect(
jobs["verify"]?.steps?.find(
(step) => step.name === "Overlay signed Darwin addons for publication",
)?.if,
).toBe(guard.if);
const workflow = `${releaseRepository}/.github/workflows/sign-darwin-release.yml`;
const branch = {
path: `${workflow}@main`,
ref: "refs/heads/main",
sha: releaseCommit,
};
const tag = { ...branch, ref: "refs/tags/main" };
const other = {
...branch,
path: `${releaseRepository}/.github/workflows/other.yml@main`,
};
const script = `set -e
gh() {
[[ "$*" == "api repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT" ]] || return 64
printf '%s\\n' "$MOCK_ATTEMPT"
}
${guard.run}
echo 'overlay reached'`;
for (const [refs, allowed] of [
[[other, branch], true],
[[{ ...branch, path: `${workflow}@${releaseCommit}` }], true],
[[tag], false],
[[other, branch, tag], false],
[[{ ...branch, ref: "refs/heads/feature" }], false],
[[{ path: branch.path, sha: releaseCommit }], false],
[[other], false],
[[], false],
] as const) {
const result = await runShell(script, {
GITHUB_REPOSITORY: releaseRepository,
GITHUB_RUN_ID: releaseRun,
GITHUB_RUN_ATTEMPT: "3",
MOCK_ATTEMPT: JSON.stringify({ referenced_workflows: refs }),
});
expect(result.status, JSON.stringify(refs) + result.stderr).toBe(
allowed ? 0 : 1,
);
expect(result.stdout.includes("overlay reached")).toBe(allowed);
}
});
test("each verified Darwin matrix leg forwards its own upload ID and addon digest", async () => {
const unix = workflow("native-unix");
const native = workflow("native-artifacts");
const build = unix.jobs["primitives"]!;
const steps = build.steps!;
const upload = steps.findIndex((step) => step.id === "upload");
const record = steps.findIndex((step) => step.id === "darwin");
const node20 = steps.findIndex(
(step) => step.with?.["node-version"] === "20.0.0",
);
expect(node20).toBeGreaterThan(0);
expect(
steps
.slice(node20 + 1, upload)
.some(
(step) =>
step.run?.includes("check.mjs") && step.run.includes("proof.mjs"),
),
).toBe(true);
expect(record).toBeGreaterThan(upload);
expect(steps[record]).toMatchObject({
if: "matrix.platform == 'darwin'",
env: {
DARWIN_TARGET: "${{ matrix.artifact }}",
ARTIFACT_ID: "${{ steps.upload.outputs.artifact-id }}",
},
});
for (const [target, id] of [
["darwin-arm64", "101"],
["darwin-x64", "202"],
] as const) {
const cwd = directories.create("native-matrix-");
const output = join(cwd, "outputs").replaceAll("\\", "/");
writeSource(cwd, `dist/${target}/unix.node`, target);
const script = `if ! command -v shasum >/dev/null; then
shasum() { [[ "$1 $2" == "-a 256" ]] || return 64; shift 2; sha256sum "$@"; }
fi
${steps[record]!.run}`;
const result = await runShell(
script,
{ DARWIN_TARGET: target, ARTIFACT_ID: id, GITHUB_OUTPUT: output },
{ cwd },
);
expect(result.status, result.stderr).toBe(0);
expect(readFileSync(output, "utf8").trim().split(/\r?\n/u)).toEqual([
`${target}-artifact-id=${id}`,
`${target}-sha256=${createHash("sha256").update(target).digest("hex")}`,
]);
for (const key of [`${target}-artifact-id`, `${target}-sha256`]) {
expect(build.outputs?.[key]).toBe(
`\${{ steps.darwin.outputs.${key} }}`,
);
expect(unix.on["workflow_call"]).toMatchObject({
outputs: {
[key]: { value: `\${{ jobs.primitives.outputs.${key} }}` },
},
});
expect(native.on["workflow_call"]).toMatchObject({
outputs: { [key]: { value: `\${{ jobs.unix.outputs.${key} }}` } },
});
}
}
});
test.each([
{ scenario: "both verified builds" },
{ scenario: "substituted arm64 addon", replaced: "darwin-arm64" },
{ scenario: "substituted x64 addon", replaced: "darwin-x64" },
{ scenario: "missing arm64 artifact ID", missingId: "ARM64_ID" },
{ scenario: "missing x64 artifact ID", missingId: "X64_ID" },
])(
"gates the Developer ID signer on $scenario",
async ({ replaced, missingId }) => {
const { jobs } = Bun.YAML.parse(
darwinSigningWorkflow,
) as WorkflowContract;
const steps = jobs["sign"]!.steps!;
const step = steps.find((step) => step.name === "Sign Darwin addons")!;
expect(step.env).toMatchObject({
RCODESIGN: "${{ steps.signing-tools.outputs.rcodesign }}",
PKCS11_LIBRARY: "${{ steps.signing-tools.outputs.pkcs11-library }}",
AKV_PKCS11_BACKEND: "signer-service",
SIGNER_SERVICE_ENDPOINT: "${{ secrets.SIGNER_SERVICE_ENDPOINT }}",
SIGNER_SERVICE_PURPOSE: "${{ secrets.SIGNER_SERVICE_PURPOSE }}",
SIGNER_SERVICE_KEY_ID: "${{ secrets.SIGNER_SERVICE_KEY_ID }}",
SIGNER_SERVICE_KEY_LABEL: expect.stringMatching(/\S/),
SIGNING_CERTIFICATE_PEM: "${{ secrets.SIGNING_CERTIFICATE_PEM }}",
});
const cwd = directories.create("release signer ");
if (process.platform === "darwin" && !Bun.which("sha256sum")) {
writeFileSync(
join(cwd, "sha256sum"),
'#!/bin/sh\nexec shasum -a 256 "$@"\n',
{ mode: 0o755 },
);
}
const architectures = ["arm64", "x64"];
for (const arch of architectures) {
const target = `darwin-${arch}`;
writeSource(
cwd,
`${arch}-build/${target}/unix.node`,
target === replaced ? "substituted artifact" : target,
);
}
const env = {
...step.env,
PATH: `${cwd}${delimiter}${process.env["PATH"]}`,
ARM64_ID: missingId === "ARM64_ID" ? "" : "11",
X64_ID: missingId === "X64_ID" ? "" : "12",
ARM64_SHA256: createHash("sha256").update("darwin-arm64").digest("hex"),
X64_SHA256: createHash("sha256").update("darwin-x64").digest("hex"),
RCODESIGN: "./fixture signer",
PKCS11_LIBRARY: "fixture provider.so",
SIGNER_SERVICE_CERTIFICATE_FILE: "fixture certificate.pem",
SIGNING_CERTIFICATE_PEM:
"-----BEGIN CERTIFICATE-----\nfixture\n-----END CERTIFICATE-----",
};
writeFileSync(
join(cwd, env.RCODESIGN),
'#!/usr/bin/env bash\nprintf "<%s>" "$@" >> "${!#}"\n',
{ mode: 0o755 },
);
const script = steps
.filter((s) =>
[
"Require both build artifact IDs",
"Verify both addon digests",
"Sign Darwin addons",
].includes(s.name!),
)
.map((s) => `(\n${s.run!}\n)`)
.join("\n");
const result = await runShell(`set -e\n${script}`, env, { cwd });
if (replaced || missingId) {
expect(result.status, result.stderr).toBe(1);
expect(existsSync(join(cwd, env.SIGNER_SERVICE_CERTIFICATE_FILE))).toBe(
false,
);
return;
}
expect(result.status, result.stderr).toBe(0);
const pem = readFileSync(
join(cwd, env.SIGNER_SERVICE_CERTIFICATE_FILE),
"utf8",
);
expect(pem.trimEnd()).toBe(env.SIGNING_CERTIFICATE_PEM);
for (const arch of architectures) {
const target = `darwin-${arch}`;
const signed = readFileSync(
join(cwd, "signed", target, "unix.node"),
"utf8",
);
expect(signed).toStartWith(`${target}<sign>`);
expect(signed).toContain(`<--pkcs11-library><${env.PKCS11_LIBRARY}>`);
expect(signed).toContain(
`<--pkcs11-certificate-file><${env.SIGNER_SERVICE_CERTIFICATE_FILE}>`,
);
expect(signed).toContain(
`<--pkcs11-key-label><${step.env!["SIGNER_SERVICE_KEY_LABEL"]}>`,
);
}
},
);
test("rejects manually publishing a tag older than npm latest", async () => {
const script = workflowStepShell(
protectedReleaseWorkflow,
"Validate release tag",
);
const mocks = [
"git() { return 0; }",
"sfw() { printf '%s\\n' '[\"0.1.1\",\"999999999999999999999999.0.0\"]'; }",
].join("\n");
const result = await runShell(
`${mocks}\n${script}`,
{
GITHUB_OUTPUT: "/dev/null",
GITHUB_REF: `refs/tags/${checkedOutTag}`,
GITHUB_REF_NAME: checkedOutTag,
GITHUB_REF_TYPE: "tag",
GITHUB_SHA: releaseCommit,
},
{ cwd: fileURLToPath(new URL("../../../", import.meta.url)) },
);
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"Release version must be greater than every published stable version.",
);
});
test("allows the exact already-published release to enter verified recovery", async () => {
const script = workflowStepShell(
protectedReleaseWorkflow,
"Validate release tag",
);
const mocks = [
"git() { return 0; }",
`sfw() { printf '%s\\n' '["0.1.0","${checkedOutVersion}"]'; }`,
].join("\n");
const result = await runShell(
`${mocks}\n${script}`,
{
GITHUB_OUTPUT: "/dev/null",
GITHUB_REF: `refs/tags/${checkedOutTag}`,
GITHUB_REF_NAME: checkedOutTag,
GITHUB_REF_TYPE: "tag",
GITHUB_SHA: releaseCommit,
},
{ cwd: fileURLToPath(new URL("../../../", import.meta.url)) },
);
expect(result.status).toBe(0);
expect(result.stdout).toContain(
"Recovering the already-published npm release.",
);
});
test("verifies recovered packages without republishing immutable versions", () => {
expect(protectedReleaseWorkflow).toContain(
"Recover the published npm release archive",
);
expect(protectedReleaseWorkflow).toContain(
"Verify recovered npm release provenance",
);
expect(protectedReleaseWorkflow).toContain("verify-recovered-provenance");
expect(protectedReleaseWorkflow).toContain(
"needs.verify.outputs.mode == 'publish'",
);
});
test("durably queues every release-cut and protected publishing run", () => {
expect(releaseCutWorkflow).toMatch(
/concurrency:\s*\n\s+group: node-release-cut\s*\n\s+queue: max/u,
);
expect(protectedReleaseWorkflow).toMatch(
/concurrency:\s*\n\s+group: \$\{\{ github\.workflow \}\}\s*\n\s+queue: max/u,
);
});
test("requires every release test shard before protected publication", () => {
const { jobs } = Bun.YAML.parse(
protectedReleaseWorkflow,
) as WorkflowContract;
expect(jobs["test"]?.strategy?.matrix["suite"]).toEqual([1, 2, 3, "mcp"]);
expect(jobs["test"]?.strategy?.["fail-fast"]).toBe(false);
expect(jobs["test"]?.permissions).toEqual({ contents: "read" });
expect(jobs["test"]?.env).toEqual(jobs["verify"]?.env);
expect(jobs["publish"]?.needs).toEqual([
"verify",
"test",
"macos-review-items",
]);
expect(
jobs["macos-review-items"]?.steps?.find(
(step) => step.name === "Download verified package",
)?.with?.["artifact-ids"],
).toBe("${{ needs.verify.outputs.artifact-id }}");
expect(jobs["publish"]?.name).toBe(
`\${{ needs.verify.outputs.mode == 'recover' && '${recoveryPublisher}' || '${combinedPublisher}' }}`,
);
expect(jobs["publish"]?.environment).toBe("npm");
expect(jobs["publish"]?.permissions).toEqual({
contents: "read",
"id-token": "write",
});
expect(
jobs["publish"]?.steps?.find(
(step) => step.name === "Download verified package",
)?.with,
).toMatchObject({
"artifact-ids": "${{ needs.verify.outputs.artifact-id }}",
"digest-mismatch": "error",
});
expect(
jobs["test"]?.steps?.find((step) => step.name === "Test TypeScript shard")
?.run,
).toContain("scripts/run-ci-tests.mts ${{ matrix.suite }}/3");
expect(
jobs["test"]?.steps?.find((step) => step.name === "Test MCP app")?.run,
).toContain("run test:mcp");
});
test("isolates optional release comments from verification and publishing", () => {
const { jobs } = Bun.YAML.parse(
protectedReleaseWorkflow,
) as WorkflowContract;
const progress = Bun.YAML.parse(
readFileSync(
new URL(
"../../../.github/workflows/release-progress.yml",
import.meta.url,
),
"utf8",
),
) as WorkflowContract;
expect(jobs["verify"]?.permissions).toEqual({ contents: "read" });
expect(jobs["native"]?.permissions).toEqual({ actions: "read" });
expect(progress.jobs["report"]?.["continue-on-error"]).toBe(true);
expect(progress.jobs["report"]?.permissions).toEqual({
actions: "read",
contents: "read",
"pull-requests": "write",
});
const checkout = progress.jobs["report"]?.steps?.find(
(step: { uses?: string }) => step.uses?.startsWith("actions/checkout@"),
);
expect(checkout?.with).toMatchObject({
ref: "refs/heads/main",
"persist-credentials": false,
});
expect(progress.concurrency?.["cancel-in-progress"]).toBe(false);
expect(progress.jobs["report"]?.concurrency?.["queue"]).toBe("max");
expect(progress.jobs["report"]?.needs).toBe("identify");
expect(progress.jobs["watch"]?.permissions).toEqual({
actions: "write",
contents: "read",
});
expect(progress.jobs["watch"]?.["continue-on-error"]).toBe(true);
expect(progress.jobs["watch"]?.if).toContain(
"github.event.action == 'in_progress'",
);
expect(progress.jobs["watch"]?.if).toContain("inputs.watch");
expect(progress.jobs["watch"]?.steps?.[0]?.with).toEqual({
ref: "refs/heads/main",
"persist-credentials": false,
});
expect(
progress.jobs["watch"]?.steps?.at(-1)?.env?.["RELEASE_PROGRESS_WATCH"],
).toBe("true");
const publication = (
Bun.YAML.parse(githubReleaseWorkflow) as WorkflowContract
).jobs;
expect(publication["installed"]?.needs).toBe("release");
});
test("dispatches GitHub releases after publishing with isolated permissions", () => {
expect(protectedReleaseWorkflow).toContain(
[
" dispatch-github-release:",
" if: ${{ !cancelled() && needs.publish.result == 'success' }}",
" name: dispatch GitHub release",
" needs: publish",
" runs-on: ubuntu-latest",
" permissions:",
" actions: write",
].join("\n"),
);
expect(protectedReleaseWorkflow).toMatch(
/ publish:\n[\s\S]*? permissions:\n contents: read\n id-token: write\n/u,
);
expect(githubReleaseWorkflow).not.toContain("workflow_run:");
expect(githubReleaseWorkflow).not.toContain("github.event.workflow_run");
expect(githubReleaseWorkflow).not.toContain("TRIGGER_TAG");
expect(githubReleaseWorkflow).not.toContain("TRIGGER_RUN_ID");
});
test.each([
{
description: "a version already visible in the registry",
missingAttempts: 0,
calls: 1,
waits: 0,
status: 0,
},
{
description:
"a version becoming visible after twenty minutes of npm processing",
missingAttempts: 40,
calls: 41,
waits: 40,
status: 0,
},
{
description: "a version remaining absent past the publication wait",
missingAttempts: 100,
calls: 61,
waits: 60,
status: 1,
},
{
description: "an unrelated registry authorization failure",
missingAttempts: 1,
errorCode: "E401",
calls: 1,
waits: 0,
status: 1,
},
])(
"handles $description before GitHub release verification",
async ({ missingAttempts, errorCode = "E404", calls, waits, status }) => {
const script = workflowStepShell(
githubReleaseWorkflow,
"Wait for the published npm package",
);
const workspace = directories.create(
"codex-security-release-visibility-",
);
const callLog = join(workspace, "calls.log");
writeFileSync(callLog, "");
const mocks = [
"npm() {",
' if [[ "$1" != "view" || "$2" != "@openai/[email protected]" ]]; then return 64; fi',
' printf "query\\n" >> "$MOCK_CALL_LOG"',
" if (( attempt <= MOCK_MISSING_ATTEMPTS )); then",
' printf \'{"error":{"code":"%s","summary":"synthetic registry diagnostic"}}\\n\' "$MOCK_ERROR_CODE"',
" return 1",
" fi",
" printf '\"0.1.2\"\\n'",
"}",
"sleep() {",
' if [[ "$1" != "30" ]]; then return 65; fi',
' printf "wait\\n" >> "$MOCK_CALL_LOG"',
"}",
].join("\n");
const result = await runShell(
`${mocks}\n${script}`,
{
MOCK_CALL_LOG: callLog,
MOCK_MISSING_ATTEMPTS: String(missingAttempts),
MOCK_ERROR_CODE: errorCode,
RELEASE_VERSION: "0.1.2",
},
{ timeout: releaseWorkflowTimeout },
);
const events = readFileSync(callLog, "utf8").split("\n");
expect(result.status).toBe(status);
expect(events.filter((event) => event === "query")).toHaveLength(calls);
expect(events.filter((event) => event === "wait")).toHaveLength(waits);
if (missingAttempts > 0) {
expect(result.stderr).toContain("synthetic registry diagnostic");
expect(result.stderr).toContain(errorCode);
}
},
);
test.each([
{
workflow: readFileSync(
new URL(
"../../../.github/workflows/container-release.yml",
import.meta.url,
),
"utf8",
),
step: "Dispatch the coordinator on protected main",
target: "node-github-release.yml",
fields: ["-f", "ref=main", "-f", "inputs[tag]=npm-v0.1.2"],
},
{
workflow: releaseCutWorkflow,
step: "Dispatch the protected npm release",
target: "node-release.yml",
fields: ["-f", "ref=npm-v0.1.2"],
},
{
workflow: protectedReleaseWorkflow,
step: "Dispatch the verified GitHub release",
target: "node-github-release.yml",
fields: [
"-f",
"ref=main",
"-f",
"inputs[tag]=npm-v0.1.2",
"-f",
`inputs[run_id]=${releaseRun}`,
],
},
])(
"dispatches $target with exact release inputs and records its run",
async ({ workflow, step, target, fields }) => {
const script = workflowStepShell(workflow, step);
const output = join(directories.create("release-dispatch-"), "output");
writeFileSync(output, "");
const mock = "gh() { printf '%s\\n' \"$@\" >&2; printf '12345\\n'; }";
const env = {
GITHUB_REPOSITORY: releaseRepository,
GITHUB_SERVER_URL: "https://github.com",
GITHUB_STEP_SUMMARY: join(
directories.create("release-summary-"),
"summary",
),
VERSION: "0.1.2",
RELEASE_RUN_ID: releaseRun,
RELEASE_TAG: "npm-v0.1.2",
GITHUB_OUTPUT: output,
};
const result = await runShell(`${mock}\n${script}`, env);
expect(result.status).toBe(0);
expect(result.stderr.trim().split("\n")).toEqual([
"api",
"--method",
"POST",
"-H",
"X-GitHub-Api-Version: 2026-03-10",
`repos/${releaseRepository}/actions/workflows/${target}/dispatches`,
...fields,
"--jq",
".workflow_run_id",
]);
expect(readFileSync(output, "utf8")).toBe("run-id=12345\n");
writeFileSync(output, "");
const failed = await runShell(`gh() { return 1; }\n${script}`, env);
expect(failed.status).toBe(1);
expect(readFileSync(output, "utf8")).toBe("");
const parsed = Bun.YAML.parse(workflow) as WorkflowContract;
const notification = Object.values(parsed.jobs)
.flatMap((job) => job.steps ?? [])
.find((step) => step.name === "Start release progress observer");
expect(notification?.["continue-on-error"]).toBe(true);
expect(notification?.if).toBe("steps.dispatch.outputs.run-id != ''");
expect(notification?.run).toContain(
"timeout --kill-after=5s 10s gh workflow run release-progress.yml",
);
expect(notification?.run).toContain(
'-f "run-id=$RELEASE_RUN_ID" -f watch=true',
);
},
);
test("serializes every GitHub release and historical backfill", () => {
expect(githubReleaseWorkflow).toMatch(
/concurrency:\s*\n\s+group: node-github-release\s*\n\s+queue: max/u,
);
expect(githubReleaseWorkflow).not.toContain(
"group: node-github-release-${{",
);
});
test("runs manually dispatched GitHub releases from the main commit selected for the run", () => {
expect(githubReleaseWorkflow).toContain("workflow_dispatch:");
expect(githubReleaseWorkflow).toContain("github.ref == 'refs/heads/main'");
expect(githubReleaseWorkflow).toMatch(
/- name: Checkout release automation\n(?:[^\n]*\n)*?\s+ref: \$\{\{ github\.sha \}\}/u,
);
});
test.each(["queued", "in_progress"])(
"waits for a %s protected npm release to finish before publishing notes",
async (pendingStatus) => {
const root = directories.create(
"codex-security-release-wait-$(echo literal)-'-",
);
const sleeps = join(root, "release-sleeps");
const result = await resolveProtectedRelease({
MOCK_RUN_STATUS: pendingStatus,
MOCK_SLEEP_LOG: sleeps,
});
expect(result.status).toBe(0);
expect(readFileSync(sleeps, "utf8")).toBe("2\n");
},
);
test("rejects a pending npm release for a different tagged commit", async () => {
const result = await resolveProtectedRelease({
MOCK_RUN_STATUS: "in_progress",
MOCK_RUN_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
});
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"The release run must successfully publish the exact tagged commit.",
);
});
test("rejects a different workflow presenting the release tag and commit", async () => {
const result = await resolveProtectedRelease({
MOCK_WORKFLOW_PATH: ".github/workflows/other.yml",
});
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"The release run must successfully publish the exact tagged commit.",
);
});
test.each([
["push", 0],
["workflow_dispatch", 0],
["pull_request", 1],
] as const)(
"checks the %s event for the selected npm run",
async (event, status) => {
const result = await resolveProtectedRelease({ MOCK_RUN_EVENT: event });
expect(result.status).toBe(status);
},
);
test("times out safely if a protected npm release never completes", async () => {
const sleeps = join(directories.create("release-timeout-"), "sleeps");
const result = await resolveProtectedRelease({
MOCK_RUN_STATUS: "in_progress",
MOCK_NEXT_STATUS: "in_progress",
MOCK_SLEEP_LOG: sleeps,
});
expect(readFileSync(sleeps, "utf8")).toBe("2\n".repeat(29));
expect(result.status).toBe(1);
expect(result.stderr).toContain(
`Timed out waiting for protected npm release ${releaseRun} to complete.`,
);
});
test("rejects a release-shaped branch before resolving its commit", async () => {
const calls = join(directories.create("missing-tag-"), "gh-calls");
const result = await resolveProtectedRelease({
MOCK_TAG_TYPE: "",
MOCK_GH_LOG: calls,
});
expect(readFileSync(calls, "utf8")).not.toContain("/commits/");
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"GitHub releases require an existing stable npm-vX.Y.Z tag.",
);
});
test.each([
{ kind: "lightweight", tagType: "commit", objectSha: releaseCommit },
{
kind: "annotated with no locally fetched tag object",
tagType: "tag",
objectSha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
},
])(
"resolves the exact $kind tag when a same-named branch exists",
async ({ tagType, objectSha }) => {
const result = await resolveProtectedRelease({
MOCK_TAG_TYPE: tagType,
MOCK_TAG_OBJECT: objectSha,
MOCK_BRANCH_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
});
expect(result.status).toBe(0);
expect(result.stderr).not.toContain(
"The release run must successfully publish the exact tagged commit.",
);
},
);
test.each(["0", "01", "000123"])(
"rejects the noncanonical protected GitHub release run ID %j",
async (runId) => {
const result = await resolveProtectedRelease({ INPUT_RUN_ID: runId });
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"No successful protected npm release exists for npm-v0.1.2.",
);
},
);
test.each([
{
description: "verified lightweight release tag",
status: 0,
},
{
description: "verified annotated release tag",
tagType: "tag",
tagObject: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
peeledCommit: releaseCommit,
status: 0,
},
{
description: "deleted release tag",
tagType: "missing",
tagObject: "",
status: 1,
},
{
description: "retargeted lightweight release tag",
tagObject: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
status: 1,
},
{
description: "retargeted annotated release tag",
tagType: "tag",
tagObject: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
peeledCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
status: 1,
},
{
description: "release lookup returning an unexpected HTTP 500",
existingLookup: "unavailable",
status: 1,
},
{
description: "release lookup rejecting its authentication",
existingLookup: "unauthorized",
status: 1,
},
{
description: "release lookup without an HTTP response",
existingLookup: "network",
status: 1,
},
{
description: "release lookup returning a malformed HTTP 404",
existingLookup: "malformed",
status: 1,
},
])(
"revalidates the $description immediately before creating its GitHub release",
async ({
existingLookup = "missing",
tagType = "commit",
tagObject = releaseCommit,
peeledCommit = "",
status,
}) => {
const result = runPublisher({
existingLookup,
tagType,
tagObject,
peeledCommit,
});
expect(result.status).toBe(status);
expect(
result.calls.some(
(call) => call[1] === "release" && call[2] === "create",
),
).toBe(status === 0);
if (status === 0) {
expect(result.stdout).toContain("created verified GitHub release");
} else {
expect(result.stdout).not.toContain("created verified GitHub release");
expect(result.stderr).toContain(
existingLookup === "missing"
? "GitHub release tag must still point to the verified commit."
: "Unable to resolve the existing GitHub Release.",
);
}
},
);
test.each([0, 17, 143])(
"runs publication and cleans up after exit %p",
async (exit) => {
const workspace = mkdtempSync(join(tmpdir(), "publish boundary "));
const preload = join(workspace, "commands.mjs");
const receipt = join(workspace, "notes-path");
writeFileSync(
preload,
`
import childProcess from "node:child_process";
import { syncBuiltinESMExports } from "node:module";
import { readFileSync, writeFileSync } from "node:fs";
childProcess.spawnSync = (program, args, options) => {
if (program === "npm") return { status: 0, stdout: '["0.1.2","0.1.3"]' };
if (program === "git") return { status: 1, stdout: "" };
if (program !== "gh") throw new Error("Unexpected command: " + program);
if (args.includes("repos/test/codex-security/releases?per_page=100")) return { status: 0, stdout: "" };
if (args.includes("repos/test/codex-security/releases/tags/npm-v0.1.2")) {
return { status: 1, stdout: 'HTTP/2.0 404 Not Found\\n\\n{"message":"Not Found"}' };
}
if (args.includes("repos/test/codex-security/releases/generate-notes")) {
return { status: 0, stdout: '{"body":"Generated notes"}' };
}
if (args.includes("repos/test/codex-security/git/ref/tags/npm-v0.1.2")) {
return { status: 0, stdout: "commit\\t${releaseCommit}\\n" };
}
if (args[0] !== "release" || args[1] !== "create" || options.stdio[1] !== "inherit") {
throw new Error("Unexpected command: " + args);
}
const notes = args[args.indexOf("--notes-file") + 1];
if (readFileSync(notes, "utf8") !== "Generated notes") throw new Error("Wrong notes");
writeFileSync(${JSON.stringify(receipt)}, notes);
process.stdout.write("release output\\n");
return { status: ${exit === 143 ? "null" : exit}, signal: ${JSON.stringify(exit === 143 ? "SIGTERM" : null)} };
};
syncBuiltinESMExports();
`,
);
try {
const result = await runShell(
workflowStepShell(
githubReleaseWorkflow,
"Publish GitHub Release and generated notes",
),
{
NODE_OPTIONS: `--import=${pathToFileURL(preload).href}`,
GH_TOKEN: "synthetic-unused-token",
GITHUB_REPOSITORY: "test/codex-security",
RELEASE_TAG: "npm-v0.1.2",
RELEASE_VERSION: "0.1.2",
RELEASE_SHA: releaseCommit,
RELEASE_ARCHIVE: join(workspace, "verified archive.tgz"),
},
{ cwd: fileURLToPath(new URL("../../../", import.meta.url)) },
);
expect(result.status).toBe(exit);
expect(result.stdout).toContain("release output");
expect(result.stderr).toBe("");
expect(existsSync(readFileSync(receipt, "utf8"))).toBe(false);
} finally {
rmSync(workspace, { recursive: true, force: true });
}
},
);
test("explicitly prevents historical releases becoming latest", () => {
const result = runPublisher({ existingLookup: "missing" });
expect(result.status).toBe(0);
expect(result.calls.at(-1)).toContain("--latest=false");
});
test.each(["", "not-json"])(
"preserves note-generation failure before parsing %j output",
(stdout) => {
const result = runPublisher({
existingLookup: "missing",
generateNotesResult: {
status: 17,
stdout,
stderr: "Release notes are unavailable.",
},
});
expect(result.status).toBe(17);
expect(result.stderr).toBe("Release notes are unavailable.\n");
expect(result.calls.at(-1)).toContain(
"repos/test/codex-security/releases/generate-notes",
);
},
);
test("generates notes only from a previously published release", () => {
const result = runPublisher({ existingLookup: "missing" });
expect(
result.calls.find((call) =>
call.includes("repos/test/codex-security/releases/generate-notes"),
),
).toContain("previous_tag_name=npm-v0.1.1");
});
test("recovers a public npm tarball after release artifacts expire", () => {
expect(githubReleaseWorkflow).toContain("if ! gh run download");
expect(githubReleaseWorkflow).toContain(
'npm pack "@openai/codex-security@$RELEASE_VERSION"',
);
expect(githubReleaseWorkflow).toContain("--ignore-scripts");
});
test("downloads and verifies existing GitHub release asset bytes", () => {
const result = runPublisher({
downloadedArchive: Buffer.from("wrong archive"),
});
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"Existing GitHub Release asset must match the verified npm artifact.",
);
expect(result.calls.map((call) => call.slice(0, 3))).toEqual([
["npm", "view", "@openai/codex-security"],
["gh", "api", "--paginate"],
["git", "rev-parse", `${releaseCommit}^`],
["git", "tag", "--merged"],
["gh", "api", "--include"],
["gh", "release", "download"],
]);
});
test.each([
{
description:
"a successful signing attempt carried into a failed-jobs-only rerun",
exact: true,
resolvedPublisher: combinedPublisher,
originalCombined: true,
combined: true,
recoveryConclusion: "skipped",
status: 0,
message: "Verified npm provenance for the resolved release run.",
},
{
description:
"a failed signing attempt followed by a successful recovery in the same run",
exact: true,
resolvedPublisher: recoveryPublisher,
originalCombined: true,
originalConclusion: "failure",
recoveryConclusion: "success",
status: 0,
message: "Verified npm provenance for the resolved release run.",
},
{
description: "the historical npm-only publisher",
exact: true,
recoveryConclusion: "skipped",
status: 0,
message: "Verified npm provenance for the resolved release run.",
},
{
description: "a different signing run without authenticated recovery",
recoveryConclusion: "skipped",
status: 1,
message:
"Signed npm provenance must identify the resolved successful release run.",
},
{
description: "a combined signing run recovered by an npm-only job",
originalCombined: true,
latestOriginalConclusion: "failure",
combined: true,
recoveryConclusion: "success",
status: 0,
message:
"Verified protected recovery provenance from its original signing run.",
},
{
description:
"an npm-only signing run verified by the current recovery job",
resolvedPublisher: recoveryPublisher,
recoveryConclusion: "success",
status: 0,
message:
"Verified protected recovery provenance from its original signing run.",
},
{
description: "a recovery run impersonating the combined publisher",
resolvedPublisher: combinedPublisher,
recoveryConclusion: "success",
status: 1,
message:
"Signed npm provenance must identify the resolved successful release run.",
},
{
description: "an exact historical signing run whose jobs are unavailable",
exact: true,
jobsUnavailable: true,
recoveryConclusion: "skipped",
status: 0,
message: "Verified npm provenance for the resolved release run.",
},
{
description: "a package signed before its combined publisher failed",
originalCombined: true,
originalConclusion: "failure",
recoveryConclusion: "success",
status: 0,
message:
"Verified protected recovery provenance from its original signing run.",
},
{
description: "a recovery provenance run for a different source commit",
recoveryConclusion: "success",
originalCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
status: 1,
message:
"Recovered npm provenance must identify the exact protected release.",
},
])(
"binds GitHub release provenance to $description",
async ({
combined = false,
exact = false,
jobsUnavailable = false,
resolvedPublisher = "publish",
originalCombined = false,
originalConclusion = "success",
latestOriginalConclusion = originalConclusion,
recoveryConclusion,
originalCommit = releaseCommit,
status,
message,
}) => {
const script = workflowStepShell(
githubReleaseWorkflow,
"Verify the public npm package and signed provenance",
);
const workspace = directories.create(
"codex-security-release-provenance-",
);
mkdirSync(join(workspace, "dist"));
writeFileSync(join(workspace, "dist", "release.tgz"), "verified archive");
const signingRun = exact ? releaseRun : "30481596228";
function jobsPayload(name: string, conclusion: string, recovery: string) {
return JSON.stringify({
jobs: [
{
name,
conclusion,
steps: [
{
name: "Verify recovered npm release provenance",
conclusion: recovery,
},
],
},
],
});
}
const mocks = [
// Native Windows jq otherwise writes CRLF to the workflow's Bash substitutions.
process.platform === "win32"
? 'jq() { command jq --binary "$@"; }'
: "",
"npm() {",
' case "$1" in',
" view) printf '%s\\n' '{}' ;;",
" install) return 0 ;;",
" audit) printf '%s\\n' '{\"verified\":[]}' ;;",
" *) return 65 ;;",
" esac",
"}",
"node() {",
' if [[ "${1:-}" == "sdk/typescript/scripts/release-automation.mjs" ]]; then',
" cat >/dev/null",
' case "$2" in',
" verify-github-publication)",
' if [[ -z "${CODEX_SECURITY_VERIFIED_PROVENANCE:-}" ||',
' "$6" != "$GITHUB_REPOSITORY" ]]; then return 69; fi',
" printf '%s\\n' 'verified published artifact'",
" ;;",
" verify-provenance)",
' if [[ "$MOCK_EXACT_PROVENANCE" != "1" || "$7" != "$RELEASE_RUN_ID" ]]; then',
" return 68",
" fi",
' printf \'{"runId":"%s","runAttempt":"3"}\\n\' "$RELEASE_RUN_ID"',
" ;;",
" verify-recovered-provenance)",
' printf \'{"runId":"%s","runAttempt":"3"}\\n\' "$MOCK_ORIGINAL_RUN_ID"',
" ;;",
" *) return 66 ;;",
" esac",
" return 0",
" fi",
' command node "$@"',
"}",
"gh() {",
' if [[ "$1" != "api" ]]; then return 64; fi',
" shift",
' case "$1" in',
` "repos/test/codex-security/actions/runs/${releaseRun}/jobs?per_page=100")`,
' jq -r "${!#}" <<< "$MOCK_RESOLVED_JOBS"',
" ;;",
' "repos/test/codex-security/actions/runs/30481596228/jobs?per_page=100")',
' jq -r "${!#}" <<< "$MOCK_ORIGINAL_LATEST_JOBS"',
" ;;",
` "repos/test/codex-security/actions/runs/${signingRun}/attempts/3/jobs?per_page=100")`,
' if [[ "$MOCK_JOBS_UNAVAILABLE" == true ]]; then return 68; fi',
' jq -r "${!#}" <<< "$MOCK_SIGNING_JOBS"',
" ;;",
' "repos/test/codex-security/actions/runs/30481596228")',
" printf '%s\\t%s\\t%s\\t%s\\n' .github/workflows/node-release.yml completed \"$MOCK_ORIGINAL_COMMIT\" npm-v0.1.2",
" ;;",
" *) return 67 ;;",
" esac",
"}",
].join("\n");
const result = await runShell(
`${mocks}\n${script}`,
{
GITHUB_OUTPUT: "outputs",
GITHUB_REPOSITORY: "test/codex-security",
MOCK_EXACT_PROVENANCE: exact ? "1" : "0",
MOCK_ORIGINAL_COMMIT: originalCommit,
MOCK_ORIGINAL_RUN_ID: "30481596228",
MOCK_JOBS_UNAVAILABLE: String(jobsUnavailable),
MOCK_RESOLVED_JOBS: jobsPayload(
resolvedPublisher,
"success",
recoveryConclusion,
),
MOCK_ORIGINAL_LATEST_JOBS: jobsPayload(
originalCombined ? combinedPublisher : "publish",
latestOriginalConclusion,
"skipped",
),
MOCK_SIGNING_JOBS: jobsPayload(
originalCombined ? combinedPublisher : "publish",
originalConclusion,
"skipped",
),
RELEASE_RUN_ID: releaseRun,
RELEASE_SHA: releaseCommit,
RELEASE_TAG: "npm-v0.1.2",
RELEASE_VERSION: "0.1.2",
},
{ cwd: workspace },
);
expect(result.status).toBe(status);
if (status === 0) {
expect(result.stdout).toContain(message);
expect(
readFileSync(join(workspace, "outputs"), "utf8").includes(
"combined-approval=true",
),
).toBe(combined);
} else {
expect(result.stderr).toContain(message);
}
},
);
test.each([
{
description: "empty",
existingNotes: "",
updated: true,
latestUpdated: false,
status: 0,
},
{
description: "manually written",
existingNotes: "Manually written release notes",
updated: true,
latestUpdated: false,
status: 0,
},
{
description: "stale generated notes with a marked historical summary",
existingNotes:
"<!-- codex-security-release-summary:start -->\n## Highlights\nPreserved historical summary\n<!-- codex-security-release-summary:end -->\n\nStale generated notes",
expectedSummary: "Preserved historical summary",
updated: true,
latestUpdated: false,
status: 0,
},
{
description: "CRLF historical summary marker block",
existingNotes:
"<!-- codex-security-release-summary:start -->\r\nPreserved historical summary\r\n<!-- codex-security-release-summary:end -->\r\n\r\nStale generated notes",
expectedSummary: "Preserved historical summary",
updated: true,
latestUpdated: false,
status: 0,
},
{
description: "CRLF marker-only body with a terminal line ending",
existingNotes:
"<!-- codex-security-release-summary:start -->\r\nPreserved historical summary\r\n<!-- codex-security-release-summary:end -->\r\n",
expectedSummary: "Preserved historical summary",
updated: true,
latestUpdated: false,
status: 0,
},
{
description: "inline marker-like substrings in generated notes",
existingNotes:
"* fix: <!-- codex-security-release-summary:start -->Unreviewed injected highlight<!-- codex-security-release-summary:end --> by @contributor",
excludedSummary: "Unreviewed injected highlight",
updated: true,
latestUpdated: false,
status: 0,
},
{
description: "standalone marker pair outside the canonical prefix",
existingNotes:
"* fix: generated title\n<!-- codex-security-release-summary:start -->\nUnreviewed injected highlight\n<!-- codex-security-release-summary:end -->",
excludedSummary: "Unreviewed injected highlight",
updated: true,
latestUpdated: false,
status: 0,
},
{
description: "duplicate standalone marker pair after a blank line",
existingNotes:
"<!-- codex-security-release-summary:start -->\nFirst\n<!-- codex-security-release-summary:end -->\n\n<!-- codex-security-release-summary:start -->\nSecond\n<!-- codex-security-release-summary:end -->",
expectedError: "Existing release summary markers are malformed.",
updated: false,
latestUpdated: false,
status: 1,
},
{
description: "NUL-bearing historical summary",
existingNotes:
"<!-- codex-security-release-summary:start -->\nReviewed\u0000summary\n<!-- codex-security-release-summary:end -->",
expectedError: "Existing release summary is empty.",
updated: false,
latestUpdated: false,
status: 1,
},
{
description: "NUL-bearing tagged summary",
releaseSummary: "<!-- release-version: 0.1.2 -->\nReviewed\0summary",
expectedError: "include a reviewed summary",
updated: false,
latestUpdated: false,
status: 1,
},
{
description: "generated notes without its tagged reviewed summary",
releaseSummary:
"<!-- release-version: 0.1.2 -->\n## Highlights\nReviewed tagged summary",
expectedSummary: "Reviewed tagged summary",
updated: true,
latestUpdated: false,
status: 0,
},
{
description: "malformed historical summary markers",
existingNotes:
"<!-- codex-security-release-summary:start -->\nIncomplete summary",
expectedError: "Existing release summary markers are malformed.",
updated: false,
latestUpdated: false,
status: 1,
},
{
description: "already current",
updated: false,
latestUpdated: false,
status: 0,
},
{
description: "already current with trailing line feeds",
existingNotes: "Generated release notes\n\n",
updated: false,
latestUpdated: false,
status: 0,
},
{
description: "already current with CRLF-equivalent generated notes",
existingNotes: "Generated release notes\r\n",
generatedNotes: "Generated release notes\r\n",
updated: false,
latestUpdated: false,
status: 0,
},
{
description: "already current with NUL-bearing generated notes",
existingNotes: "Generated release notes\u0000",
generatedNotes: "Generated release notes\0",
updated: false,
latestUpdated: false,
status: 0,
},
{
description: "different strings encoding to the same UTF-8 bytes",
existingNotes: "\ud800",
generatedNotes: "\ufffd",
updated: false,
latestUpdated: false,
status: 0,
},
{
description: "empty generated notes",
generatedNotes: "",
expectedError: "Generated GitHub release notes must not be empty.",
updated: false,
latestUpdated: false,
status: 1,
},
{
description:
"already current on a release predating release-note configuration",
updated: false,
latestTag: "npm-v0.1.2",
makeLatest: true,
latestUpdated: false,
releaseConfiguration: false,
status: 0,
},
{
description:
"already current on a newest release incorrectly marked non-Latest",
updated: false,
latestTag: "npm-v0.1.1",
makeLatest: true,
latestUpdated: true,
status: 0,
},
{
description: "already current when no GitHub release is marked Latest",
updated: false,
latestTag: "__missing__",
makeLatest: true,
latestUpdated: true,
status: 0,
},
{
description: "already current when the Latest lookup fails unexpectedly",
updated: false,
latestTag: "__error__",
makeLatest: true,
latestUpdated: false,
status: 1,
},
{
description:
"already current when the Latest lookup has no HTTP response",
updated: false,
latestTag: "__network__",
makeLatest: true,
latestUpdated: false,
status: 1,
},
{
description: "already current when the Latest 404 response is malformed",
updated: false,
latestTag: "__malformed_missing__",
makeLatest: true,
latestUpdated: false,
status: 1,
},
{
description:
"already current on a historical release incorrectly marked Latest",
updated: false,
latestTag: "npm-v0.1.2",
latestUpdated: true,
tagType: "tag",
tagObject: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
peeledCommit: releaseCommit,
status: 0,
},
{
description: "stale when its lightweight release tag was retargeted",
existingNotes: "Manually written release notes",
updated: false,
latestUpdated: false,
tagObject: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
status: 1,
},
{
description: "stale when its annotated release tag was retargeted",
existingNotes: "Manually written release notes",
updated: false,
latestUpdated: false,
tagType: "tag",
tagObject: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
peeledCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
status: 1,
},
])(
"reconciles $description notes on an existing verified GitHub release",
async ({
existingNotes = "Generated release notes",
generatedNotes = "Generated release notes\n",
updated,
latestTag = "npm-v0.1.3",
makeLatest = false,
latestUpdated,
releaseConfiguration = true,
releaseSummary = "__missing__",
expectedSummary,
excludedSummary,
expectedError,
tagType = "commit",
tagObject = releaseCommit,
peeledCommit = "",
status,
}) => {
const result = runPublisher({
existingNotes,
generatedNotes,
latestTag,
makeLatest,
releaseConfiguration,
releaseSummary,
tagType,
tagObject,
peeledCommit,
});
expect(
result.calls.some(
(call) => call[1] === "release" && call[2] === "edit",
),
).toBe(updated || latestUpdated);
if (updated || latestUpdated) {
expect(result.calls.at(-2)).toContain(
tagType === "tag"
? `repos/test/codex-security/git/tags/${tagObject}`
: "repos/test/codex-security/git/ref/tags/npm-v0.1.2",
);
}
expect(result.status).toBe(status);
expect(result.stdout).toContain(`"digest":"${digest}"`);
if (status !== 0) {
if (expectedError !== undefined) {
expect(result.stderr).toContain(expectedError);
return;
}
const latestLookupFailed = [
"__error__",
"__network__",
"__malformed_missing__",
].includes(latestTag);
expect(result.stderr).toContain(
latestLookupFailed
? "Unable to resolve the current Latest GitHub Release."
: "GitHub release tag must still point to the verified commit.",
);
return;
}
if (updated) {
if (excludedSummary !== undefined) {
expect(result.stdout).not.toContain(excludedSummary);
}
if (expectedSummary !== undefined) {
expect(result.stdout).toContain(expectedSummary);
expect(result.stdout).toContain("Generated release notes");
} else {
expect(result.stdout).toContain(
"updated release notes: Generated release notes",
);
}
expect(result.stdout).toContain(
"Updated existing GitHub Release with reviewed and generated notes.",
);
} else if (latestUpdated) {
expect(result.stdout).not.toContain("updated release notes:");
expect(result.stdout).toContain(`updated latest: ${makeLatest}`);
expect(result.stdout).toContain(
"Updated existing GitHub Release Latest status.",
);
} else {
expect(result.stdout).not.toContain("updated release notes:");
expect(result.stdout).toContain(
"The verified GitHub Release already exists.",
);
}
},
);
test("routes release-note validation through the shared helper", () => {
expect(protectedReleaseWorkflow).toContain("validate-release-notes");
expect(releaseCutWorkflow).toContain("validate-release-notes");
expect(githubReleaseWorkflow).toContain("publishGitHubRelease");
expect(protectedReleaseWorkflow).not.toContain("summary_header=");
expect(releaseCutWorkflow).not.toContain("summary_header=");
expect(githubReleaseWorkflow).not.toContain("CODEX_SECURITY_SUMMARY_START");
});
test("cryptographically verifies the exact npm provenance bundle", () => {
expect(githubReleaseWorkflow).toContain("npm audit signatures");
expect(githubReleaseWorkflow).toContain("--include-attestations");
expect(githubReleaseWorkflow).toContain("verify-recovered-provenance");
});
test("both release jobs require the npm installed through the Socket Firewall", async () => {
const action = "./.github/actions/setup-release-npm";
for (const [workflow, job] of [
[protectedReleaseWorkflow, "publish"],
[githubReleaseWorkflow, "release"],
] as const) {
const { jobs } = Bun.YAML.parse(workflow) as WorkflowContract;
expect(jobs[job]?.steps?.map((step) => step.uses)).toContain(action);
}
const manifest = new URL(`../../../${action}/action.yml`, import.meta.url);
const { runs } = Bun.YAML.parse(readFileSync(manifest, "utf8")) as {
runs: { steps: NonNullable<WorkflowContract["jobs"][string]["steps"]> };
};
const [node, install, verifyPath] = runs.steps;
expect(node?.uses).toMatch(/^actions\/setup-node@[a-f0-9]{40}$/);
expect(node?.with).toEqual({
"node-version": "24.15.0",
"registry-url": "https://registry.npmjs.org",
"package-manager-cache": false,
});
const result = await runShell(
`RUNNER_TEMP="$(cd "$RUNNER_TEMP" && pwd)"
GITHUB_PATH="$RUNNER_TEMP/path"
npm() {
printf '<%s>' "$1" "$2" "\${3#"$RUNNER_TEMP/"}" "\${@:4}"
mkdir -p "$3/node_modules/.bin"
printf '#!/bin/sh\\necho "\${TEST_NPM_VERSION:-11.12.1}"\\n' > "$3/node_modules/.bin/npm"
chmod +x "$3/node_modules/.bin/npm"
}
${install!.run}
export PATH="$(cat "$GITHUB_PATH"):$PATH"
verify() { "$BASH" <<'VERIFY'
${verifyPath!.run}
VERIFY
}
verify && echo selected
TEST_NPM_VERSION=11.10.1 verify && exit 2
cp "$release_npm_bin/npm" "$RUNNER_TEMP/npm"
PATH="$RUNNER_TEMP:$PATH" verify`,
{ RUNNER_TEMP: directories.create("release npm ") },
{ cwd: fileURLToPath(new URL("../../../", import.meta.url)) },
);
expect(result.stdout).toBe(
"<install><--prefix><codex-security-provenance-npm><--ignore-scripts><--no-audit><--no-fund><--registry=https://openai.firewall.socket.dev/npm/><[email protected]>selected\n",
);
expect(result.status, result.stderr).toBe(1);
});
test("serializes label reconciliation per pull request", () => {
expect(Bun.YAML.parse(releaseLabelsWorkflow)).toMatchObject({
concurrency: {
group: "node-release-labels-${{ github.event.pull_request.number }}",
},
});
});
test("enforces the same Conventional Commit title in required CI", () => {
const releasePattern = /conventional_title='([^']+)'/u.exec(
releaseLabelsWorkflow,
)?.[1];
const ciPattern = /conventional_title='([^']+)'/u.exec(nodeCiWorkflow)?.[1];
const titlePattern = /conventional_title='([^']+)'/u.exec(
titleWorkflow,
)?.[1];
expect(releasePattern).toBeDefined();
expect(ciPattern).toBeUndefined();
expect(titlePattern).toBe(releasePattern);
});
test("requires every selected source validation in the centralized gate", async () => {
const workflow = Bun.YAML.parse(nodeCiWorkflow) as {
concurrency: {
group: string;
"cancel-in-progress": string;
};
jobs: Record<
string,
{
name: string;
if?: string;
needs?: string | string[];
uses?: string;
outputs?: Record<string, string>;
"timeout-minutes"?: number | string;
steps: Array<{
if?: string;
name?: string;
run?: string;
env?: Record<string, string>;
with?: Record<string, unknown>;
}>;
}
>;
};
const sourceGroup =
"${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}";
expect(workflow.concurrency.group).toStartWith(sourceGroup);
expect(workflow.concurrency["cancel-in-progress"]).toBe(
"${{ github.event_name == 'pull_request' }}",
);
const classifier = workflow.jobs["classify-changes"]!;
expect(classifier["timeout-minutes"]).toEqual(
expect.stringContaining("40 || 10"),
);
expect(classifier.if).toBeUndefined();
const checkout = classifier.steps.find(
({ name }) => name === "Checkout repository for change classification",
);
expect(checkout?.with).toMatchObject({
"fetch-depth": 2,
"persist-credentials": false,
});
expect(checkout).not.toHaveProperty("if");
expect(workflow.jobs["static-checks"]?.if).toBe(
"needs.classify-changes.outputs.source-run == ''",
);
const gate = workflow.jobs["ci"]!;
const checks = Object.entries(workflow.jobs).filter(
([name]) => name !== "classify-changes" && name !== "ci",
);
expect(new Set([gate.needs].flat())).toEqual(
new Set(["classify-changes", ...checks.map(([name]) => name)]),
);
expect(gate.if).toContain("always()");
expect(workflow.jobs["action-ci"]).toMatchObject({
uses: "./.github/workflows/action-ci.yml",
with: {
"sdk-contracts":
"${{ needs.classify-changes.outputs.source-required == 'false' }}",
},
});
const sdkShards = ["test", "windows-test"];
for (const [name, job] of checks) {
if (sdkShards.includes(name))
expect(job.needs).toContain("static-checks");
else
expect(job.if ?? "", name).not.toMatch(
/\b(always|failure|cancelled)\(/u,
);
}
// With no job results, the gate must report exactly the jobs enabled in YAML.
const { failedChecks } = (await import(
new URL("../../../.github/scripts/check-ci-results.mjs", import.meta.url)
.href
)) as { failedChecks(needs: unknown): string[] };
const classifierOutputs = classifier.outputs!;
expect(classifierOutputs["source-run"]).toBe(
"${{ steps.proof.outputs.source-run }}",
);
const flags = Object.keys(classifierOutputs).filter(
(key) =>
!["source-run", "edit-attempt"].includes(key) &&
!key.startsWith("native-cache-") &&
!key.endsWith("-matrix"),
);
for (const enabled of flags) {
expect(classifierOutputs[enabled]).toBe(
`\${{ steps.scope.outputs.${enabled} }}`,
);
const outputs = Object.fromEntries(
flags.map((key) => [key, String(key === enabled)]),
);
const scheduled = checks.filter(
([name, job]) =>
name === "static-checks" ||
!job.if ||
(sdkShards.includes(name) && enabled === "source-required") ||
job.if === `needs.classify-changes.outputs.${enabled} == 'true'`,
);
expect(
new Set(
failedChecks({ "classify-changes": { result: "success", outputs } }),
),
enabled,
).toEqual(new Set(scheduled.map(([name]) => name)));
}
expect(
gate.steps.find(({ name }) => name === "Require selected checks"),
).toMatchObject({
run: "node .github/scripts/check-ci-results.mjs",
env: { CI_RESULTS: "${{ toJSON(needs) }}" },
});
});
test.each([
"fix: generated title\n<!-- codex-security-release-summary:start -->\nUnreviewed injected highlight\n<!-- codex-security-release-summary:end -->",
"fix: preserve a trailing line feed\n",
"fix: preserve a trailing carriage return\r",
])("active title gates reject %s", (title) => {
for (const [workflow, step] of [
[titleWorkflow, "Check conventional title"],
] as const) {
expect(
spawnSync(bash, ["-c", workflowStepShell(workflow, step)], {
env: { ...process.env, PR_TITLE: title },
}).status,
).toBe(1);
}
});
test.each([
"security: preserve an existing title type",
"deps(api)!: upgrade a dependency",
"dependency-update2: refresh release tooling",
])("active title gates accept previously valid title %s", (title) => {
for (const [workflow, step] of [
[titleWorkflow, "Check conventional title"],
] as const) {
expect(
spawnSync(bash, ["-c", workflowStepShell(workflow, step)], {
env: { ...process.env, PR_TITLE: title },
}).status,
).toBe(0);
}
});
test.each([
"[codex] Add a scan feature",
"Feat: use an uppercase type",
"feat(): use an empty scope",
"feat(bad scope): use whitespace in a scope",
"feat:no separator space",
"feat: use two separator spaces",
"feat: leave trailing whitespace ",
"feat:",
"fix: generated title\n<!-- codex-security-release-summary:start -->\nUnreviewed injected highlight\n<!-- codex-security-release-summary:end -->",
"fix: preserve a trailing line feed\n",
"fix: preserve a trailing carriage return\r",
])("rejects nonconventional pull request title %s", async (title) => {
const result = await runReleaseLabels(title, [
" *)",
" printf '%s\\n' 'unexpected label mutation'",
" return 70",
" ;;",
]);
expect(result.status, result.stderr).toBe(1);
expect(result.stderr).toContain(
"Pull request title must follow <type>[optional scope][!]: <description>.",
);
expect(result.stdout).not.toContain("unexpected label mutation");
});
test.each([
{ title: "fix: retitle an internal change", expectedLabel: "bug" },
{
title: "feat: retitle an internal change",
expectedLabel: "enhancement",
},
{
title: "docs: retitle an internal change",
expectedLabel: "documentation",
},
{ title: "chore: retitle an internal change", expectedLabel: null },
])(
"preserves a manually excluded release and reconciles its category after retitling to $title",
async ({ title, expectedLabel }) => {
const result = await runReleaseLabels(
title,
[
' "GET repos/test/codex-security/issues/17/labels")',
" printf '%s\\n' enhancement skip-release-notes",
" ;;",
' "GET repos/test/codex-security/issues/17/timeline?per_page=100")',
" printf '%b\\n' 'enhancement\\tgithub-actions[bot]' 'skip-release-notes\\tgithub-actions[bot]' 'skip-release-notes\\ttrusted-reviewer'",
" ;;",
' "DELETE repos/test/codex-security/issues/17/labels/enhancement")',
" printf '%s\\n' 'removed a stale managed category'",
" ;;",
' "DELETE repos/test/codex-security/issues/17/labels/skip-release-notes")',
" printf '%s\\n' 'removed a manually excluded release label'",
" return 70",
" ;;",
],
[
"base64() {",
' if [[ "${1:-}" == "--decode" ]]; then',
" return 64",
" fi",
' command base64 "$@"',
"}",
],
);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain(
"Preserving existing skip-release-notes label.",
);
expect(result.stdout).not.toContain(
"removed a manually excluded release label",
);
if (expectedLabel === "enhancement") {
expect(result.stdout).not.toContain("removed a stale managed category");
expect(result.stdout).not.toContain("labels[]=enhancement");
} else {
expect(result.stdout).toContain("removed a stale managed category");
if (expectedLabel === null) {
expect(result.stdout).not.toContain("labels[]=");
} else {
expect(result.stdout).toContain(`labels[]=${expectedLabel}`);
}
}
},
);
test("preserves the latest unattributed skip label after earlier automation", async () => {
const result = await runReleaseLabels("feat: customer-visible change", [
' "GET repos/test/codex-security/issues/17/labels")',
" printf '%s\\n' skip-release-notes",
" ;;",
' "GET repos/test/codex-security/issues/17/timeline?per_page=100")',
' if [[ "$*" == *"__unattributed__"* ]]; then',
" printf '%b\\n' 'skip-release-notes\\tgithub-actions[bot]' 'skip-release-notes\\t__unattributed__'",
" else",
" printf '%b\\n' 'skip-release-notes\\tgithub-actions[bot]' 'skip-release-notes\\t'",
" fi",
" ;;",
' "DELETE repos/test/codex-security/issues/17/labels/skip-release-notes")',
" printf '%s\\n' 'removed an unattributed release exclusion'",
" return 70",
" ;;",
]);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain(
"Preserving existing skip-release-notes label.",
);
expect(result.stdout).not.toContain("removed an unattributed release");
expect(result.stdout).toContain("labels[]=enhancement");
});
test.each([
{ title: "feat: publish a customer feature", label: "enhancement" },
{ title: "fix: publish a customer fix", label: "bug" },
{ title: "docs: publish customer documentation", label: "documentation" },
{ title: "chore: stop excluding an internal change", label: null },
{ title: "chore(deps): upgrade a dependency", label: null },
{ title: "security: publish a security fix", label: null },
{ title: "deps: upgrade a dependency", label: null },
])(
"reconciles an automatic skip label after retitling to $title",
async ({ title, label }) => {
const result = await runReleaseLabels(title, [
' "GET repos/test/codex-security/issues/17/labels")',
" printf '%s\\n' skip-release-notes",
" ;;",
' "GET repos/test/codex-security/issues/17/timeline?per_page=100")',
" printf '%b\\n' 'skip-release-notes\\tgithub-actions[bot]'",
" ;;",
' "DELETE repos/test/codex-security/issues/17/labels/skip-release-notes")',
" printf '%s\\n' 'removed automatically applied skip-release-notes'",
" ;;",
]);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain(
"removed automatically applied skip-release-notes",
);
expect(result.stdout).not.toContain(
"Preserving existing skip-release-notes label.",
);
if (label === null) {
expect(result.stdout).toContain(
"No automatic release-note category applies.",
);
} else {
expect(result.stdout).toContain(`labels[]=${label}`);
}
},
);
test.each([
{ title: "feat!: breaking feature", label: "breaking-change" },
{ title: "feat(api)!: breaking feature", label: "breaking-change" },
{ title: "fix!: breaking fix", label: "breaking-change" },
{ title: "fix(api)!: breaking fix", label: "breaking-change" },
{ title: "docs!: breaking documentation", label: "breaking-change" },
{ title: "chore(release)!: require migration", label: "breaking-change" },
{
title: "docs(api)!: breaking documentation",
label: "breaking-change",
},
{
title: "security(api)!: breaking security change",
label: "breaking-change",
},
])("categorizes breaking-change title $title", async ({ title, label }) => {
const result = await runReleaseLabels(title, [
' "GET repos/test/codex-security/issues/17/labels")',
" return 0",
" ;;",
' "GET repos/test/codex-security/labels/breaking-change")',
" return 0",
" ;;",
]);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain(`labels[]=${label}`);
});
test.each([
"chore(release): 0.1.24",
"release: automate published notes",
"test: extend coverage",
])(
"excludes %s and recovers from concurrent skip-label creation",
async (title) => {
const result = await runReleaseLabels(
title,
[
' "GET repos/test/codex-security/issues/17/labels")',
" return 0",
" ;;",
' "GET repos/test/codex-security/labels/skip-release-notes")',
' [[ "$skip_label_exists" == 1 ]]',
" ;;",
' "POST repos/test/codex-security/labels")',
" skip_label_exists=1",
" return 1",
" ;;",
' "POST repos/test/codex-security/issues/17/labels")',
' if [[ "$skip_label_exists" != 1 ]]; then return 65; fi',
" printf '%s\\n' 'applied skip-release-notes'",
" ;;",
" *) return 66 ;;",
],
["skip_label_exists=0"],
);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain("applied skip-release-notes");
},
);
test("documents JSON stdin for every verification command", async () => {
const result = await runCommand(
"node",
[fileURLToPath(automationScript), "unknown"],
{ timeout: 10_000 },
);
expect(result.status).toBe(1);
expect(result.stderr).toContain("published npm versions JSON from stdin");
expect(result.stderr).toContain("package metadata JSON from stdin");
expect(result.stderr).toContain("signature audit JSON from stdin");
});
});
test.skipIf(process.platform === "win32").each([
{
title: "fix: synthetic correction",
manual: "documentation",
stale: "enhancement",
},
{ title: "feat: synthetic feature", manual: "breaking-change", stale: "bug" },
{ title: "feat: synthetic feature", manual: "bug", stale: "documentation" },
{ title: "feat: synthetic feature", manual: "bug", stale: "enhancement" },
])(
"uses the manual $manual release category over stale $stale after retitling to $title",
async ({ title, manual, stale }) => {
const result = await runReleaseLabels(
title,
[
` "GET repos/test/codex-security/issues/17/labels") printf '%s\\n' "$MANUAL_LABEL" "$STALE_LABEL" ;;`,
` "GET repos/test/codex-security/issues/17/timeline?per_page=100") printf '%s\\t%s\\n' "$MANUAL_LABEL" synthetic-reviewer "$STALE_LABEL" 'github-actions[bot]' ;;`,
' "DELETE repos/test/codex-security/issues/17/labels/$STALE_LABEL") echo removed-stale-label ;;',
' "DELETE "*) echo removed-manual-label; return 70 ;;',
' "POST "*) echo added-automatic-category ;;',
],
[`MANUAL_LABEL=${manual}`, `STALE_LABEL=${stale}`],
);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain(`Preserving existing ${manual} label.`);
expect(result.stdout).toContain("removed-stale-label");
expect(result.stdout).not.toContain("removed-manual-label");
expect(result.stdout).not.toContain("added-automatic-category");
},
);