Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Prevent stable release pointers from moving backward (#51186)

andrewgu-oai5cbcf810e5f4
diff --git a/.github/scripts/check_github_canary.py b/.github/scripts/check_github_canary.py
deleted file mode 100644
index b8b74f4a1e..0000000000
--- a/.github/scripts/check_github_canary.py
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-"""Compare a publishing release with the canary Cargo manifest from stdin.
-
-Usage: check_github_canary.py RELEASE_VERSION
-"""
-
-import sys
-import tomllib
-
-from releases import should_update_version
-
-
-def main() -> None:
- try:
- current_version = tomllib.loads(sys.stdin.read())["workspace"]["package"][
- "version"
- ]
- except (tomllib.TOMLDecodeError, KeyError, TypeError):
- current_version = ""
- if not isinstance(current_version, str):
- current_version = ""
- print(str(should_update_version(sys.argv[1], current_version)).lower())
-
-
-if __name__ == "__main__":
- main()
diff --git a/.github/scripts/npm_alpha_tag.py b/.github/scripts/npm_alpha_tag.py
index cef206ff83..47e3498fe0 100644
--- a/.github/scripts/npm_alpha_tag.py
+++ b/.github/scripts/npm_alpha_tag.py
@@ -1,4 +1,4 @@
-"""Choose a forward-only npm alpha dist-tag under rust-release's concurrency lock."""
+"""Choose forward-only npm dist-tags under rust-release's publication lock."""
import json
from pathlib import Path
@@ -6,12 +6,12 @@ import subprocess
import sys
import tarfile
-from releases import should_update_version
+from releases import is_valid_stable_release_version, should_update_version
def npm_alpha_tag(tarball: Path, version: str, tag: str) -> str:
# Read the actual package name: platform archives share @openai/codex,
- # while the SDK and proxy have independent alpha pointers.
+ # while the SDK and proxy have independent latest/alpha pointers.
with tarfile.open(tarball, "r:gz") as archive:
with archive.extractfile("package/package.json") as manifest:
package = json.load(manifest)["name"]
@@ -23,14 +23,19 @@ def npm_alpha_tag(tarball: Path, version: str, tag: str) -> str:
text=True,
)
current = json.loads(result.stdout).get(tag, "")
- if tag != "alpha":
+ if tag not in ("alpha", "latest"):
# build_npm_package.py appends -<platform> to the release version.
current = current.removesuffix(f"-{tag.removeprefix('alpha-')}")
+ # Stable tags fail closed on unreadable metadata. Alpha tags keep their
+ # existing recovery behavior, as with the other prerelease channels.
+ is_alpha = tag == "alpha" or tag.startswith("alpha-")
+ if current and not is_alpha and not is_valid_stable_release_version(current):
+ raise ValueError(f"Cannot compare current {package}@{tag} version: {current}")
if should_update_version(version, current):
return tag
# npm publish always writes a tag. Keep this version installable without
- # changing alpha, alpha-<platform>, or latest. Prefix avoids semver tags,
+ # changing latest, alpha, or their platform tags. Prefix avoids semver tags,
# which npm rejects. Include the platform for versions of the same package.
print(
f"Keeping {package}@{tag} at {current}; publishing {version} separately",
diff --git a/.github/scripts/publish_r2_release.py b/.github/scripts/publish_r2_release.py
index 5b6ded1331..456cab2cdb 100755
--- a/.github/scripts/publish_r2_release.py
+++ b/.github/scripts/publish_r2_release.py
@@ -10,8 +10,9 @@ versioned prefix includes every release asset plus installer-facing
objects are verified, the same metadata advances ``codex/channels/latest`` when
the release is marked latest and ``codex/channels/prerelease`` for prereleases
that are newer than its current version, or when that version is unreadable.
-Stable releases also update the mutable ``codex/install.sh`` and
+Latest stable releases also update the mutable ``codex/install.sh`` and
``codex/install.ps1`` bootstrap aliases from their verified versioned assets.
+An older stable release cannot move these aliases or the latest channel back.
"""
import argparse
@@ -27,13 +28,16 @@ from pathlib import Path
from typing import Any, NamedTuple, NoReturn
from urllib.parse import quote
-from releases import is_valid_release_version, should_update_version
+from releases import (
+ is_valid_release_version,
+ is_valid_stable_release_version,
+ should_update_version,
+)
BUCKET = "releases"
PREFIX = "codex"
REPOSITORY = "openai/codex"
RELEASE_METADATA_NAME = "release.json"
-PRERELEASE_CHANNEL_KEY = f"{PREFIX}/channels/prerelease"
INSTALLER_NAMES = ("install.sh", "install.ps1")
MAX_UPLOAD_WORKERS = 8
CRC64_RE = re.compile(r"^[A-Za-z0-9+/]{11}=$")
@@ -263,27 +267,39 @@ def verify_remote(
)
-def should_update_prerelease(endpoint: str, version: str) -> bool:
+def should_update_channel(endpoint: str, version: str, channel: str) -> bool:
# Read R2 directly: the public CDN may still serve an older version.
- url = f"s3://{BUCKET}/{PRERELEASE_CHANNEL_KEY}"
+ key = f"{PREFIX}/channels/{channel}"
+ url = f"s3://{BUCKET}/{key}"
try:
raw = run_command(["aws", "s3", "cp", url, "-", "--endpoint-url", endpoint])
except subprocess.CalledProcessError as error:
if MISSING_OBJECT_RE.search(error.stderr or ""):
return True
- raise_s3("read", PRERELEASE_CHANNEL_KEY, error, (error.stderr or "").strip())
+ raise_s3("read", key, error, (error.stderr or "").strip())
except OSError as error:
- raise_s3("read", PRERELEASE_CHANNEL_KEY, error)
+ raise_s3("read", key, error)
try:
metadata = json.loads(raw)
- except json.JSONDecodeError:
+ except json.JSONDecodeError as error:
+ if channel == "latest":
+ raise PublishError(f"invalid channel metadata for {key}") from error
return True
tag = metadata.get("tag_name") if isinstance(metadata, dict) else None
if not isinstance(tag, str) or not tag.startswith("rust-v"):
+ if channel == "latest":
+ raise PublishError(f"invalid channel metadata for {key}")
return True
- return should_update_version(version, tag.removeprefix("rust-v"))
+ current = tag.removeprefix("rust-v")
+ if channel == "latest":
+ if not is_valid_stable_release_version(current):
+ raise PublishError(f"invalid stable version in channel metadata for {key}")
+ # Reruns can repair installer writes interrupted before channel finalization.
+ if version == current:
+ return True
+ return should_update_version(version, current)
def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> None:
@@ -435,7 +451,7 @@ def main() -> int:
asset for asset in assets if asset.path.name not in previously_published
]
required_downloads = list(remaining)
- if args.prerelease == "false":
+ if args.prerelease == "false" and args.make_latest == "true":
required_downloads.extend(
asset
for asset in assets
@@ -497,15 +513,17 @@ def main() -> int:
f"size={metadata_size} sha256={metadata_sha256}",
file=sys.stderr,
)
- if args.prerelease == "false":
- publish_installers(endpoint, args.tag, assets)
channels = []
- if args.make_latest == "true":
+ if args.make_latest == "true" and should_update_channel(
+ endpoint, version, "latest"
+ ):
channels.append("latest")
- if args.prerelease == "true" and should_update_prerelease(
- endpoint, version
+ if args.prerelease == "true" and should_update_channel(
+ endpoint, version, "prerelease"
):
channels.append("prerelease")
+ if "latest" in channels:
+ publish_installers(endpoint, args.tag, assets)
for channel in channels:
channel_key = f"{PREFIX}/channels/{channel}"
put_object(
diff --git a/.github/scripts/releases.py b/.github/scripts/releases.py
index efb0bba116..bc0bb30985 100644
--- a/.github/scripts/releases.py
+++ b/.github/scripts/releases.py
@@ -1,5 +1,7 @@
-"""Validate and compare the version formats used by Codex releases."""
+"""Validate Codex releases and check whether their GitHub channel pointers can advance."""
+import argparse
+import json
import re
import sys
@@ -13,6 +15,10 @@ def is_valid_release_version(version: str) -> bool:
return _VERSION_RE.fullmatch(version) is not None
+def is_valid_stable_release_version(version: str) -> bool:
+ return is_valid_release_version(version) and "-" not in version
+
+
def _version_key(version: str) -> tuple[tuple[int, ...], int, tuple[int, ...]]:
base, _, prerelease = version.partition("-")
channel, _, suffix = prerelease.partition(".")
@@ -34,5 +40,46 @@ def should_update_version(version: str, current_version: str) -> bool:
return _version_key(version) > _version_key(current_version)
+def should_make_latest(version: str, latest: dict) -> bool:
+ if not is_valid_stable_release_version(version):
+ raise ValueError(f"expected a stable release version: {version}")
+ tag = latest.get("tag_name", "")
+ current = tag.removeprefix("rust-v") if isinstance(tag, str) else ""
+ if (
+ not isinstance(tag, str)
+ or not tag.startswith("rust-v")
+ or not is_valid_stable_release_version(current)
+ or latest.get("draft") is not False
+ or latest.get("prerelease") is not False
+ ):
+ raise ValueError("Cannot determine the current GitHub latest stable release")
+ # An equal version must be able to finish a partially completed publication.
+ return version == current or should_update_version(version, current)
+
+
if __name__ == "__main__":
- raise SystemExit(0 if is_valid_release_version(sys.argv[1]) else 1)
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("version")
+ parser.add_argument(
+ "--check-github",
+ choices=("latest", "canary"),
+ help="Compare with the latest release JSON or canary Cargo.toml on stdin.",
+ )
+ args = parser.parse_args()
+ if args.check_github is None:
+ raise SystemExit(0 if is_valid_release_version(args.version) else 1)
+ if args.check_github == "latest":
+ update = should_make_latest(args.version, json.load(sys.stdin))
+ else:
+ import tomllib
+
+ try:
+ current_version = tomllib.loads(sys.stdin.read())["workspace"]["package"][
+ "version"
+ ]
+ except (tomllib.TOMLDecodeError, KeyError, TypeError):
+ current_version = ""
+ if not isinstance(current_version, str):
+ current_version = ""
+ update = should_update_version(args.version, current_version)
+ print(str(update).lower())
diff --git a/.github/scripts/test_releases.py b/.github/scripts/test_releases.py
index 2133724e63..b2fa6a5163 100644
--- a/.github/scripts/test_releases.py
+++ b/.github/scripts/test_releases.py
@@ -8,7 +8,8 @@ import unittest
from unittest.mock import patch
from npm_alpha_tag import npm_alpha_tag
-from releases import is_valid_release_version, should_update_version
+import publish_r2_release as r2
+from releases import is_valid_release_version, should_make_latest, should_update_version
class VersionComparisonTest(unittest.TestCase):
@@ -61,10 +62,32 @@ class VersionComparisonTest(unittest.TestCase):
class NpmAlphaTagTest(unittest.TestCase):
+ def write_package(self, tarball: Path, package: str) -> None:
+ content = json.dumps({"name": package}).encode()
+ with tarfile.open(tarball, "w:gz") as archive:
+ info = tarfile.TarInfo("package/package.json")
+ info.size = len(content)
+ archive.addfile(info, io.BytesIO(content))
+
def test_published_tags_across_packages_and_platforms(self) -> None:
# Each package/tag reads its own registry value, even after a partially
# published release. Also exercise a missing pointer and dotted hotfix.
cases = [
+ (
+ "@openai/codex-sdk",
+ "latest",
+ "0.100.0",
+ "0.99.9",
+ "release-0.99.9-latest",
+ ),
+ (
+ "@openai/codex",
+ "linux-x64",
+ "0.99.9-linux-x64",
+ "0.100.0",
+ "linux-x64",
+ ),
+ ("@openai/codex-responses-api-proxy", "latest", None, "0.100.0", "latest"),
(
"@openai/codex",
"alpha",
@@ -101,16 +124,19 @@ class NpmAlphaTagTest(unittest.TestCase):
"release-0.158.0-alpha.10.1-alpha",
),
("@openai/codex-sdk", "alpha", None, "0.158.0-alpha.10", "alpha"),
+ (
+ "@openai/codex",
+ "alpha-linux-x64",
+ "broken-linux-x64",
+ "0.158.0-alpha.10",
+ "alpha-linux-x64",
+ ),
]
with tempfile.TemporaryDirectory() as tmpdir:
tarball = Path(tmpdir) / "package.tgz"
for package, tag, current, version, expected in cases:
with self.subTest(package=package, tag=tag, version=version):
- content = json.dumps({"name": package}).encode()
- with tarfile.open(tarball, "w:gz") as archive:
- info = tarfile.TarInfo("package/package.json")
- info.size = len(content)
- archive.addfile(info, io.BytesIO(content))
+ self.write_package(tarball, package)
tags = {tag: current} if current is not None else {}
result = subprocess.CompletedProcess([], 0, json.dumps(tags))
with patch(
@@ -131,14 +157,26 @@ class NpmAlphaTagTest(unittest.TestCase):
text=True,
)
+ def test_stable_tags_reject_malformed_and_prerelease_versions(self) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ tarball = Path(tmpdir) / "package.tgz"
+ self.write_package(tarball, "@openai/codex")
+ for tag, current in (
+ ("latest", "broken"),
+ ("linux-x64", "0.100.0-alpha.1-linux-x64"),
+ ):
+ with self.subTest(tag=tag, current=current):
+ result = subprocess.CompletedProcess(
+ [], 0, json.dumps({tag: current})
+ )
+ with patch("npm_alpha_tag.subprocess.run", return_value=result):
+ with self.assertRaises(ValueError):
+ npm_alpha_tag(tarball, "0.99.9", tag)
+
def test_registry_failure_prevents_publish(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
tarball = Path(tmpdir) / "package.tgz"
- content = b'{"name": "@openai/codex"}'
- with tarfile.open(tarball, "w:gz") as archive:
- info = tarfile.TarInfo("package/package.json")
- info.size = len(content)
- archive.addfile(info, io.BytesIO(content))
+ self.write_package(tarball, "@openai/codex")
with patch(
"npm_alpha_tag.subprocess.run",
side_effect=subprocess.CalledProcessError(1, "npm"),
@@ -147,5 +185,39 @@ class NpmAlphaTagTest(unittest.TestCase):
npm_alpha_tag(tarball, "0.158.0-alpha.10", "alpha")
+class StablePublicationTests(unittest.TestCase):
+ def test_github_latest_orders_numerically_and_allows_equal_reruns(self):
+ latest = {"tag_name": "rust-v0.100.0", "draft": False, "prerelease": False}
+ self.assertFalse(should_make_latest("0.99.9", latest))
+ self.assertTrue(should_make_latest("0.100.0", latest))
+ self.assertTrue(should_make_latest("0.101.0", latest))
+
+ def test_github_latest_rejects_unpublished_or_unreadable_metadata(self):
+ for latest in (
+ {},
+ {"tag_name": "rust-vunknown", "draft": False, "prerelease": False},
+ {"tag_name": "rust-v0.100.0", "draft": True, "prerelease": False},
+ {"tag_name": "rust-v0.100.0", "draft": False, "prerelease": True},
+ ):
+ with self.subTest(latest=latest):
+ with self.assertRaises(ValueError):
+ should_make_latest("0.99.9", latest)
+
+ def test_r2_checks_its_channel_and_only_repairs_unreadable_prereleases(self):
+ with patch.object(r2, "run_command") as command:
+ command.return_value = '{"tag_name":"rust-v0.100.0"}'
+ self.assertFalse(r2.should_update_channel("endpoint", "0.99.9", "latest"))
+ self.assertTrue(r2.should_update_channel("endpoint", "0.100.0", "latest"))
+ self.assertIn(
+ "s3://releases/codex/channels/latest", command.call_args.args[0]
+ )
+ command.return_value = "invalid"
+ with self.assertRaises(r2.PublishError):
+ r2.should_update_channel("endpoint", "0.99.9", "latest")
+ self.assertTrue(
+ r2.should_update_channel("endpoint", "0.99.9-alpha.1", "prerelease")
+ )
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml
index 13e58ee28e..6dd30bce85 100644
--- a/.github/workflows/rust-release.yml
+++ b/.github/workflows/rust-release.yml
@@ -1639,7 +1639,10 @@ jobs:
- name: Define release name
id: release_name
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
+ set -euo pipefail
# Extract the version from the tag name, which is in the format
# "rust-v0.1.0".
version="${GITHUB_REF_NAME#rust-v}"
@@ -1648,7 +1651,9 @@ jobs:
echo "make_latest=false" >> $GITHUB_OUTPUT
echo "prerelease=true" >> $GITHUB_OUTPUT
else
- echo "make_latest=true" >> $GITHUB_OUTPUT
+ make_latest="$(gh api "repos/${GITHUB_REPOSITORY}/releases/latest" \
+ | python3 .github/scripts/releases.py "$version" --check-github latest)"
+ echo "make_latest=${make_latest}" >> $GITHUB_OUTPUT
echo "prerelease=false" >> $GITHUB_OUTPUT
fi
@@ -1903,11 +1908,9 @@ jobs:
;;
esac
- # The workflow concurrency group holds the lock from this lookup
- # through publish, including concurrent alpha/hotfix release runs.
- if [[ "${NPM_TAG}" == "alpha" ]]; then
- tag="$(python3 .github/scripts/npm_alpha_tag.py "${tarball}" "${VERSION}" "${tag}")"
- fi
+ # The publication workflow holds the lock from this lookup through
+ # publish, including concurrent releases from other version lines.
+ tag="$(python3 .github/scripts/npm_alpha_tag.py "${tarball}" "${VERSION}" "${tag:-latest}")"
publish_cmd=(npm publish "${GITHUB_WORKSPACE}/${tarball}")
if [[ -n "${tag}" ]]; then
@@ -2059,7 +2062,7 @@ jobs:
# is JSON metadata with the file contents encoded as base64.
update="$(gh api "repos/$GITHUB_REPOSITORY/contents/codex-rs/Cargo.toml?ref=latest-alpha-cli" \
-H 'Accept: application/vnd.github.raw+json' \
- | python3 .github/scripts/check_github_canary.py "$RELEASE_VERSION")"
+ | python3 .github/scripts/releases.py "$RELEASE_VERSION" --check-github canary)"
echo "update=$update" >> "$GITHUB_OUTPUT"
- name: Update latest-alpha-cli branch
if: success() && steps.github_canary.outputs.update == 'true'