Lightweight coding agent that runs in your terminal
Rust
Parallelize R2 asset publishing with DotSlash (#36871)
zsol-openai2a16af823456
diff --git a/.github/scripts/publish_r2_release.py b/.github/scripts/publish_r2_release.pyindex 59825358aa..e7d351ae75 100755--- a/.github/scripts/publish_r2_release.py+++ b/.github/scripts/publish_r2_release.py@@ -21,6 +21,7 @@ import reimport subprocessimport sysimport tempfile+from concurrent.futures import ThreadPoolExecutor, as_completedfrom pathlib import Pathfrom typing import Any, NamedTuple, NoReturnfrom urllib.parse import quote@@ -30,6 +31,7 @@ PREFIX = "codex"REPOSITORY = "openai/codex"RELEASE_METADATA_NAME = "release.json"INSTALLER_NAMES = ("install.sh", "install.ps1")+MAX_UPLOAD_WORKERS = 8# Keep this pattern in sync with release-tag validation in# .github/workflows/rust-release.yml.VERSION_RE = re.compile(@@ -38,6 +40,7 @@ VERSION_RE = re.compile()CRC64_RE = re.compile(r"^[A-Za-z0-9+/]{11}=$")SHA256_RE = re.compile(r"^sha256:([0-9a-f]{64})$")+MISSING_OBJECT_RE = re.compile(r"\((?:404|NoSuchKey|NotFound)\)")class PublishError(RuntimeError):@@ -65,7 +68,7 @@ def run_command(args: list[str]) -> str:return result.stdout or ""-def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:+def get_release_metadata(tag: str, directory: Path, stage: str) -> list[ReleaseAsset]:try:metadata = json.loads(run_command(@@ -83,21 +86,9 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:]))- run_command(- [- "gh",- "release",- "download",- tag,- "--repo",- REPOSITORY,- "--dir",- str(directory),- ]- )except (OSError, subprocess.CalledProcessError) as error:raise PublishError(- f"GitHub release download failed for {tag}: {error}"+ f"GitHub release metadata request failed for {tag}: {error}") from errorexcept json.JSONDecodeError as error:raise PublishError(@@ -112,6 +103,9 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:raise PublishError(f"GitHub returned invalid release metadata for {tag}: {asset!r}")+ # DotSlash runs concurrently, so defer its in-progress assets to finalization.+ if stage == "assets" and asset.get("state") != "uploaded":+ continuename = asset.get("name")size = asset.get("size")digest = asset.get("digest")@@ -131,14 +125,37 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:)expected[name] = ReleaseAsset(directory / name, size, match.group(1))- assets = sorted(directory.iterdir(), key=lambda path: path.name)- if not assets:+ if not expected:raise PublishError(f"GitHub Release {tag} has no assets")- if any(not path.is_file() for path in assets) or {- path.name for path in assets- } != set(expected):+ return sorted(expected.values(), key=lambda asset: asset.path.name)+++def download_assets(tag: str, directory: Path, assets: list[ReleaseAsset]) -> None:+ if not assets:+ return++ command = [+ "gh",+ "release",+ "download",+ tag,+ "--repo",+ REPOSITORY,+ "--dir",+ str(directory),+ ]+ for asset in assets:+ command.extend(["--pattern", asset.path.name])++ try:+ run_command(command)+ except (OSError, subprocess.CalledProcessError) as error:+ raise PublishError(+ f"GitHub release download failed for {tag}: {error}"+ ) from error++ if any(not asset.path.is_file() for asset in assets):raise PublishError("GitHub returned invalid release assets")- return [expected[path.name] for path in assets]def stream_digest(source: Any) -> tuple[int, str]:@@ -150,6 +167,17 @@ def stream_digest(source: Any) -> tuple[int, str]:return size, digest.hexdigest()+def validate_asset(asset: ReleaseAsset) -> None:+ with asset.path.open("rb") as source:+ size, sha256 = stream_digest(source)+ if size != asset.size or sha256 != asset.sha256:+ raise PublishError(+ f"GitHub asset mismatch for {asset.path.name}: expected "+ f"size={asset.size} sha256={asset.sha256}, got "+ f"size={size} sha256={sha256}"+ )++def raise_s3(action: str, key: str, error: Exception, detail: str | None = None) -> NoReturn:@@ -246,6 +274,7 @@ def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> N)for name in INSTALLER_NAMES:asset = installers[name]+ validate_asset(asset)installer_key = f"{PREFIX}/{name}"put_object(endpoint, installer_key, asset.path, asset.sha256, extra_args=[])verify_remote(endpoint, installer_key, asset.size, asset.sha256)@@ -256,11 +285,92 @@ def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> N)+def publish_asset(+ endpoint: str, version: str, asset: ReleaseAsset, *, verify: bool = True+) -> dict[str, Any]:+ validate_asset(asset)+ key = f"{PREFIX}/releases/{version}/{asset.path.name}"+ put_object(endpoint, key, asset.path, asset.sha256, extra_args=["--no-overwrite"])+ if verify:+ verify_remote(endpoint, key, asset.size, asset.sha256)+ status = "published and verified" if verify else "published"+ print(+ f"{status} s3://{BUCKET}/{key} size={asset.size} sha256={asset.sha256}",+ file=sys.stderr,+ )+ return {+ "key": key,+ "name": asset.path.name,+ "sha256": asset.sha256,+ "size": asset.size,+ }+++def publish_assets(+ endpoint: str, version: str, assets: list[ReleaseAsset], *, verify: bool = True+) -> list[dict[str, Any]]:+ if not assets:+ return []++ published = {}+ with ThreadPoolExecutor(+ max_workers=min(MAX_UPLOAD_WORKERS, len(assets))+ ) as executor:+ futures = {+ executor.submit(+ publish_asset, endpoint, version, asset, verify=verify+ ): asset+ for asset in assets+ }+ for future in as_completed(futures):+ asset = futures[future]+ published[asset.path.name] = future.result()+ return [published[asset.path.name] for asset in assets]+++def find_published_assets(+ endpoint: str, version: str, assets: list[ReleaseAsset]+) -> dict[str, dict[str, Any]]:+ published = {}+ with ThreadPoolExecutor(+ max_workers=min(MAX_UPLOAD_WORKERS, len(assets))+ ) as executor:+ futures = {+ executor.submit(+ verify_remote,+ endpoint,+ f"{PREFIX}/releases/{version}/{asset.path.name}",+ asset.size,+ asset.sha256,+ ): asset+ for asset in assets+ }+ for future in as_completed(futures):+ asset = futures[future]+ try:+ future.result()+ except PublishError as error:+ cause = error.__cause__+ if isinstance(cause, subprocess.CalledProcessError) and (+ MISSING_OBJECT_RE.search(cause.stderr or "")+ ):+ continue+ raise+ published[asset.path.name] = {+ "key": f"{PREFIX}/releases/{version}/{asset.path.name}",+ "name": asset.path.name,+ "sha256": asset.sha256,+ "size": asset.size,+ }+ return published++def parse_args() -> argparse.Namespace:parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("--tag", required=True)parser.add_argument("--make-latest", choices=("true", "false"), required=True)parser.add_argument("--prerelease", choices=("true", "false"), required=True)+ parser.add_argument("--stage", choices=("assets", "finalize"), required=True)return parser.parse_args()@@ -276,50 +386,62 @@ def main() -> int:version = args.tag.removeprefix("rust-v")if args.tag == version or not VERSION_RE.fullmatch(version):raise PublishError(f"invalid rust release tag: {args.tag}")- published = []- metadata_assets = []with tempfile.TemporaryDirectory() as temp_dir:assets_directory = Path(temp_dir) / "assets"assets_directory.mkdir()- assets = download_assets(args.tag, assets_directory)- for asset in assets:- with asset.path.open("rb") as source:- size, sha256 = stream_digest(source)- if size != asset.size or sha256 != asset.sha256:- raise PublishError(- f"GitHub asset mismatch for {asset.path.name}: expected "- f"size={asset.size} sha256={asset.sha256}, got "- f"size={size} sha256={sha256}"- )- for asset in assets:- path = asset.path- size = asset.size- sha256 = asset.sha256- key = f"{PREFIX}/releases/{version}/{path.name}"- put_object(endpoint, key, path, sha256, extra_args=["--no-overwrite"])- verify_remote(endpoint, key, size, sha256)+ assets = get_release_metadata(args.tag, assets_directory, args.stage)++ if args.stage == "assets":+ download_assets(args.tag, assets_directory, assets)+ published = publish_assets(endpoint, version, assets, verify=False)print(- f"published and verified s3://{BUCKET}/{key} "- f"size={size} sha256={sha256}",- file=sys.stderr,- )- published.append(- {- "key": key,- "sha256": sha256,- "size": size,- }+ json.dumps(+ {+ "assetCount": len(published),+ "assets": published,+ "releasePrefix": f"{PREFIX}/releases/{version}/",+ "stage": args.stage,+ "tag": args.tag,+ "version": version,+ },+ sort_keys=True,+ ))- metadata_assets.append(- {- "name": path.name,- "digest": f"sha256:{sha256}",- "browser_download_url": (- f"https://releases.openai.com/{PREFIX}/releases/"- f"{version}/{quote(path.name, safe='')}"- ),- }+ return 0++ previously_published = find_published_assets(endpoint, version, assets)+ remaining = [+ asset for asset in assets if asset.path.name not in previously_published+ ]+ required_downloads = list(remaining)+ if args.prerelease == "false":+ required_downloads.extend(+ asset+ for asset in assets+ if asset.path.name in INSTALLER_NAMES+ and asset.path.name in previously_published)+ download_assets(args.tag, assets_directory, required_downloads)+ additionally_published = {+ asset["name"]: asset+ for asset in publish_assets(endpoint, version, remaining)+ }+ published = [+ previously_published.get(asset.path.name)+ or additionally_published[asset.path.name]+ for asset in assets+ ]+ metadata_assets = [+ {+ "name": asset.path.name,+ "digest": f"sha256:{asset.sha256}",+ "browser_download_url": (+ f"https://releases.openai.com/{PREFIX}/releases/"+ f"{version}/{quote(asset.path.name, safe='')}"+ ),+ }+ for asset in assets+ ]metadata_path = Path(temp_dir) / RELEASE_METADATA_NAMEmetadata_path.write_text(@@ -393,6 +515,7 @@ def main() -> int:"size": metadata_size,},"releasePrefix": f"{PREFIX}/releases/{version}/",+ "stage": args.stage,"tag": args.tag,"version": version,},diff --git a/.github/workflows/r2-release.yml b/.github/workflows/r2-release.ymlindex 3abe94dde0..96a07685cf 100644--- a/.github/workflows/r2-release.yml+++ b/.github/workflows/r2-release.yml@@ -12,6 +12,9 @@ on:prerelease:required: truetype: boolean+ stage:+ required: true+ type: stringpermissions: {}@@ -40,9 +43,11 @@ jobs:RELEASE_TAG: ${{ inputs.tag }}RELEASE_MAKE_LATEST: ${{ inputs.make_latest }}RELEASE_PRERELEASE: ${{ inputs.prerelease }}+ RELEASE_STAGE: ${{ inputs.stage }}run: |set -euo pipefailpython3 .github/scripts/publish_r2_release.py \--tag "${RELEASE_TAG}" \--make-latest "${RELEASE_MAKE_LATEST}" \- --prerelease "${RELEASE_PRERELEASE}"+ --prerelease "${RELEASE_PRERELEASE}" \+ --stage "${RELEASE_STAGE}"diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.ymlindex 04ae92db0c..1b423273af 100644--- a/.github/workflows/rust-release.yml+++ b/.github/workflows/rust-release.yml@@ -1356,9 +1356,22 @@ jobs:# (e.g. -alpha, -beta). Otherwise publish a normal release.prerelease: ${{ steps.release_name.outputs.prerelease }}+ publish-r2-assets:+ name: publish-r2-assets+ needs: release+ permissions:+ contents: read+ uses: ./.github/workflows/r2-release.yml+ secrets: inherit+ with:+ tag: ${{ needs.release.outputs.tag }}+ make_latest: ${{ fromJSON(needs.release.outputs.make_latest) }}+ prerelease: ${{ fromJSON(needs.release.outputs.prerelease) }}+ stage: assets+publish-r2:name: publish-r2- needs: [release, publish-dotslash]+ needs: [release, publish-dotslash, publish-r2-assets]permissions:contents: readuses: ./.github/workflows/r2-release.yml@@ -1367,6 +1380,7 @@ jobs:tag: ${{ needs.release.outputs.tag }}make_latest: ${{ fromJSON(needs.release.outputs.make_latest) }}prerelease: ${{ fromJSON(needs.release.outputs.prerelease) }}+ stage: finalizepublish-dotslash:name: publish-dotslash