Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Pin V8 release manifests and prevent published release replacement (#43444)

Charlie Marshc0b6285711f7
diff --git a/.github/actions/setup-rusty-v8/action.yml b/.github/actions/setup-rusty-v8/action.yml
index 449ab653fd..7b78d8cbdb 100644
--- a/.github/actions/setup-rusty-v8/action.yml
+++ b/.github/actions/setup-rusty-v8/action.yml
@@ -33,11 +33,25 @@ runs:
archive_path="${binding_dir}/${archive_name}"
binding_path="${binding_dir}/${binding_name}"
checksums_path="${binding_dir}/${checksums_name}"
+ trusted_checksums="${GITHUB_WORKSPACE}/third_party/v8/rusty_v8_${version//./_}_release_manifests.sha256"
+
+ if command -v sha256sum >/dev/null 2>&1; then
+ checksum_command=(sha256sum --check -)
+ else
+ checksum_command=(shasum -a 256 --check -)
+ fi
mkdir -p "${binding_dir}"
+ curl -fsSL "${base_url}/${checksums_name}" -o "${checksums_path}"
+ # Check the original manifest bytes even when either checksum file uses CRLF.
+ expected_manifest_checksum="$(grep -F " ${checksums_name}" "${trusted_checksums}" | cut -d ' ' -f 1)"
+ actual_manifest_checksum="$(python3 -c 'import hashlib, pathlib, sys; print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())' "${checksums_path}")"
+ if [[ "${actual_manifest_checksum}" != "${expected_manifest_checksum}" ]]; then
+ echo "Checksum mismatch for ${checksums_name}: expected ${expected_manifest_checksum}, got ${actual_manifest_checksum}" >&2
+ exit 1
+ fi
curl -fsSL "${base_url}/${archive_name}" -o "${archive_path}"
curl -fsSL "${base_url}/${binding_name}" -o "${binding_path}"
- curl -fsSL "${base_url}/${checksums_name}" -o "${checksums_path}"
if [[ "$(wc -l < "${checksums_path}")" -ne 2 ]]; then
echo "Expected exactly two checksums for ${TARGET} in ${checksums_path}" >&2
@@ -45,10 +59,6 @@ runs:
fi
# Existing Windows-built release manifests use CRLF line endings.
- if command -v sha256sum >/dev/null 2>&1; then
- (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | sha256sum -c -)
- else
- (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | shasum -a 256 -c -)
- fi
+ (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | "${checksum_command[@]}")
echo "RUSTY_V8_ARCHIVE=${archive_path}" >> "${GITHUB_ENV}"
echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "${GITHUB_ENV}"
diff --git a/.github/workflows/rusty-v8-release.yml b/.github/workflows/rusty-v8-release.yml
index 37c8c65370..32ec3af836 100644
--- a/.github/workflows/rusty-v8-release.yml
+++ b/.github/workflows/rusty-v8-release.yml
@@ -354,22 +354,18 @@ jobs:
restore-keys: |
rusty-v8-source-${{ matrix.target }}-sandbox-
- - name: Install and start sccache
+ - name: Install sccache
+ uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49
+ with:
+
+ - name: Start sccache
shell: pwsh
env:
SCCACHE_CACHE_SIZE: 256M
SCCACHE_DIR: ${{ github.workspace }}/upstream-rusty-v8/target/sccache
SCCACHE_IDLE_TIMEOUT: 0
- run: |
- $version = "v0.8.2"
- $platform = "x86_64-pc-windows-msvc"
- $basename = "sccache-$version-$platform"
- $url = "https://github.com/mozilla/sccache/releases/download/$version/$basename.tar.gz"
- cd ~
- curl -LO $url
- tar -xzvf "$basename.tar.gz"
- . $basename/sccache --start-server
- echo "$(pwd)/$basename" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+ run: sccache --start-server
- name: Install Chromium clang for ARM64 MSVC cross build
if: matrix.target == 'aarch64-pc-windows-msvc'
@@ -437,7 +433,6 @@ jobs:
steps:
- name: Check whether release already exists
- id: release
env:
GH_TOKEN: ${{ github.token }}
RELEASE_TAG: ${{ needs.metadata.outputs.release_tag }}
@@ -445,10 +440,14 @@ jobs:
run: |
set -euo pipefail
- if gh release view "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" > /dev/null 2>&1; then
- echo "exists=true" >> "${GITHUB_OUTPUT}"
- else
- echo "exists=false" >> "${GITHUB_OUTPUT}"
+ if is_draft="$(gh release view "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq '.isDraft' 2>/dev/null)"; then
+ if [[ "${is_draft}" != "true" ]]; then
+ echo "Refusing to replace existing published release ${RELEASE_TAG}." >&2
+ exit 1
+ fi
+
+ echo "Removing unfinished draft release for ${RELEASE_TAG}."
+ gh release delete "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" --yes
fi
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
@@ -456,22 +455,25 @@ jobs:
path: dist
- name: Create GitHub Release
- if: ${{ steps.release.outputs.exists != 'true' }}
- uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1
- with:
- tag_name: ${{ needs.metadata.outputs.release_tag }}
- name: ${{ needs.metadata.outputs.release_tag }}
- files: dist/**
- # Keep V8 artifact releases out of Codex's normal "latest release" channel.
- prerelease: true
-
- - name: Amend existing GitHub Release
- if: ${{ steps.release.outputs.exists == 'true' }}
- uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1
- with:
- tag_name: ${{ needs.metadata.outputs.release_tag }}
- name: ${{ needs.metadata.outputs.release_tag }}
- files: dist/**
- overwrite_files: true
- # Keep V8 artifact releases out of Codex's normal "latest release" channel.
- prerelease: true
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RELEASE_TAG: ${{ needs.metadata.outputs.release_tag }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ assets=()
+ while IFS= read -r -d '' asset; do
+ assets+=("${asset}")
+ done < <(find dist -type f -print0)
+ if [[ "${#assets[@]}" -eq 0 ]]; then
+ echo "No V8 release artifacts were found." >&2
+ exit 1
+ fi
+
+ # GitHub CLI keeps asset-bearing releases in draft until all uploads finish.
+ gh release create "${RELEASE_TAG}" \
+ --repo "${GITHUB_REPOSITORY}" \
+ --title "${RELEASE_TAG}" \
+ --verify-tag \
+ --prerelease \
+ "${assets[@]}"
diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.yml
index 29cb72e7c5..94f1aa0199 100644
--- a/.github/workflows/v8-canary.yml
+++ b/.github/workflows/v8-canary.yml
@@ -372,22 +372,18 @@ jobs:
restore-keys: |
rusty-v8-source-${{ matrix.target }}-${{ matrix.runner }}-sandbox-
- - name: Install and start sccache
+ - name: Install sccache
+ uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49
+ with:
+
+ - name: Start sccache
shell: pwsh
env:
SCCACHE_CACHE_SIZE: 256M
SCCACHE_DIR: ${{ steps.setup_ci.outputs.cargo-target-dir }}/sccache
SCCACHE_IDLE_TIMEOUT: 0
- run: |
- $version = "v0.8.2"
- $platform = "x86_64-pc-windows-msvc"
- $basename = "sccache-$version-$platform"
- $url = "https://github.com/mozilla/sccache/releases/download/$version/$basename.tar.gz"
- cd ~
- curl -LO $url
- tar -xzvf "$basename.tar.gz"
- . $basename/sccache --start-server
- echo "$(pwd)/$basename" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+ run: sccache --start-server
- name: Install Chromium clang for ARM64 MSVC cross build
if: matrix.target == 'aarch64-pc-windows-msvc'
diff --git a/scripts/codex_package/test_v8.py b/scripts/codex_package/test_v8.py
new file mode 100644
index 0000000000..a47f15a348
--- /dev/null
+++ b/scripts/codex_package/test_v8.py
@@ -0,0 +1,152 @@
+import hashlib
+import sys
+import tempfile
+import unittest
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from codex_package import v8
+from codex_package.targets import TARGET_SPECS, TargetSpec
+
+
+class FetchCodexV8ArtifactsTest(unittest.TestCase):
+ version = "150.4.0"
+
+ def setUp(self) -> None:
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name)
+
+ @contextmanager
+ def release(
+ self,
+ target: str,
+ *,
+ line_ending: bytes = b"\n",
+ trusted_digest: str | None = None,
+ trusted_name: str | None = None,
+ create_pins: bool = True,
+ ) -> Iterator[tuple[TargetSpec, MagicMock, str]]:
+ spec = TARGET_SPECS[target]
+ profile = v8.V8_ARTIFACT_PROFILE
+ archive_name = (
+ f"rusty_v8_{profile}_{target}.lib.gz"
+ if spec.is_windows
+ else f"librusty_v8_{profile}_{target}.a.gz"
+ )
+ binding_name = f"src_binding_{profile}_{target}.rs"
+ manifest_name = f"rusty_v8_{profile}_{target}.sha256"
+ archive = b"trusted V8 archive"
+ binding = b"trusted V8 binding"
+ manifest = (
+ line_ending.join(
+ (
+ f"{hashlib.sha256(archive).hexdigest()} {archive_name}".encode(),
+ f"{hashlib.sha256(binding).hexdigest()} {binding_name}".encode(),
+ )
+ )
+ + line_ending
+ )
+ payloads = {
+ manifest_name: manifest,
+ archive_name: archive,
+ binding_name: binding,
+ }
+
+ if create_pins:
+ pins = (
+ self.root / "third_party/v8/rusty_v8_150_4_0_release_manifests.sha256"
+ )
+ pins.parent.mkdir(parents=True)
+ digest = trusted_digest or hashlib.sha256(manifest).hexdigest()
+ name = trusted_name or manifest_name
+ pins.write_bytes(f"{digest} {name}".encode() + line_ending)
+
+ def download(_url: str, destination: Path) -> None:
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ destination.write_bytes(payloads[destination.name])
+
+ with (
+ patch.object(v8, "REPO_ROOT", self.root),
+ patch.object(v8, "download_file", side_effect=download) as download_file,
+ ):
+ yield spec, download_file, manifest_name
+
+ def test_fetches_artifacts_after_authenticating_manifest(self) -> None:
+ with self.release("x86_64-unknown-linux-gnu") as (
+ spec,
+ download,
+ manifest_name,
+ ):
+ artifacts = v8.fetch_codex_v8_artifacts(
+ spec, version=self.version, cache_root=self.root / "cache"
+ )
+
+ self.assertEqual(artifacts.archive.read_bytes(), b"trusted V8 archive")
+ self.assertEqual(artifacts.binding.read_bytes(), b"trusted V8 binding")
+ self.assertEqual(download.call_args_list[0].args[1].name, manifest_name)
+ self.assertEqual(download.call_count, 3)
+
+ def test_authenticates_windows_manifest_with_crlf(self) -> None:
+ with self.release("x86_64-pc-windows-msvc", line_ending=b"\r\n") as (
+ spec,
+ download,
+ _manifest_name,
+ ):
+ artifacts = v8.fetch_codex_v8_artifacts(
+ spec, version=self.version, cache_root=self.root / "cache"
+ )
+
+ self.assertEqual(artifacts.archive.read_bytes(), b"trusted V8 archive")
+ self.assertEqual(artifacts.binding.read_bytes(), b"trusted V8 binding")
+ self.assertEqual(download.call_count, 3)
+
+ def test_rejects_tampered_manifest_before_downloading_artifacts(self) -> None:
+ with self.release("x86_64-unknown-linux-gnu", trusted_digest="0" * 64) as (
+ spec,
+ download,
+ manifest_name,
+ ):
+ with self.assertRaisesRegex(
+ RuntimeError, "does not match its trusted SHA-256"
+ ):
+ v8.fetch_codex_v8_artifacts(
+ spec, version=self.version, cache_root=self.root / "cache"
+ )
+
+ download.assert_called_once()
+ self.assertEqual(download.call_args.args[1].name, manifest_name)
+
+ def test_rejects_missing_manifest_pin_before_downloading_artifacts(self) -> None:
+ with self.release(
+ "x86_64-unknown-linux-gnu", trusted_name="another-target.sha256"
+ ) as (spec, download, manifest_name):
+ with self.assertRaisesRegex(RuntimeError, "has no trusted SHA-256"):
+ v8.fetch_codex_v8_artifacts(
+ spec, version=self.version, cache_root=self.root / "cache"
+ )
+
+ download.assert_called_once()
+ self.assertEqual(download.call_args.args[1].name, manifest_name)
+
+ def test_rejects_missing_pin_file_before_downloading_artifacts(self) -> None:
+ with self.release("x86_64-unknown-linux-gnu", create_pins=False) as (
+ spec,
+ download,
+ manifest_name,
+ ):
+ with self.assertRaises(FileNotFoundError):
+ v8.fetch_codex_v8_artifacts(
+ spec, version=self.version, cache_root=self.root / "cache"
+ )
+
+ download.assert_called_once()
+ self.assertEqual(download.call_args.args[1].name, manifest_name)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/codex_package/v8.py b/scripts/codex_package/v8.py
index 66ecb6351e..f2ac15dd09 100644
--- a/scripts/codex_package/v8.py
+++ b/scripts/codex_package/v8.py
@@ -74,6 +74,7 @@ def fetch_codex_v8_artifacts(
checksums = cache_dir / checksums_name
download_file(f"{release_url}/{checksums.name}", checksums)
+ verify_release_checksum_manifest(checksums, version=version)
expected_checksums = load_checksums(checksums, {archive.name, binding.name})
for artifact in [archive, binding]:
ensure_valid_artifact(
@@ -107,6 +108,32 @@ def default_cache_root() -> Path:
return Path(tempfile.gettempdir()) / "codex-package"
+def verify_release_checksum_manifest(checksums_path: Path, *, version: str) -> None:
+ version_suffix = version.replace(".", "_")
+ trusted_checksums = (
+ REPO_ROOT
+ / "third_party"
+ / "v8"
+ / f"rusty_v8_{version_suffix}_release_manifests.sha256"
+ )
+
+ for line in trusted_checksums.read_text(encoding="utf-8").splitlines():
+ digest, artifact_name = line.split(maxsplit=1)
+ if artifact_name != checksums_path.name:
+ continue
+ if has_checksum(checksums_path, digest):
+ return
+
+ checksums_path.unlink(missing_ok=True)
+ raise RuntimeError(
+ f"V8 checksum manifest {checksums_path} does not match its trusted SHA-256."
+ )
+
+ raise RuntimeError(
+ f"V8 checksum manifest {checksums_path.name} has no trusted SHA-256 for {version}."
+ )
+
+
def load_checksums(checksums_path: Path, artifact_names: set[str]) -> dict[str, str]:
checksums: dict[str, str] = {}
lines = checksums_path.read_text(encoding="utf-8").splitlines()
diff --git a/third_party/v8/README.md b/third_party/v8/README.md
index 21adf3899d..fc9879b80d 100644
--- a/third_party/v8/README.md
+++ b/third_party/v8/README.md
@@ -30,7 +30,10 @@ Use this as the maintainer flow for a version bump:
matching checksum manifest and generated checksums as described below.
3. Publish a release-candidate PR and validate that `v8-canary` passes.
4. If the canary is green, publish the release tag and release build.
-5. Once the release build completes, rerun the build on the candidate branch
+5. Independently verify the published Codex-built checksum manifests and record
+ their SHA-256 digests in
+ `third_party/v8/rusty_v8_<version>_release_manifests.sha256`.
+6. Once the release build completes, rerun the build on the candidate branch
and verify that the final artifact builds and tests pass.
When changing the remaining prebuilt `rusty_v8` `http_file` inputs, keep the
diff --git a/third_party/v8/rusty_v8_150_4_0_release_manifests.sha256 b/third_party/v8/rusty_v8_150_4_0_release_manifests.sha256
new file mode 100644
index 0000000000..ec1ee4a92f
--- /dev/null
+++ b/third_party/v8/rusty_v8_150_4_0_release_manifests.sha256
@@ -0,0 +1,8 @@
+4079b4b84a8b4fcf34a2a5ca7f080dffd0e1b53404b0032087b821e247febb43 rusty_v8_ptrcomp_sandbox_release_aarch64-apple-darwin.sha256
+9d153e6534d50961329132a64dd7f7cd18ba96501a4a43c1a6d8bfaeec454b2b rusty_v8_ptrcomp_sandbox_release_aarch64-pc-windows-msvc.sha256
+4ee879a8bc7b0f482cac891415e22300dff4429a28897fddac88bb296ce07920 rusty_v8_ptrcomp_sandbox_release_aarch64-unknown-linux-gnu.sha256
+9c40a51e4d5fcedaec527757b8660115b2a10ca3e2ddacadc3075924ad005b66 rusty_v8_ptrcomp_sandbox_release_aarch64-unknown-linux-musl.sha256
+d85c7ae0cf437a4415376c4b5b7daba50b0dcbe30f52612d21df8ce52eb8ada0 rusty_v8_ptrcomp_sandbox_release_x86_64-apple-darwin.sha256
+a4d6221dddb4b5724b23411eaac47caf6095489fbf9d126f65b33cef96a0a8ef rusty_v8_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.sha256
+6774b42c9424c098c72a805c08d4e94be17c591cf02b1dc2633060255a8a61be rusty_v8_ptrcomp_sandbox_release_x86_64-unknown-linux-gnu.sha256
+9bd5beb3a7bfa4f95bc887476ec3e4d564254c1815efe63296740e09bcc8665b rusty_v8_ptrcomp_sandbox_release_x86_64-unknown-linux-musl.sha256