Lightweight coding agent that runs in your terminal
Rust
Pin V8 release manifests and prevent published release replacement (#43444)
Charlie Marshc0b6285711f7
diff --git a/.github/actions/setup-rusty-v8/action.yml b/.github/actions/setup-rusty-v8/action.ymlindex 449ab653fd..7b78d8cbdb 100644--- a/.github/actions/setup-rusty-v8/action.yml+++ b/.github/actions/setup-rusty-v8/action.yml@@ -33,11 +33,25 @@ runs:archive_path="${binding_dir}/${archive_name}"binding_path="${binding_dir}/${binding_name}"checksums_path="${binding_dir}/${checksums_name}"+ trusted_checksums="${GITHUB_WORKSPACE}/third_party/v8/rusty_v8_${version//./_}_release_manifests.sha256"++ if command -v sha256sum >/dev/null 2>&1; then+ checksum_command=(sha256sum --check -)+ else+ checksum_command=(shasum -a 256 --check -)+ fimkdir -p "${binding_dir}"+ curl -fsSL "${base_url}/${checksums_name}" -o "${checksums_path}"+ # Check the original manifest bytes even when either checksum file uses CRLF.+ expected_manifest_checksum="$(grep -F " ${checksums_name}" "${trusted_checksums}" | cut -d ' ' -f 1)"+ actual_manifest_checksum="$(python3 -c 'import hashlib, pathlib, sys; print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())' "${checksums_path}")"+ if [[ "${actual_manifest_checksum}" != "${expected_manifest_checksum}" ]]; then+ echo "Checksum mismatch for ${checksums_name}: expected ${expected_manifest_checksum}, got ${actual_manifest_checksum}" >&2+ exit 1+ ficurl -fsSL "${base_url}/${archive_name}" -o "${archive_path}"curl -fsSL "${base_url}/${binding_name}" -o "${binding_path}"- curl -fsSL "${base_url}/${checksums_name}" -o "${checksums_path}"if [[ "$(wc -l < "${checksums_path}")" -ne 2 ]]; thenecho "Expected exactly two checksums for ${TARGET} in ${checksums_path}" >&2@@ -45,10 +59,6 @@ runs:fi# Existing Windows-built release manifests use CRLF line endings.- if command -v sha256sum >/dev/null 2>&1; then- (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | sha256sum -c -)- else- (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | shasum -a 256 -c -)- fi+ (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | "${checksum_command[@]}")echo "RUSTY_V8_ARCHIVE=${archive_path}" >> "${GITHUB_ENV}"echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "${GITHUB_ENV}"diff --git a/.github/workflows/rusty-v8-release.yml b/.github/workflows/rusty-v8-release.ymlindex 37c8c65370..32ec3af836 100644--- a/.github/workflows/rusty-v8-release.yml+++ b/.github/workflows/rusty-v8-release.yml@@ -354,22 +354,18 @@ jobs:restore-keys: |rusty-v8-source-${{ matrix.target }}-sandbox-- - name: Install and start sccache+ - name: Install sccache+ uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49+ with:+ tool: [email protected]++ - name: Start sccacheshell: pwshenv:SCCACHE_CACHE_SIZE: 256MSCCACHE_DIR: ${{ github.workspace }}/upstream-rusty-v8/target/sccacheSCCACHE_IDLE_TIMEOUT: 0- run: |- $version = "v0.8.2"- $platform = "x86_64-pc-windows-msvc"- $basename = "sccache-$version-$platform"- $url = "https://github.com/mozilla/sccache/releases/download/$version/$basename.tar.gz"- cd ~- curl -LO $url- tar -xzvf "$basename.tar.gz"- . $basename/sccache --start-server- echo "$(pwd)/$basename" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append+ run: sccache --start-server- name: Install Chromium clang for ARM64 MSVC cross buildif: matrix.target == 'aarch64-pc-windows-msvc'@@ -437,7 +433,6 @@ jobs:steps:- name: Check whether release already exists- id: releaseenv:GH_TOKEN: ${{ github.token }}RELEASE_TAG: ${{ needs.metadata.outputs.release_tag }}@@ -445,10 +440,14 @@ jobs:run: |set -euo pipefail- if gh release view "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" > /dev/null 2>&1; then- echo "exists=true" >> "${GITHUB_OUTPUT}"- else- echo "exists=false" >> "${GITHUB_OUTPUT}"+ if is_draft="$(gh release view "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq '.isDraft' 2>/dev/null)"; then+ if [[ "${is_draft}" != "true" ]]; then+ echo "Refusing to replace existing published release ${RELEASE_TAG}." >&2+ exit 1+ fi++ echo "Removing unfinished draft release for ${RELEASE_TAG}."+ gh release delete "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" --yesfi- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1@@ -456,22 +455,25 @@ jobs:path: dist- name: Create GitHub Release- if: ${{ steps.release.outputs.exists != 'true' }}- uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1- with:- tag_name: ${{ needs.metadata.outputs.release_tag }}- name: ${{ needs.metadata.outputs.release_tag }}- files: dist/**- # Keep V8 artifact releases out of Codex's normal "latest release" channel.- prerelease: true-- - name: Amend existing GitHub Release- if: ${{ steps.release.outputs.exists == 'true' }}- uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1- with:- tag_name: ${{ needs.metadata.outputs.release_tag }}- name: ${{ needs.metadata.outputs.release_tag }}- files: dist/**- overwrite_files: true- # Keep V8 artifact releases out of Codex's normal "latest release" channel.- prerelease: true+ env:+ GH_TOKEN: ${{ github.token }}+ RELEASE_TAG: ${{ needs.metadata.outputs.release_tag }}+ shell: bash+ run: |+ set -euo pipefail+ assets=()+ while IFS= read -r -d '' asset; do+ assets+=("${asset}")+ done < <(find dist -type f -print0)+ if [[ "${#assets[@]}" -eq 0 ]]; then+ echo "No V8 release artifacts were found." >&2+ exit 1+ fi++ # GitHub CLI keeps asset-bearing releases in draft until all uploads finish.+ gh release create "${RELEASE_TAG}" \+ --repo "${GITHUB_REPOSITORY}" \+ --title "${RELEASE_TAG}" \+ --verify-tag \+ --prerelease \+ "${assets[@]}"diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.ymlindex 29cb72e7c5..94f1aa0199 100644--- a/.github/workflows/v8-canary.yml+++ b/.github/workflows/v8-canary.yml@@ -372,22 +372,18 @@ jobs:restore-keys: |rusty-v8-source-${{ matrix.target }}-${{ matrix.runner }}-sandbox-- - name: Install and start sccache+ - name: Install sccache+ uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49+ with:+ tool: [email protected]++ - name: Start sccacheshell: pwshenv:SCCACHE_CACHE_SIZE: 256MSCCACHE_DIR: ${{ steps.setup_ci.outputs.cargo-target-dir }}/sccacheSCCACHE_IDLE_TIMEOUT: 0- run: |- $version = "v0.8.2"- $platform = "x86_64-pc-windows-msvc"- $basename = "sccache-$version-$platform"- $url = "https://github.com/mozilla/sccache/releases/download/$version/$basename.tar.gz"- cd ~- curl -LO $url- tar -xzvf "$basename.tar.gz"- . $basename/sccache --start-server- echo "$(pwd)/$basename" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append+ run: sccache --start-server- name: Install Chromium clang for ARM64 MSVC cross buildif: matrix.target == 'aarch64-pc-windows-msvc'diff --git a/scripts/codex_package/test_v8.py b/scripts/codex_package/test_v8.pynew file mode 100644index 0000000000..a47f15a348--- /dev/null+++ b/scripts/codex_package/test_v8.py@@ -0,0 +1,152 @@+import hashlib+import sys+import tempfile+import unittest+from collections.abc import Iterator+from contextlib import contextmanager+from pathlib import Path+from unittest.mock import MagicMock, patch++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from codex_package import v8+from codex_package.targets import TARGET_SPECS, TargetSpec+++class FetchCodexV8ArtifactsTest(unittest.TestCase):+ version = "150.4.0"++ def setUp(self) -> None:+ temporary = tempfile.TemporaryDirectory()+ self.addCleanup(temporary.cleanup)+ self.root = Path(temporary.name)++ @contextmanager+ def release(+ self,+ target: str,+ *,+ line_ending: bytes = b"\n",+ trusted_digest: str | None = None,+ trusted_name: str | None = None,+ create_pins: bool = True,+ ) -> Iterator[tuple[TargetSpec, MagicMock, str]]:+ spec = TARGET_SPECS[target]+ profile = v8.V8_ARTIFACT_PROFILE+ archive_name = (+ f"rusty_v8_{profile}_{target}.lib.gz"+ if spec.is_windows+ else f"librusty_v8_{profile}_{target}.a.gz"+ )+ binding_name = f"src_binding_{profile}_{target}.rs"+ manifest_name = f"rusty_v8_{profile}_{target}.sha256"+ archive = b"trusted V8 archive"+ binding = b"trusted V8 binding"+ manifest = (+ line_ending.join(+ (+ f"{hashlib.sha256(archive).hexdigest()} {archive_name}".encode(),+ f"{hashlib.sha256(binding).hexdigest()} {binding_name}".encode(),+ )+ )+ + line_ending+ )+ payloads = {+ manifest_name: manifest,+ archive_name: archive,+ binding_name: binding,+ }++ if create_pins:+ pins = (+ self.root / "third_party/v8/rusty_v8_150_4_0_release_manifests.sha256"+ )+ pins.parent.mkdir(parents=True)+ digest = trusted_digest or hashlib.sha256(manifest).hexdigest()+ name = trusted_name or manifest_name+ pins.write_bytes(f"{digest} {name}".encode() + line_ending)++ def download(_url: str, destination: Path) -> None:+ destination.parent.mkdir(parents=True, exist_ok=True)+ destination.write_bytes(payloads[destination.name])++ with (+ patch.object(v8, "REPO_ROOT", self.root),+ patch.object(v8, "download_file", side_effect=download) as download_file,+ ):+ yield spec, download_file, manifest_name++ def test_fetches_artifacts_after_authenticating_manifest(self) -> None:+ with self.release("x86_64-unknown-linux-gnu") as (+ spec,+ download,+ manifest_name,+ ):+ artifacts = v8.fetch_codex_v8_artifacts(+ spec, version=self.version, cache_root=self.root / "cache"+ )++ self.assertEqual(artifacts.archive.read_bytes(), b"trusted V8 archive")+ self.assertEqual(artifacts.binding.read_bytes(), b"trusted V8 binding")+ self.assertEqual(download.call_args_list[0].args[1].name, manifest_name)+ self.assertEqual(download.call_count, 3)++ def test_authenticates_windows_manifest_with_crlf(self) -> None:+ with self.release("x86_64-pc-windows-msvc", line_ending=b"\r\n") as (+ spec,+ download,+ _manifest_name,+ ):+ artifacts = v8.fetch_codex_v8_artifacts(+ spec, version=self.version, cache_root=self.root / "cache"+ )++ self.assertEqual(artifacts.archive.read_bytes(), b"trusted V8 archive")+ self.assertEqual(artifacts.binding.read_bytes(), b"trusted V8 binding")+ self.assertEqual(download.call_count, 3)++ def test_rejects_tampered_manifest_before_downloading_artifacts(self) -> None:+ with self.release("x86_64-unknown-linux-gnu", trusted_digest="0" * 64) as (+ spec,+ download,+ manifest_name,+ ):+ with self.assertRaisesRegex(+ RuntimeError, "does not match its trusted SHA-256"+ ):+ v8.fetch_codex_v8_artifacts(+ spec, version=self.version, cache_root=self.root / "cache"+ )++ download.assert_called_once()+ self.assertEqual(download.call_args.args[1].name, manifest_name)++ def test_rejects_missing_manifest_pin_before_downloading_artifacts(self) -> None:+ with self.release(+ "x86_64-unknown-linux-gnu", trusted_name="another-target.sha256"+ ) as (spec, download, manifest_name):+ with self.assertRaisesRegex(RuntimeError, "has no trusted SHA-256"):+ v8.fetch_codex_v8_artifacts(+ spec, version=self.version, cache_root=self.root / "cache"+ )++ download.assert_called_once()+ self.assertEqual(download.call_args.args[1].name, manifest_name)++ def test_rejects_missing_pin_file_before_downloading_artifacts(self) -> None:+ with self.release("x86_64-unknown-linux-gnu", create_pins=False) as (+ spec,+ download,+ manifest_name,+ ):+ with self.assertRaises(FileNotFoundError):+ v8.fetch_codex_v8_artifacts(+ spec, version=self.version, cache_root=self.root / "cache"+ )++ download.assert_called_once()+ self.assertEqual(download.call_args.args[1].name, manifest_name)+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/codex_package/v8.py b/scripts/codex_package/v8.pyindex 66ecb6351e..f2ac15dd09 100644--- a/scripts/codex_package/v8.py+++ b/scripts/codex_package/v8.py@@ -74,6 +74,7 @@ def fetch_codex_v8_artifacts(checksums = cache_dir / checksums_namedownload_file(f"{release_url}/{checksums.name}", checksums)+ verify_release_checksum_manifest(checksums, version=version)expected_checksums = load_checksums(checksums, {archive.name, binding.name})for artifact in [archive, binding]:ensure_valid_artifact(@@ -107,6 +108,32 @@ def default_cache_root() -> Path:return Path(tempfile.gettempdir()) / "codex-package"+def verify_release_checksum_manifest(checksums_path: Path, *, version: str) -> None:+ version_suffix = version.replace(".", "_")+ trusted_checksums = (+ REPO_ROOT+ / "third_party"+ / "v8"+ / f"rusty_v8_{version_suffix}_release_manifests.sha256"+ )++ for line in trusted_checksums.read_text(encoding="utf-8").splitlines():+ digest, artifact_name = line.split(maxsplit=1)+ if artifact_name != checksums_path.name:+ continue+ if has_checksum(checksums_path, digest):+ return++ checksums_path.unlink(missing_ok=True)+ raise RuntimeError(+ f"V8 checksum manifest {checksums_path} does not match its trusted SHA-256."+ )++ raise RuntimeError(+ f"V8 checksum manifest {checksums_path.name} has no trusted SHA-256 for {version}."+ )++def load_checksums(checksums_path: Path, artifact_names: set[str]) -> dict[str, str]:checksums: dict[str, str] = {}lines = checksums_path.read_text(encoding="utf-8").splitlines()diff --git a/third_party/v8/README.md b/third_party/v8/README.mdindex 21adf3899d..fc9879b80d 100644--- a/third_party/v8/README.md+++ b/third_party/v8/README.md@@ -30,7 +30,10 @@ Use this as the maintainer flow for a version bump:matching checksum manifest and generated checksums as described below.3. Publish a release-candidate PR and validate that `v8-canary` passes.4. If the canary is green, publish the release tag and release build.-5. Once the release build completes, rerun the build on the candidate branch+5. Independently verify the published Codex-built checksum manifests and record+ their SHA-256 digests in+ `third_party/v8/rusty_v8_<version>_release_manifests.sha256`.+6. Once the release build completes, rerun the build on the candidate branchand verify that the final artifact builds and tests pass.When changing the remaining prebuilt `rusty_v8` `http_file` inputs, keep thediff --git a/third_party/v8/rusty_v8_150_4_0_release_manifests.sha256 b/third_party/v8/rusty_v8_150_4_0_release_manifests.sha256new file mode 100644index 0000000000..ec1ee4a92f--- /dev/null+++ b/third_party/v8/rusty_v8_150_4_0_release_manifests.sha256@@ -0,0 +1,8 @@+4079b4b84a8b4fcf34a2a5ca7f080dffd0e1b53404b0032087b821e247febb43 rusty_v8_ptrcomp_sandbox_release_aarch64-apple-darwin.sha256+9d153e6534d50961329132a64dd7f7cd18ba96501a4a43c1a6d8bfaeec454b2b rusty_v8_ptrcomp_sandbox_release_aarch64-pc-windows-msvc.sha256+4ee879a8bc7b0f482cac891415e22300dff4429a28897fddac88bb296ce07920 rusty_v8_ptrcomp_sandbox_release_aarch64-unknown-linux-gnu.sha256+9c40a51e4d5fcedaec527757b8660115b2a10ca3e2ddacadc3075924ad005b66 rusty_v8_ptrcomp_sandbox_release_aarch64-unknown-linux-musl.sha256+d85c7ae0cf437a4415376c4b5b7daba50b0dcbe30f52612d21df8ce52eb8ada0 rusty_v8_ptrcomp_sandbox_release_x86_64-apple-darwin.sha256+a4d6221dddb4b5724b23411eaac47caf6095489fbf9d126f65b33cef96a0a8ef rusty_v8_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.sha256+6774b42c9424c098c72a805c08d4e94be17c591cf02b1dc2633060255a8a61be rusty_v8_ptrcomp_sandbox_release_x86_64-unknown-linux-gnu.sha256+9bd5beb3a7bfa4f95bc887476ec3e4d564254c1815efe63296740e09bcc8665b rusty_v8_ptrcomp_sandbox_release_x86_64-unknown-linux-musl.sha256