Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add Bazel preparation for native voice runtimes (#43114)

Benjamin Carlssona947db131b62
diff --git a/.bazelrc b/.bazelrc
index 0aec7b8953..ea06d44080 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -11,6 +11,9 @@ startup --experimental_remote_repo_contents_cache
common --experimental_platform_in_output_dir
+build --workspace_status_command=./scripts/workspace-status.sh
+build:windows --workspace_status_command=./scripts/workspace-status.cmd
+
# Runfiles strategy rationale: codex-rs/utils/cargo-bin/README.md
common --noenable_runfiles
@@ -85,6 +88,7 @@ common --jobs=30
common:remote --strategy=remote
# Native Mac voice actions need the local Apple tools; Linux remains remote.
common:macos --strategy=VoiceNativePrefix=remote,sandboxed,local
+common:macos --strategy=VoiceNativeRuntime=remote,sandboxed,local
common:remote --extra_execution_platforms=//:rbe
common:remote --jobs=800
# TODO(team): Evaluate if this actually helps, zbarsky is not sure, everything seems bottlenecked on `core` either way.
diff --git a/scripts/workspace-status.cmd b/scripts/workspace-status.cmd
new file mode 100644
index 0000000000..6a63292c78
--- /dev/null
+++ b/scripts/workspace-status.cmd
@@ -0,0 +1,20 @@
+@echo off
+setlocal
+
+if defined STABLE_GIT_COMMIT (
+ echo STABLE_GIT_COMMIT %STABLE_GIT_COMMIT%
+ exit /b 0
+)
+
+for /f "delims=" %%I in ('git rev-parse --verify HEAD 2^>nul') do set "BUILD_COMMIT=%%I"
+if defined BUILD_COMMIT (
+ echo STABLE_GIT_COMMIT %BUILD_COMMIT%
+ exit /b 0
+)
+
+if defined GITHUB_SHA (
+ echo STABLE_GIT_COMMIT %GITHUB_SHA%
+ exit /b 0
+)
+
+echo STABLE_GIT_COMMIT unknown
diff --git a/scripts/workspace-status.sh b/scripts/workspace-status.sh
new file mode 100755
index 0000000000..ba4cd3869e
--- /dev/null
+++ b/scripts/workspace-status.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ -n "${STABLE_GIT_COMMIT:-}" ]]; then
+ build_commit="${STABLE_GIT_COMMIT}"
+elif build_commit="$(git rev-parse --verify HEAD 2>/dev/null)"; then
+ :
+elif [[ -n "${GITHUB_SHA:-}" ]]; then
+ build_commit="${GITHUB_SHA}"
+else
+ build_commit="unknown"
+fi
+
+printf 'STABLE_GIT_COMMIT %s\n' "${build_commit}"
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel
index 87ea515b98..6c807508a0 100644
--- a/third_party/voice/BUILD.bazel
+++ b/third_party/voice/BUILD.bazel
@@ -1,6 +1,7 @@
load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain")
load(":native.bzl", "native_prefix")
load(":pkg_config.bzl", "pkg_config")
+load(":runtime.bzl", "native_runtime")
# All native build consumers must receive the same pinned source manifest.
exports_files(["opus-toolchain.cmake"])
@@ -177,3 +178,31 @@ alias(
tags = ["manual"],
visibility = ["//visibility:public"],
)
+
+[
+ native_runtime(
+ name = "native_runtime_" + os + "_" + cpu,
+ exec_compatible_with = [
+ "@platforms//os:" + os,
+ "@platforms//cpu:" + cpu,
+ ],
+ prefix = ":native_prefix_" + os + "_" + cpu,
+ tags = ["manual"],
+ target = cpu + "-" + suffix,
+ target_compatible_with = select({
+ ":" + os + "_" + cpu: [],
+ "//conditions:default": ["@platforms//:incompatible"],
+ }),
+ )
+ for os, cpu, suffix in _NATIVE_PLATFORMS
+]
+
+alias(
+ name = "native_runtime",
+ actual = select({
+ ":" + os + "_" + cpu: ":native_runtime_" + os + "_" + cpu
+ for os, cpu, _ in _NATIVE_PLATFORMS
+ }),
+ tags = ["manual"],
+ visibility = ["//visibility:public"],
+)
diff --git a/third_party/voice/README.md b/third_party/voice/README.md
index a4b7499e58..7488eef434 100644
--- a/third_party/voice/README.md
+++ b/third_party/voice/README.md
@@ -183,6 +183,18 @@ preparation or shipped in the runtime. The output is development-only, uses the
host glibc, and does not establish musl or minimum-glibc support, dynamic-only
dependency closure, helper loading policy or working voice.
+## Prepare Bazel's native build output
+
+`bazel build //third_party/voice:native_runtime` prepares the selected Mac or GNU
+Linux prefix archive using the existing platform inspection and preparation code.
+It checks the completed build receipt, records the current workspace build commit,
+inspects physical libraries, and prepares verified copies. These receipts describe
+build inputs and inspection; they are not signatures or release approval.
+
+This manual target requires the same host inspection/signing tools as the standalone
+platform preparer. It does not link Rust, change Windows builds, assemble a CLI
+package, or enable voice. The prepared runtime is the input to those later steps.
+
## Private Windows runtime preparation
`windows_runtime.py` takes the same arguments for x64/ARM64 MSVC build prefixes.
diff --git a/third_party/voice/macos_runtime.py b/third_party/voice/macos_runtime.py
index 23b674207e..38fdf009df 100644
--- a/third_party/voice/macos_runtime.py
+++ b/third_party/voice/macos_runtime.py
@@ -15,6 +15,7 @@ from runtime import PLUGINS, RuntimeFormat, digest, prepare, required_library_pa
SYSTEM_IMPORTS = frozenset(
{
"/usr/lib/libSystem.B.dylib",
+ "/usr/lib/libc++.1.dylib",
"/usr/lib/libobjc.A.dylib",
"/usr/lib/libiconv.2.dylib",
"/usr/lib/libresolv.9.dylib",
diff --git a/third_party/voice/prepare_built_runtime.py b/third_party/voice/prepare_built_runtime.py
new file mode 100644
index 0000000000..9a070e479b
--- /dev/null
+++ b/third_party/voice/prepare_built_runtime.py
@@ -0,0 +1,113 @@
+"""Prepare an inspected build output using the existing platform runtime policy.
+
+Receipts describe the declared build inputs and inspection, not authenticity or
+approval. The current build commit comes from Bazel's workspace status file.
+"""
+
+import argparse
+import importlib
+import json
+from pathlib import Path
+import re
+import sys
+import tarfile
+import tempfile
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from runtime import digest
+
+
+def prepare_built(prefix, build_receipt, status, target, output):
+ prefix = prefix.resolve(strict=True)
+ if build_receipt.stat().st_size > 1024 * 1024 or status.stat().st_size > 65536:
+ raise ValueError("native build metadata exceeds limits")
+ build = json.loads(build_receipt.read_text())
+ commits = [
+ line.removeprefix("STABLE_GIT_COMMIT ")
+ for line in status.read_text().splitlines()
+ if line.startswith("STABLE_GIT_COMMIT ")
+ ]
+ manifest_hash = digest(Path(__file__).with_name("sources.json"))
+ steps = build.get("steps", [])
+ if (
+ len(commits) != 1
+ or not re.fullmatch(r"[0-9a-f]{40}", commits[0])
+ or build.get("target") != target
+ or build.get("manifest_sha256") != manifest_hash
+ or not 1 <= len(steps) <= 128
+ or any(step.get("exit_code") != 0 for step in steps)
+ or steps[-1].get("name") != "gst-plugins-good-install"
+ ):
+ raise ValueError("native build receipt is incomplete or mismatched")
+ suffix = target.partition("-")[2]
+ modules = {
+ "apple-darwin": "macos_runtime",
+ "unknown-linux-gnu": "linux_runtime",
+ "pc-windows-msvc": "windows_runtime",
+ }
+ if suffix not in modules or target.partition("-")[0] not in ("aarch64", "x86_64"):
+ raise ValueError("unsupported native runtime target")
+ platform = importlib.import_module(modules[suffix])
+ records = []
+ for path in sorted(prefix.rglob("*")):
+ if not re.fullmatch(r".+\.(?:dylib|dll|so(?:\.[0-9]+)*)", path.name):
+ continue
+ if path.is_symlink():
+ if not path.resolve(strict=True).is_relative_to(prefix):
+ raise ValueError("native library link escapes its prefix")
+ continue
+ if not path.is_file():
+ continue
+ if len(records) >= 128:
+ raise ValueError("native inventory exceeds limits")
+ platform.inspect(path, target)
+ records.append(
+ {
+ "path": path.relative_to(prefix).as_posix(),
+ "target": target,
+ "sha256": digest(path),
+ }
+ )
+ if not records:
+ raise ValueError("native prefix contains no libraries")
+ with tempfile.TemporaryDirectory(prefix="voice-receipts-") as temporary:
+ receipts = Path(temporary)
+ (receipts / "inspection").mkdir()
+ (receipts / "inspection/binaries.json").write_text(json.dumps(records))
+ (receipts / "ci.json").write_text(
+ json.dumps(
+ {
+ "commit": commits[0],
+ "target": target,
+ "manifest_sha256": manifest_hash,
+ "build_complete": True,
+ "inspection_complete": True,
+ }
+ )
+ )
+ platform.project(prefix, receipts, target, output)
+
+
+def prepare_archive(archive, build_receipt, status, target, output):
+ # Archives preserve native aliases through Bazel's cache and sandbox links.
+ with tempfile.TemporaryDirectory(prefix="voice-prefix-") as temporary:
+ prefix = Path(temporary)
+ with tarfile.open(archive) as source:
+ source.extractall(prefix, filter="data")
+ prepare_built(prefix, build_receipt, status, target, output)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description=__doc__)
+ for name in ("prefix", "build-receipt", "status", "output"):
+ parser.add_argument("--" + name, type=Path, required=True)
+ parser.add_argument("--target", required=True)
+ args = parser.parse_args()
+ # Executors may leave the TreeArtifact absent or create an empty directory.
+ try:
+ args.output.rmdir()
+ except FileNotFoundError:
+ pass
+ prepare_archive(
+ args.prefix, args.build_receipt, args.status, args.target, args.output
+ )
diff --git a/third_party/voice/runtime.bzl b/third_party/voice/runtime.bzl
new file mode 100644
index 0000000000..3334e3ffd7
--- /dev/null
+++ b/third_party/voice/runtime.bzl
@@ -0,0 +1,46 @@
+"""Prepare build-produced native libraries with the existing platform policy."""
+
+load("@rules_python//python:py_runtime_info.bzl", "PyRuntimeInfo")
+
+def _native_runtime_impl(ctx):
+ python = ctx.attr._python[PyRuntimeInfo]
+ prefix = ctx.attr.prefix[DefaultInfo].files.to_list()[0]
+ receipt = ctx.attr.prefix[OutputGroupInfo].receipt.to_list()[0]
+ output = ctx.actions.declare_directory(ctx.label.name)
+ ctx.actions.run(
+ executable = python.interpreter,
+ arguments = [
+ ctx.file._driver.path,
+ "--prefix",
+ prefix.path,
+ "--build-receipt",
+ receipt.path,
+ "--status",
+ ctx.info_file.path,
+ "--target",
+ ctx.attr.target,
+ "--output",
+ output.path,
+ ],
+ inputs = depset(
+ [prefix, receipt, ctx.info_file, ctx.file._driver, python.interpreter] + ctx.files._preparers,
+ transitive = [python.files],
+ ),
+ outputs = [output],
+ env = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"},
+ execution_requirements = {"no-remote-exec": "1", "no-remote-cache": "1"} if ctx.attr.target.endswith("apple-darwin") else {},
+ mnemonic = "VoiceNativeRuntime",
+ progress_message = "Preparing private voice runtime for " + ctx.attr.target,
+ )
+ return [DefaultInfo(files = depset([output]))]
+
+native_runtime = rule(
+ implementation = _native_runtime_impl,
+ attrs = {
+ "prefix": attr.label(mandatory = True, allow_single_file = True),
+ "target": attr.string(mandatory = True),
+ "_driver": attr.label(default = "//third_party/voice:prepare_built_runtime.py", allow_single_file = True),
+ "_preparers": attr.label(default = "//third_party/voice:build_inputs"),
+ "_python": attr.label(default = "@python_3_12//:py3_runtime", cfg = "exec"),
+ },
+)
diff --git a/third_party/voice/test_prepare_built_runtime.py b/third_party/voice/test_prepare_built_runtime.py
new file mode 100644
index 0000000000..6aa51d0884
--- /dev/null
+++ b/third_party/voice/test_prepare_built_runtime.py
@@ -0,0 +1,181 @@
+"""Exercise the build receipt boundary and real runtime preparation adapter."""
+
+import json
+from pathlib import Path
+import sys
+import subprocess
+import tarfile
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import macos_runtime
+from prepare_built_runtime import prepare_archive, prepare_built
+from runtime import digest
+import test_macos_runtime
+
+
+class BuiltRuntimeTests(unittest.TestCase):
+ def setUp(self):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name)
+ self.prefix = self.root / "prefix"
+ self.prefix.mkdir()
+ self.library = self.prefix / "libfixture.dylib"
+ self.library.write_bytes(b"synthetic inventory fixture")
+ self.target = "aarch64-apple-darwin"
+ self.status = self.root / "status"
+ self.status.write_text("STABLE_GIT_COMMIT " + "a" * 40 + "\n")
+ self.receipt = self.root / "built.json"
+ self.build = {
+ "target": self.target,
+ "manifest_sha256": digest(Path(__file__).with_name("sources.json")),
+ "steps": [{"name": "gst-plugins-good-install", "exit_code": 0}],
+ }
+ self.receipt.write_text(json.dumps(self.build))
+
+ def test_inspection_precedes_truthful_receipts_and_projection(self):
+ def project(prefix, receipts, target, output):
+ inspect.assert_called_once_with(self.library.resolve(), target)
+ self.assertEqual(
+ json.loads((receipts / "ci.json").read_text()),
+ {
+ "commit": "a" * 40,
+ "target": self.target,
+ "manifest_sha256": self.build["manifest_sha256"],
+ "build_complete": True,
+ "inspection_complete": True,
+ },
+ )
+ self.assertEqual(
+ json.loads((receipts / "inspection/binaries.json").read_text()),
+ [
+ {
+ "path": self.library.name,
+ "target": self.target,
+ "sha256": digest(self.library),
+ }
+ ],
+ )
+
+ with (
+ patch.object(macos_runtime, "inspect") as inspect,
+ patch.object(macos_runtime, "project", side_effect=project),
+ ):
+ prepare_built(
+ self.prefix, self.receipt, self.status, self.target, self.root / "out"
+ )
+
+ def test_rejects_failed_incomplete_and_mismatched_builds_before_inspection(self):
+ for change in (
+ {"target": "x86_64-apple-darwin"},
+ {"manifest_sha256": "0" * 64},
+ {"steps": []},
+ {"steps": [{"name": "glib-install", "exit_code": 0}]},
+ {"steps": [{"name": "gst-plugins-good-install", "exit_code": 1}]},
+ ):
+ with (
+ self.subTest(change=change),
+ patch.object(macos_runtime, "inspect") as inspect,
+ ):
+ self.receipt.write_text(json.dumps({**self.build, **change}))
+ with self.assertRaises(ValueError):
+ prepare_built(
+ self.prefix,
+ self.receipt,
+ self.status,
+ self.target,
+ self.root / "out",
+ )
+ inspect.assert_not_called()
+
+ def test_rejects_unknown_or_ambiguous_build_commit(self):
+ for text in ("STABLE_GIT_COMMIT unknown\n", self.status.read_text() * 2):
+ self.status.write_text(text)
+ with self.assertRaises(ValueError):
+ prepare_built(
+ self.prefix,
+ self.receipt,
+ self.status,
+ self.target,
+ self.root / "out",
+ )
+
+ def test_inspection_failure_never_reaches_projection(self):
+ with (
+ patch.object(
+ macos_runtime, "inspect", side_effect=ValueError("bad library")
+ ),
+ patch.object(macos_runtime, "project") as project,
+ ):
+ with self.assertRaisesRegex(ValueError, "bad library"):
+ prepare_built(
+ self.prefix,
+ self.receipt,
+ self.status,
+ self.target,
+ self.root / "out",
+ )
+ project.assert_not_called()
+
+ @unittest.skipUnless(sys.platform == "darwin", "Uses real Mach-O fixture libraries")
+ def test_projects_archive_through_sandbox_link_with_native_aliases(self):
+ fixture = test_macos_runtime.RuntimeTests("runTest")
+ self.addCleanup(fixture.doCleanups)
+ fixture.setUp()
+ self.receipt.write_text(json.dumps({**self.build, "target": fixture.target}))
+ library = next((fixture.prefix / "lib").glob("*.dylib"))
+ (library.parent / "development-alias.dylib").symlink_to(library.name)
+ archive = self.root / "prefix.tar"
+ with tarfile.open(archive, "w") as source:
+ source.add(fixture.prefix, arcname=".")
+ sandbox_input = self.root / "sandbox-input.tar"
+ sandbox_input.symlink_to(archive)
+ for state in ("absent", "empty"):
+ with self.subTest(output_state=state):
+ output = self.root / f"runtime-{state}"
+ if state == "empty":
+ output.mkdir()
+ subprocess.run(
+ [
+ sys.executable,
+ str(Path(__file__).with_name("prepare_built_runtime.py")),
+ "--prefix",
+ str(sandbox_input),
+ "--build-receipt",
+ str(self.receipt),
+ "--status",
+ str(self.status),
+ "--target",
+ fixture.target,
+ "--output",
+ str(output),
+ ],
+ check=True,
+ )
+ manifest = json.loads((output / "runtime.json").read_text())
+ self.assertEqual(manifest["sourceCommit"], "a" * 40)
+ self.assertEqual(manifest["target"], fixture.target)
+ self.assertTrue(
+ all(
+ not macos_runtime.inspect(
+ output / record["path"], fixture.target
+ ).rpaths
+ for record in manifest["libraries"]
+ )
+ )
+
+ def test_archive_rejects_escaping_native_alias_before_inspection(self):
+ archive = self.root / "prefix.tar"
+ with tarfile.open(archive, "w") as source:
+ link = tarfile.TarInfo("lib/escape.dylib")
+ link.type = tarfile.SYMTYPE
+ link.linkname = "../../outside.dylib"
+ source.addfile(link)
+ with patch.object(macos_runtime, "inspect") as inspect:
+ with self.assertRaises(tarfile.LinkOutsideDestinationError):
+ prepare_archive(
+ archive, self.receipt, self.status, self.target, self.root / "out"
+ )
+ inspect.assert_not_called()