Lightweight coding agent that runs in your terminal
Rust
Add Bazel preparation for native voice runtimes (#43114)
Benjamin Carlssona947db131b62
diff --git a/.bazelrc b/.bazelrcindex 0aec7b8953..ea06d44080 100644--- a/.bazelrc+++ b/.bazelrc@@ -11,6 +11,9 @@ startup --experimental_remote_repo_contents_cachecommon --experimental_platform_in_output_dir+build --workspace_status_command=./scripts/workspace-status.sh+build:windows --workspace_status_command=./scripts/workspace-status.cmd+# Runfiles strategy rationale: codex-rs/utils/cargo-bin/README.mdcommon --noenable_runfiles@@ -85,6 +88,7 @@ common --jobs=30common:remote --strategy=remote# Native Mac voice actions need the local Apple tools; Linux remains remote.common:macos --strategy=VoiceNativePrefix=remote,sandboxed,local+common:macos --strategy=VoiceNativeRuntime=remote,sandboxed,localcommon:remote --extra_execution_platforms=//:rbecommon:remote --jobs=800# TODO(team): Evaluate if this actually helps, zbarsky is not sure, everything seems bottlenecked on `core` either way.diff --git a/scripts/workspace-status.cmd b/scripts/workspace-status.cmdnew file mode 100644index 0000000000..6a63292c78--- /dev/null+++ b/scripts/workspace-status.cmd@@ -0,0 +1,20 @@+@echo off+setlocal++if defined STABLE_GIT_COMMIT (+ echo STABLE_GIT_COMMIT %STABLE_GIT_COMMIT%+ exit /b 0+)++for /f "delims=" %%I in ('git rev-parse --verify HEAD 2^>nul') do set "BUILD_COMMIT=%%I"+if defined BUILD_COMMIT (+ echo STABLE_GIT_COMMIT %BUILD_COMMIT%+ exit /b 0+)++if defined GITHUB_SHA (+ echo STABLE_GIT_COMMIT %GITHUB_SHA%+ exit /b 0+)++echo STABLE_GIT_COMMIT unknowndiff --git a/scripts/workspace-status.sh b/scripts/workspace-status.shnew file mode 100755index 0000000000..ba4cd3869e--- /dev/null+++ b/scripts/workspace-status.sh@@ -0,0 +1,15 @@+#!/usr/bin/env bash++set -euo pipefail++if [[ -n "${STABLE_GIT_COMMIT:-}" ]]; then+ build_commit="${STABLE_GIT_COMMIT}"+elif build_commit="$(git rev-parse --verify HEAD 2>/dev/null)"; then+ :+elif [[ -n "${GITHUB_SHA:-}" ]]; then+ build_commit="${GITHUB_SHA}"+else+ build_commit="unknown"+fi++printf 'STABLE_GIT_COMMIT %s\n' "${build_commit}"diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazelindex 87ea515b98..6c807508a0 100644--- a/third_party/voice/BUILD.bazel+++ b/third_party/voice/BUILD.bazel@@ -1,6 +1,7 @@load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain")load(":native.bzl", "native_prefix")load(":pkg_config.bzl", "pkg_config")+load(":runtime.bzl", "native_runtime")# All native build consumers must receive the same pinned source manifest.exports_files(["opus-toolchain.cmake"])@@ -177,3 +178,31 @@ alias(tags = ["manual"],visibility = ["//visibility:public"],)++[+ native_runtime(+ name = "native_runtime_" + os + "_" + cpu,+ exec_compatible_with = [+ "@platforms//os:" + os,+ "@platforms//cpu:" + cpu,+ ],+ prefix = ":native_prefix_" + os + "_" + cpu,+ tags = ["manual"],+ target = cpu + "-" + suffix,+ target_compatible_with = select({+ ":" + os + "_" + cpu: [],+ "//conditions:default": ["@platforms//:incompatible"],+ }),+ )+ for os, cpu, suffix in _NATIVE_PLATFORMS+]++alias(+ name = "native_runtime",+ actual = select({+ ":" + os + "_" + cpu: ":native_runtime_" + os + "_" + cpu+ for os, cpu, _ in _NATIVE_PLATFORMS+ }),+ tags = ["manual"],+ visibility = ["//visibility:public"],+)diff --git a/third_party/voice/README.md b/third_party/voice/README.mdindex a4b7499e58..7488eef434 100644--- a/third_party/voice/README.md+++ b/third_party/voice/README.md@@ -183,6 +183,18 @@ preparation or shipped in the runtime. The output is development-only, uses thehost glibc, and does not establish musl or minimum-glibc support, dynamic-onlydependency closure, helper loading policy or working voice.+## Prepare Bazel's native build output++`bazel build //third_party/voice:native_runtime` prepares the selected Mac or GNU+Linux prefix archive using the existing platform inspection and preparation code.+It checks the completed build receipt, records the current workspace build commit,+inspects physical libraries, and prepares verified copies. These receipts describe+build inputs and inspection; they are not signatures or release approval.++This manual target requires the same host inspection/signing tools as the standalone+platform preparer. It does not link Rust, change Windows builds, assemble a CLI+package, or enable voice. The prepared runtime is the input to those later steps.+## Private Windows runtime preparation`windows_runtime.py` takes the same arguments for x64/ARM64 MSVC build prefixes.diff --git a/third_party/voice/macos_runtime.py b/third_party/voice/macos_runtime.pyindex 23b674207e..38fdf009df 100644--- a/third_party/voice/macos_runtime.py+++ b/third_party/voice/macos_runtime.py@@ -15,6 +15,7 @@ from runtime import PLUGINS, RuntimeFormat, digest, prepare, required_library_paSYSTEM_IMPORTS = frozenset({"/usr/lib/libSystem.B.dylib",+ "/usr/lib/libc++.1.dylib","/usr/lib/libobjc.A.dylib","/usr/lib/libiconv.2.dylib","/usr/lib/libresolv.9.dylib",diff --git a/third_party/voice/prepare_built_runtime.py b/third_party/voice/prepare_built_runtime.pynew file mode 100644index 0000000000..9a070e479b--- /dev/null+++ b/third_party/voice/prepare_built_runtime.py@@ -0,0 +1,113 @@+"""Prepare an inspected build output using the existing platform runtime policy.++Receipts describe the declared build inputs and inspection, not authenticity or+approval. The current build commit comes from Bazel's workspace status file.+"""++import argparse+import importlib+import json+from pathlib import Path+import re+import sys+import tarfile+import tempfile++sys.path.insert(0, str(Path(__file__).resolve().parent))+from runtime import digest+++def prepare_built(prefix, build_receipt, status, target, output):+ prefix = prefix.resolve(strict=True)+ if build_receipt.stat().st_size > 1024 * 1024 or status.stat().st_size > 65536:+ raise ValueError("native build metadata exceeds limits")+ build = json.loads(build_receipt.read_text())+ commits = [+ line.removeprefix("STABLE_GIT_COMMIT ")+ for line in status.read_text().splitlines()+ if line.startswith("STABLE_GIT_COMMIT ")+ ]+ manifest_hash = digest(Path(__file__).with_name("sources.json"))+ steps = build.get("steps", [])+ if (+ len(commits) != 1+ or not re.fullmatch(r"[0-9a-f]{40}", commits[0])+ or build.get("target") != target+ or build.get("manifest_sha256") != manifest_hash+ or not 1 <= len(steps) <= 128+ or any(step.get("exit_code") != 0 for step in steps)+ or steps[-1].get("name") != "gst-plugins-good-install"+ ):+ raise ValueError("native build receipt is incomplete or mismatched")+ suffix = target.partition("-")[2]+ modules = {+ "apple-darwin": "macos_runtime",+ "unknown-linux-gnu": "linux_runtime",+ "pc-windows-msvc": "windows_runtime",+ }+ if suffix not in modules or target.partition("-")[0] not in ("aarch64", "x86_64"):+ raise ValueError("unsupported native runtime target")+ platform = importlib.import_module(modules[suffix])+ records = []+ for path in sorted(prefix.rglob("*")):+ if not re.fullmatch(r".+\.(?:dylib|dll|so(?:\.[0-9]+)*)", path.name):+ continue+ if path.is_symlink():+ if not path.resolve(strict=True).is_relative_to(prefix):+ raise ValueError("native library link escapes its prefix")+ continue+ if not path.is_file():+ continue+ if len(records) >= 128:+ raise ValueError("native inventory exceeds limits")+ platform.inspect(path, target)+ records.append(+ {+ "path": path.relative_to(prefix).as_posix(),+ "target": target,+ "sha256": digest(path),+ }+ )+ if not records:+ raise ValueError("native prefix contains no libraries")+ with tempfile.TemporaryDirectory(prefix="voice-receipts-") as temporary:+ receipts = Path(temporary)+ (receipts / "inspection").mkdir()+ (receipts / "inspection/binaries.json").write_text(json.dumps(records))+ (receipts / "ci.json").write_text(+ json.dumps(+ {+ "commit": commits[0],+ "target": target,+ "manifest_sha256": manifest_hash,+ "build_complete": True,+ "inspection_complete": True,+ }+ )+ )+ platform.project(prefix, receipts, target, output)+++def prepare_archive(archive, build_receipt, status, target, output):+ # Archives preserve native aliases through Bazel's cache and sandbox links.+ with tempfile.TemporaryDirectory(prefix="voice-prefix-") as temporary:+ prefix = Path(temporary)+ with tarfile.open(archive) as source:+ source.extractall(prefix, filter="data")+ prepare_built(prefix, build_receipt, status, target, output)+++if __name__ == "__main__":+ parser = argparse.ArgumentParser(description=__doc__)+ for name in ("prefix", "build-receipt", "status", "output"):+ parser.add_argument("--" + name, type=Path, required=True)+ parser.add_argument("--target", required=True)+ args = parser.parse_args()+ # Executors may leave the TreeArtifact absent or create an empty directory.+ try:+ args.output.rmdir()+ except FileNotFoundError:+ pass+ prepare_archive(+ args.prefix, args.build_receipt, args.status, args.target, args.output+ )diff --git a/third_party/voice/runtime.bzl b/third_party/voice/runtime.bzlnew file mode 100644index 0000000000..3334e3ffd7--- /dev/null+++ b/third_party/voice/runtime.bzl@@ -0,0 +1,46 @@+"""Prepare build-produced native libraries with the existing platform policy."""++load("@rules_python//python:py_runtime_info.bzl", "PyRuntimeInfo")++def _native_runtime_impl(ctx):+ python = ctx.attr._python[PyRuntimeInfo]+ prefix = ctx.attr.prefix[DefaultInfo].files.to_list()[0]+ receipt = ctx.attr.prefix[OutputGroupInfo].receipt.to_list()[0]+ output = ctx.actions.declare_directory(ctx.label.name)+ ctx.actions.run(+ executable = python.interpreter,+ arguments = [+ ctx.file._driver.path,+ "--prefix",+ prefix.path,+ "--build-receipt",+ receipt.path,+ "--status",+ ctx.info_file.path,+ "--target",+ ctx.attr.target,+ "--output",+ output.path,+ ],+ inputs = depset(+ [prefix, receipt, ctx.info_file, ctx.file._driver, python.interpreter] + ctx.files._preparers,+ transitive = [python.files],+ ),+ outputs = [output],+ env = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"},+ execution_requirements = {"no-remote-exec": "1", "no-remote-cache": "1"} if ctx.attr.target.endswith("apple-darwin") else {},+ mnemonic = "VoiceNativeRuntime",+ progress_message = "Preparing private voice runtime for " + ctx.attr.target,+ )+ return [DefaultInfo(files = depset([output]))]++native_runtime = rule(+ implementation = _native_runtime_impl,+ attrs = {+ "prefix": attr.label(mandatory = True, allow_single_file = True),+ "target": attr.string(mandatory = True),+ "_driver": attr.label(default = "//third_party/voice:prepare_built_runtime.py", allow_single_file = True),+ "_preparers": attr.label(default = "//third_party/voice:build_inputs"),+ "_python": attr.label(default = "@python_3_12//:py3_runtime", cfg = "exec"),+ },+)diff --git a/third_party/voice/test_prepare_built_runtime.py b/third_party/voice/test_prepare_built_runtime.pynew file mode 100644index 0000000000..6aa51d0884--- /dev/null+++ b/third_party/voice/test_prepare_built_runtime.py@@ -0,0 +1,181 @@+"""Exercise the build receipt boundary and real runtime preparation adapter."""++import json+from pathlib import Path+import sys+import subprocess+import tarfile+import tempfile+import unittest+from unittest.mock import patch++import macos_runtime+from prepare_built_runtime import prepare_archive, prepare_built+from runtime import digest+import test_macos_runtime+++class BuiltRuntimeTests(unittest.TestCase):+ def setUp(self):+ temporary = tempfile.TemporaryDirectory()+ self.addCleanup(temporary.cleanup)+ self.root = Path(temporary.name)+ self.prefix = self.root / "prefix"+ self.prefix.mkdir()+ self.library = self.prefix / "libfixture.dylib"+ self.library.write_bytes(b"synthetic inventory fixture")+ self.target = "aarch64-apple-darwin"+ self.status = self.root / "status"+ self.status.write_text("STABLE_GIT_COMMIT " + "a" * 40 + "\n")+ self.receipt = self.root / "built.json"+ self.build = {+ "target": self.target,+ "manifest_sha256": digest(Path(__file__).with_name("sources.json")),+ "steps": [{"name": "gst-plugins-good-install", "exit_code": 0}],+ }+ self.receipt.write_text(json.dumps(self.build))++ def test_inspection_precedes_truthful_receipts_and_projection(self):+ def project(prefix, receipts, target, output):+ inspect.assert_called_once_with(self.library.resolve(), target)+ self.assertEqual(+ json.loads((receipts / "ci.json").read_text()),+ {+ "commit": "a" * 40,+ "target": self.target,+ "manifest_sha256": self.build["manifest_sha256"],+ "build_complete": True,+ "inspection_complete": True,+ },+ )+ self.assertEqual(+ json.loads((receipts / "inspection/binaries.json").read_text()),+ [+ {+ "path": self.library.name,+ "target": self.target,+ "sha256": digest(self.library),+ }+ ],+ )++ with (+ patch.object(macos_runtime, "inspect") as inspect,+ patch.object(macos_runtime, "project", side_effect=project),+ ):+ prepare_built(+ self.prefix, self.receipt, self.status, self.target, self.root / "out"+ )++ def test_rejects_failed_incomplete_and_mismatched_builds_before_inspection(self):+ for change in (+ {"target": "x86_64-apple-darwin"},+ {"manifest_sha256": "0" * 64},+ {"steps": []},+ {"steps": [{"name": "glib-install", "exit_code": 0}]},+ {"steps": [{"name": "gst-plugins-good-install", "exit_code": 1}]},+ ):+ with (+ self.subTest(change=change),+ patch.object(macos_runtime, "inspect") as inspect,+ ):+ self.receipt.write_text(json.dumps({**self.build, **change}))+ with self.assertRaises(ValueError):+ prepare_built(+ self.prefix,+ self.receipt,+ self.status,+ self.target,+ self.root / "out",+ )+ inspect.assert_not_called()++ def test_rejects_unknown_or_ambiguous_build_commit(self):+ for text in ("STABLE_GIT_COMMIT unknown\n", self.status.read_text() * 2):+ self.status.write_text(text)+ with self.assertRaises(ValueError):+ prepare_built(+ self.prefix,+ self.receipt,+ self.status,+ self.target,+ self.root / "out",+ )++ def test_inspection_failure_never_reaches_projection(self):+ with (+ patch.object(+ macos_runtime, "inspect", side_effect=ValueError("bad library")+ ),+ patch.object(macos_runtime, "project") as project,+ ):+ with self.assertRaisesRegex(ValueError, "bad library"):+ prepare_built(+ self.prefix,+ self.receipt,+ self.status,+ self.target,+ self.root / "out",+ )+ project.assert_not_called()++ @unittest.skipUnless(sys.platform == "darwin", "Uses real Mach-O fixture libraries")+ def test_projects_archive_through_sandbox_link_with_native_aliases(self):+ fixture = test_macos_runtime.RuntimeTests("runTest")+ self.addCleanup(fixture.doCleanups)+ fixture.setUp()+ self.receipt.write_text(json.dumps({**self.build, "target": fixture.target}))+ library = next((fixture.prefix / "lib").glob("*.dylib"))+ (library.parent / "development-alias.dylib").symlink_to(library.name)+ archive = self.root / "prefix.tar"+ with tarfile.open(archive, "w") as source:+ source.add(fixture.prefix, arcname=".")+ sandbox_input = self.root / "sandbox-input.tar"+ sandbox_input.symlink_to(archive)+ for state in ("absent", "empty"):+ with self.subTest(output_state=state):+ output = self.root / f"runtime-{state}"+ if state == "empty":+ output.mkdir()+ subprocess.run(+ [+ sys.executable,+ str(Path(__file__).with_name("prepare_built_runtime.py")),+ "--prefix",+ str(sandbox_input),+ "--build-receipt",+ str(self.receipt),+ "--status",+ str(self.status),+ "--target",+ fixture.target,+ "--output",+ str(output),+ ],+ check=True,+ )+ manifest = json.loads((output / "runtime.json").read_text())+ self.assertEqual(manifest["sourceCommit"], "a" * 40)+ self.assertEqual(manifest["target"], fixture.target)+ self.assertTrue(+ all(+ not macos_runtime.inspect(+ output / record["path"], fixture.target+ ).rpaths+ for record in manifest["libraries"]+ )+ )++ def test_archive_rejects_escaping_native_alias_before_inspection(self):+ archive = self.root / "prefix.tar"+ with tarfile.open(archive, "w") as source:+ link = tarfile.TarInfo("lib/escape.dylib")+ link.type = tarfile.SYMTYPE+ link.linkname = "../../outside.dylib"+ source.addfile(link)+ with patch.object(macos_runtime, "inspect") as inspect:+ with self.assertRaises(tarfile.LinkOutsideDestinationError):+ prepare_archive(+ archive, self.receipt, self.status, self.target, self.root / "out"+ )+ inspect.assert_not_called()