Lightweight coding agent that runs in your terminal
Rust
Add native voice dependency build recipe (#41890)
Benjamin Carlssonfc7d34ad67d0
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazelindex 817a9d67f8..efcc1950fd 100644--- a/third_party/voice/BUILD.bazel+++ b/third_party/voice/BUILD.bazel@@ -27,3 +27,12 @@ filegroup(tags = ["manual"],visibility = ["//visibility:public"],)++filegroup(+ name = "build_inputs",+ srcs = [+ "build_native.py",+ ":source_inputs",+ ],+ visibility = ["//visibility:public"],+)diff --git a/third_party/voice/README.md b/third_party/voice/README.mdindex 90a853529c..fe3235fff8 100644--- a/third_party/voice/README.md+++ b/third_party/voice/README.md@@ -46,3 +46,32 @@ Checksums establish input identity, not security or license approval. Nativecompilation, final Cargo/Bazel linking, installed packages, minimum OS supportand duplex audio validation remain separate stages. These inputs do not establisha shared Opus build with Rust consumers or a reduced dependency count.++Rust `opus` 0.4.0 is available through Socket. Adding Rust transport dependencies+and establishing a shared Opus build remain separate integration work.++## Native build recipe++`build_native.py` runs the unmodified upstream build systems in a new output+directory, using the same archives. Specify the target and existing compiler,+CMake, make, pkg-config and shell paths explicitly. It requires a matching+native host: GNU Linux, macOS, or Windows MSVC, on x64 or ARM64.++On macOS, specify the existing release deployment target with+`--deployment-target`; the host OS version is not an acceptable default.+Windows requires the normal Visual Studio SDK environment, GNU make and a+POSIX shell for upstream libffi, and `--bootstrap-make` pointing to NMake.+The recipe does not install these build prerequisites or patch upstream sources.++Outputs are under `prefix/`, build tools under `tools/`, and logs beside them.+`build-state.json` records completed commands and failures; `built.json` exists+only when every build/install command succeeds. Failed builds retain their logs+and must use a new output directory on retry. CMake compiler-identification logs+and the recorded tool/configuration inputs remain part of the build provenance.++The recipe disables optional plugins and Meson fallback dependency resolution,+with pkg-config restricted to this prefix. Only system ABI libraries/frameworks+may remain external; runtime closure inspection must verify that independently.+`//third_party/voice:build_inputs` exposes the recipe and source inputs to Bazel.+Neither this filegroup nor a successful prefix build proves final Cargo/Bazel+linkage, safe private runtime loading, or an installed voice-capable Codex package.diff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.pynew file mode 100644index 0000000000..33e16e03f8--- /dev/null+++ b/third_party/voice/build_native.py@@ -0,0 +1,401 @@+#!/usr/bin/env python3+"""Build the candidate native voice prefix with upstream build systems."""++import argparse+import hashlib+import json+import os+from pathlib import Path+import platform+import re+import shlex+import subprocess+import sys++# Match the package builder: import this script's sibling under PYTHONSAFEPATH,+# without adding the caller's working directory to the module search path.+sys.path.insert(0, str(Path(__file__).resolve().parent))+from prepare_sources import MANIFEST, load_sources, prepare_sources++TARGET_SYSTEMS = {+ "apple-darwin": "Darwin",+ "unknown-linux-gnu": "Linux",+ "pc-windows-msvc": "Windows",+}+++def validate_target(target, system, machine, libc, deployment_target):+ architecture, separator, suffix = target.partition("-")+ host_architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get(+ machine.lower(), machine.lower()+ )+ if (+ not separator+ or architecture not in ("aarch64", "x86_64")+ or suffix not in TARGET_SYSTEMS+ ):+ raise ValueError(f"Unsupported native voice target: {target}")+ if TARGET_SYSTEMS[suffix] != system or architecture != host_architecture:+ raise ValueError("Use a native build host matching the requested target")+ if system == "Linux" and libc != "glibc":+ raise ValueError("Native Linux voice builds require glibc")+ if system == "Darwin" and not re.fullmatch(+ r"[0-9]+\.[0-9]+(?:\.[0-9]+)?", deployment_target or ""+ ):+ raise ValueError(+ "Declare the macOS deployment target; do not inherit the host default"+ )+++class NativeBuild:+ def __init__(self, args, inherited_environment):+ validate_target(+ args.target,+ platform.system(),+ platform.machine(),+ platform.libc_ver()[0],+ args.deployment_target,+ )+ self.args = args+ self.output = args.output.absolute()+ self.prefix = self.output / "prefix"+ self.tools = self.output / "tools"+ self.windows = args.target.endswith("windows-msvc")+ # Compiler drivers such as clang++ select link behavior from argv[0].+ self.toolchain = {+ name: getattr(args, name).absolute()+ for name in ("cc", "cxx", "cmake", "make", "pkg_config", "shell")+ }+ self.bootstrap_make = (args.bootstrap_make or args.make).absolute()+ for tool in (*self.toolchain.values(), self.bootstrap_make):+ if not tool.is_file():+ raise ValueError(f"Missing build tool: {tool}")+ if self.windows and not all(+ inherited_environment.get(key) for key in ("INCLUDE", "LIB")+ ):+ raise ValueError(+ "Initialize the standard Visual Studio build environment first"+ )+ self.environment = {+ key: value+ for key, value in inherited_environment.items()+ if key+ in (+ "HOME",+ "TMPDIR",+ "TMP",+ "TEMP",+ "SYSTEMROOT",+ "SystemRoot",+ "WINDIR",+ "COMSPEC",+ "INCLUDE",+ "LIB",+ "LIBPATH",+ "HTTPS_PROXY",+ "HTTP_PROXY",+ "NO_PROXY",+ )+ }+ paths = [+ str(self.tools / "bin"),+ *(str(p.parent) for p in self.toolchain.values()),+ ]+ paths += (+ inherited_environment.get("PATH", "").split(os.pathsep)+ if self.windows+ else ["/usr/bin", "/bin", "/usr/sbin", "/sbin"]+ )+ self.environment.update(+ {+ "PATH": os.pathsep.join(dict.fromkeys(paths)),+ "LANG": "C",+ "LC_ALL": "C",+ "PYTHONDONTWRITEBYTECODE": "1",+ "CC": str(self.toolchain["cc"]),+ "CXX": str(self.toolchain["cxx"]),+ "PKG_CONFIG": str(self.toolchain["pkg_config"]),+ "PKG_CONFIG_PATH": "",+ "PKG_CONFIG_LIBDIR": os.pathsep.join(+ str(self.prefix / p) for p in ("lib/pkgconfig", "share/pkgconfig")+ ),+ "CMAKE_PREFIX_PATH": str(self.prefix),+ "NINJA": str(+ self.tools / "bin" / ("ninja.exe" if self.windows else "ninja")+ ),+ }+ )+ self.cmake_platform = []+ if args.target.endswith("apple-darwin"):+ self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_target+ self.cmake_platform = [+ f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}"+ ]+ self.record = {+ "target": args.target,+ "deployment_target": args.deployment_target,+ "steps": [],+ }++ def run(self, name, command, cwd=None, environment=None):+ command = [str(part) for part in command]+ step = {"name": name, "command": command}+ self.record["steps"].append(step)+ print(f"Building {name}", flush=True)+ with (self.output / f"{name}.log").open("w", encoding="utf-8") as log:+ result = subprocess.run(+ command,+ cwd=cwd or self.output,+ env=environment or self.environment,+ stdout=log,+ stderr=subprocess.STDOUT,+ check=False,+ )+ step["exit_code"] = result.returncode+ (self.output / "build-state.json").write_text(+ json.dumps(self.record, indent=2) + "\n", encoding="utf-8"+ )+ result.check_returncode()++ def cmake(self, name, options, *, bootstrap=False):+ directory = self.output / "build" / name+ prefix = self.tools if bootstrap else self.prefix+ generator = (+ ("NMake Makefiles" if self.windows else "Unix Makefiles")+ if bootstrap+ else "Ninja"+ )+ make = self.bootstrap_make if bootstrap else self.environment["NINJA"]+ self.run(+ name + "-configure",+ [+ self.toolchain["cmake"],+ "-S",+ self.sources[name],+ "-B",+ directory,+ "-G",+ generator,+ f"-DCMAKE_MAKE_PROGRAM={make}",+ "-DCMAKE_BUILD_TYPE=Release",+ f"-DCMAKE_INSTALL_PREFIX={prefix}",+ "-DCMAKE_INSTALL_LIBDIR=lib",+ f"-DCMAKE_C_COMPILER={self.toolchain['cc']}",+ f"-DCMAKE_CXX_COMPILER={self.toolchain['cxx']}",+ f"-DCMAKE_PREFIX_PATH={self.prefix}",+ "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF",+ "-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF",+ "-DCMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH=OFF",+ "-DFETCHCONTENT_FULLY_DISCONNECTED=ON",+ *self.cmake_platform,+ *options,+ ],+ )+ self.run(+ name + "-build",+ [+ self.toolchain["cmake"],+ "--build",+ directory,+ "--parallel",+ self.args.jobs,+ ],+ )+ self.run(name + "-install", [self.toolchain["cmake"], "--install", directory])++ def meson(self, name, options):+ directory = self.output / "build" / name+ meson = [sys.executable, self.sources["meson"] / "meson.py"]+ quote = subprocess.list2cmdline if self.windows else shlex.join+ include = quote([f"{'/I' if self.windows else '-I'}{self.prefix / 'include'}"])+ link = (+ [f"/LIBPATH:{self.prefix / 'lib'}"]+ if self.windows+ else [f"-L{self.prefix / 'lib'}", f"-Wl,-rpath,{self.prefix / 'lib'}"]+ )+ self.environment.update(+ {"CFLAGS": include, "CXXFLAGS": include, "LDFLAGS": quote(link)}+ )+ self.run(+ name + "-configure",+ [+ *meson,+ "setup",+ directory,+ self.sources[name],+ f"--prefix={self.prefix}",+ "--libdir=lib",+ "--buildtype=release",+ "--wrap-mode=nofallback",+ "-Dauto_features=disabled",+ "-Ddefault_library=shared",+ *options,+ ],+ )+ self.run(+ name + "-build", [*meson, "compile", "-C", directory, "-j", self.args.jobs]+ )+ self.run(+ name + "-install", [*meson, "install", "-C", directory, "--no-rebuild"]+ )++ def build(self):+ self.output.mkdir()+ manifest = MANIFEST.read_bytes()+ prepare_sources(self.args.archives, self.output / "sources", manifest)+ self.sources = {+ s.name: self.output / "sources" / s.root for s in load_sources(manifest)+ }+ self.record["manifest_sha256"] = hashlib.sha256(manifest).hexdigest()+ self.record["tools"] = {+ name: str(path) for name, path in self.toolchain.items()+ }+ self.record["python"] = sys.version+ self.run("cmake-version", [self.toolchain["cmake"], "--version"])+ self.run("pkg-config-version", [self.toolchain["pkg_config"], "--version"])+ self.cmake("ninja", ["-DBUILD_TESTING=OFF"], bootstrap=True)+ self.cmake(+ "zlib",+ [+ "-DZLIB_BUILD_TESTING=OFF",+ "-DZLIB_BUILD_SHARED=ON",+ "-DZLIB_BUILD_STATIC=OFF",+ ],+ )+ self.cmake(+ "pcre2",+ [+ "-DBUILD_SHARED_LIBS=ON",+ "-DBUILD_STATIC_LIBS=OFF",+ "-DPCRE2_BUILD_TESTS=OFF",+ "-DPCRE2_BUILD_PCRE2GREP=OFF",+ "-DPCRE2_SUPPORT_LIBZ=OFF",+ "-DPCRE2_SUPPORT_LIBBZ2=OFF",+ "-DPCRE2_SUPPORT_LIBREADLINE=OFF",+ "-DPCRE2_SUPPORT_LIBEDIT=OFF",+ ],+ )+ ffi_build = self.output / "build/libffi"+ ffi_build.mkdir()+ environment = self.environment.copy()+ if self.windows:+ wrapper = shlex.quote((self.sources["libffi"] / "msvcc.sh").as_posix())+ architecture = (+ "-m64" if self.args.target.startswith("x86_64-") else "-marm64"+ )+ environment.update(+ {+ "CC": f"{wrapper} {architecture}",+ "CXX": f"{wrapper} {architecture}",+ "LD": "link",+ "CPP": "cl -nologo -EP",+ "CXXCPP": "cl -nologo -EP",+ "CPPFLAGS": "-DFFI_BUILDING_DLL",+ }+ )+ self.run(+ "libffi-configure",+ [+ self.toolchain["shell"],+ (self.sources["libffi"] / "configure").as_posix(),+ f"--prefix={self.prefix.as_posix()}",+ "--enable-shared",+ "--disable-static",+ "--disable-docs",+ ],+ cwd=ffi_build,+ environment=environment,+ )+ self.run(+ "libffi-build",+ [self.toolchain["make"], f"-j{self.args.jobs}"],+ cwd=ffi_build,+ environment=environment,+ )+ self.run(+ "libffi-install",+ [self.toolchain["make"], "install"],+ cwd=ffi_build,+ environment=environment,+ )+ self.cmake(+ "opus",+ [+ "-DOPUS_BUILD_SHARED_LIBRARY=ON",+ "-DOPUS_BUILD_TESTING=OFF",+ "-DOPUS_BUILD_PROGRAMS=OFF",+ ],+ )+ self.meson("proxy-libintl", [])+ self.meson(+ "glib",+ [+ "-Dtests=false",+ "-Dinstalled_tests=false",+ "-Dnls=disabled",+ "-Ddocumentation=false",+ "-Dintrospection=disabled",+ "-Dlibmount=disabled",+ "-Dselinux=disabled",+ "-Dxattr=false",+ ],+ )+ self.meson(+ "gstreamer",+ [+ "-Dregistry=false",+ "-Doption-parsing=false",+ "-Dtracer_hooks=false",+ "-Dgst_parse=false",+ "-Dtools=disabled",+ "-Dptp-helper=disabled",+ ],+ )+ self.meson(+ "gst-plugins-base",+ [+ "-Dapp=enabled",+ "-Daudioconvert=enabled",+ "-Daudioresample=enabled",+ "-Dopus=enabled",+ "-Dgl=disabled",+ ],+ )+ self.meson("gst-plugins-good", ["-Drtp=enabled", "-Drtpmanager=enabled"])+ (self.output / "built.json").write_text(+ json.dumps(self.record, indent=2) + "\n", encoding="utf-8"+ )+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ for name in (+ "archives",+ "output",+ "cc",+ "cxx",+ "cmake",+ "make",+ "pkg-config",+ "shell",+ ):+ parser.add_argument(f"--{name}", type=Path, required=True)+ parser.add_argument(+ "--bootstrap-make",+ type=Path,+ help="NMake on Windows; defaults to --make elsewhere",+ )+ parser.add_argument("--target", required=True)+ parser.add_argument(+ "--deployment-target",+ help="Required on macOS; use the existing supported release minimum",+ )+ parser.add_argument("--jobs", type=int, default=8)+ args = parser.parse_args()+ if sys.version_info < (3, 12) or args.jobs < 1:+ parser.error("Python 3.12+ and a positive --jobs value are required")+ NativeBuild(args, os.environ).build()+++if __name__ == "__main__":+ main()diff --git a/third_party/voice/test_build_native.py b/third_party/voice/test_build_native.pynew file mode 100644index 0000000000..bf04af8a2c--- /dev/null+++ b/third_party/voice/test_build_native.py@@ -0,0 +1,182 @@+"""Check native build preconditions and subprocess failure propagation."""++import json+import os+from pathlib import Path+import platform+import shlex+import shutil+import subprocess+import sys+import tempfile+from types import SimpleNamespace+import unittest+from unittest.mock import patch++from build_native import NativeBuild, validate_target+++class NativeBuildTests(unittest.TestCase):+ def setUp(self):+ temporary = tempfile.TemporaryDirectory()+ self.addCleanup(temporary.cleanup)+ self.root = Path(temporary.name)+ machine = platform.machine().lower()+ architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get(machine, machine)+ suffix = {+ "Darwin": "apple-darwin",+ "Linux": "unknown-linux-gnu",+ "Windows": "pc-windows-msvc",+ }[platform.system()]+ self.args = SimpleNamespace(+ target=f"{architecture}-{suffix}",+ deployment_target="11.0",+ output=self.root / "build output",+ archives=self.root / "archives",+ cc=Path(sys.executable),+ cxx=Path(sys.executable),+ cmake=Path(sys.executable),+ make=Path(sys.executable),+ pkg_config=Path(sys.executable),+ shell=Path(sys.executable),+ bootstrap_make=None,+ jobs=2,+ )+ self.environment = {+ **os.environ,+ "INCLUDE": "fixture include",+ "LIB": "fixture lib",+ }++ def test_cli_entrypoint_imports_its_sibling_with_safe_path_enabled(self):+ result = subprocess.run(+ [+ sys.executable,+ "-P",+ str(Path(__file__).with_name("build_native.py")),+ "--help",+ ],+ cwd=self.root,+ capture_output=True,+ text=True,+ check=False,+ )+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertIn("--archives", result.stdout)++ def test_rejects_cross_host_and_musl_builds(self):+ for target, system, machine, libc in [+ ("x86_64-pc-windows-msvc", "Darwin", "x86_64", ""),+ ("aarch64-apple-darwin", "Darwin", "x86_64", ""),+ ("x86_64-unknown-linux-gnu", "Linux", "x86_64", "musl"),+ ("x86_64-unknown-linux-musl", "Linux", "x86_64", "musl"),+ ]:+ with self.subTest(target=target, system=system):+ with self.assertRaises(ValueError):+ validate_target(target, system, machine, libc, "11.0")++ def test_requires_explicit_macos_deployment_target(self):+ with self.assertRaisesRegex(ValueError, "Declare the macOS deployment target"):+ validate_target("aarch64-apple-darwin", "Darwin", "arm64", "", None)++ def test_missing_tool_does_not_create_output(self):+ self.args.cc = self.root / "missing compiler"+ with self.assertRaisesRegex(ValueError, "Missing build tool"):+ NativeBuild(self.args, self.environment)+ self.assertFalse(self.args.output.exists())++ @unittest.skipIf(os.name == "nt", "Exercises the Unix clang++ driver symlink")+ def test_bootstrap_links_cpp_runtime_through_compiler_symlink(self):+ tools = {name: shutil.which(name) for name in ("clang", "cmake", "make")}+ if not all(tools.values()):+ self.skipTest("Requires clang, CMake and make")+ compiler = self.root / "clang++"+ compiler.symlink_to(Path(tools["clang"]).resolve())+ self.args.cc = Path(tools["clang"])+ self.args.cxx = compiler+ self.args.cmake = Path(tools["cmake"])+ self.args.make = Path(tools["make"])+ source = self.root / "cpp-source"+ source.mkdir()+ (source / "CMakeLists.txt").write_text(+ "cmake_minimum_required(VERSION 3.15)\n"+ "project(cpp_driver LANGUAGES CXX)\n"+ "add_executable(cpp_driver main.cpp)\n"+ "install(TARGETS cpp_driver DESTINATION bin)\n"+ )+ (source / "main.cpp").write_text(+ '#include <iostream>\nint main() { std::cout << "linked"; }\n'+ )+ build = NativeBuild(self.args, self.environment)+ build.output.mkdir()+ build.sources = {"cpp-driver": source}+ build.cmake("cpp-driver", [], bootstrap=True)+ result = subprocess.run(+ [build.tools / "bin" / "cpp_driver"],+ check=True,+ capture_output=True,+ text=True,+ )+ self.assertEqual(result.stdout, "linked")++ def test_build_refuses_existing_output_before_reading_sources(self):+ self.args.output.mkdir()+ (self.args.output / "keep").write_text("untouched")+ with self.assertRaises(FileExistsError):+ NativeBuild(self.args, self.environment).build()+ self.assertEqual(+ {p.name: p.read_text() for p in self.args.output.iterdir()},+ {"keep": "untouched"},+ )++ def test_failure_retains_log_and_state_without_completion_marker(self):+ build = NativeBuild(self.args, self.environment)+ build.output.mkdir()+ command = [+ sys.executable,+ "-c",+ "print('synthetic failure'); raise SystemExit(23)",+ ]+ with self.assertRaises(subprocess.CalledProcessError) as error:+ build.run("failure", command)+ self.assertEqual(error.exception.returncode, 23)+ self.assertEqual(+ (build.output / "failure.log").read_text().strip(), "synthetic failure"+ )+ self.assertEqual(+ json.loads((build.output / "build-state.json").read_text()),+ {+ "target": self.args.target,+ "deployment_target": "11.0",+ "steps": [{"name": "failure", "command": command, "exit_code": 23}],+ },+ )+ self.assertFalse((build.output / "built.json").exists())++ def test_ambient_native_discovery_variables_are_not_inherited(self):+ inherited = {+ **self.environment,+ "PKG_CONFIG_PATH": "/ambient",+ "CMAKE_PREFIX_PATH": "/ambient",+ "CPATH": "/ambient",+ "CFLAGS": "-I/ambient",+ "LDFLAGS": "-L/ambient",+ }+ environment = NativeBuild(self.args, inherited).environment+ self.assertFalse(any("/ambient" in value for value in environment.values()))+ self.assertEqual(environment["PKG_CONFIG_PATH"], "")++ def test_meson_receives_private_link_inputs_with_spaces(self):+ build = NativeBuild(self.args, self.environment)+ build.sources = {"meson": self.root / "meson", "glib": self.root / "glib"}+ with patch.object(build, "run"):+ build.meson("glib", [])+ if build.windows:+ self.assertEqual(+ build.environment["LDFLAGS"], f'"/LIBPATH:{build.prefix / "lib"}"'+ )+ else:+ self.assertEqual(+ shlex.split(build.environment["LDFLAGS"]),+ [f"-L{build.prefix / 'lib'}", f"-Wl,-rpath,{build.prefix / 'lib'}"],+ )