Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add native voice dependency build recipe (#41890)

Benjamin Carlssonfc7d34ad67d0
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel
index 817a9d67f8..efcc1950fd 100644
--- a/third_party/voice/BUILD.bazel
+++ b/third_party/voice/BUILD.bazel
@@ -27,3 +27,12 @@ filegroup(
tags = ["manual"],
visibility = ["//visibility:public"],
)
+
+filegroup(
+ name = "build_inputs",
+ srcs = [
+ "build_native.py",
+ ":source_inputs",
+ ],
+ visibility = ["//visibility:public"],
+)
diff --git a/third_party/voice/README.md b/third_party/voice/README.md
index 90a853529c..fe3235fff8 100644
--- a/third_party/voice/README.md
+++ b/third_party/voice/README.md
@@ -46,3 +46,32 @@ Checksums establish input identity, not security or license approval. Native
compilation, final Cargo/Bazel linking, installed packages, minimum OS support
and duplex audio validation remain separate stages. These inputs do not establish
a shared Opus build with Rust consumers or a reduced dependency count.
+
+Rust `opus` 0.4.0 is available through Socket. Adding Rust transport dependencies
+and establishing a shared Opus build remain separate integration work.
+
+## Native build recipe
+
+`build_native.py` runs the unmodified upstream build systems in a new output
+directory, using the same archives. Specify the target and existing compiler,
+CMake, make, pkg-config and shell paths explicitly. It requires a matching
+native host: GNU Linux, macOS, or Windows MSVC, on x64 or ARM64.
+
+On macOS, specify the existing release deployment target with
+`--deployment-target`; the host OS version is not an acceptable default.
+Windows requires the normal Visual Studio SDK environment, GNU make and a
+POSIX shell for upstream libffi, and `--bootstrap-make` pointing to NMake.
+The recipe does not install these build prerequisites or patch upstream sources.
+
+Outputs are under `prefix/`, build tools under `tools/`, and logs beside them.
+`build-state.json` records completed commands and failures; `built.json` exists
+only when every build/install command succeeds. Failed builds retain their logs
+and must use a new output directory on retry. CMake compiler-identification logs
+and the recorded tool/configuration inputs remain part of the build provenance.
+
+The recipe disables optional plugins and Meson fallback dependency resolution,
+with pkg-config restricted to this prefix. Only system ABI libraries/frameworks
+may remain external; runtime closure inspection must verify that independently.
+`//third_party/voice:build_inputs` exposes the recipe and source inputs to Bazel.
+Neither this filegroup nor a successful prefix build proves final Cargo/Bazel
+linkage, safe private runtime loading, or an installed voice-capable Codex package.
diff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.py
new file mode 100644
index 0000000000..33e16e03f8
--- /dev/null
+++ b/third_party/voice/build_native.py
@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+"""Build the candidate native voice prefix with upstream build systems."""
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import shlex
+import subprocess
+import sys
+
+# Match the package builder: import this script's sibling under PYTHONSAFEPATH,
+# without adding the caller's working directory to the module search path.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from prepare_sources import MANIFEST, load_sources, prepare_sources
+
+TARGET_SYSTEMS = {
+ "apple-darwin": "Darwin",
+ "unknown-linux-gnu": "Linux",
+ "pc-windows-msvc": "Windows",
+}
+
+
+def validate_target(target, system, machine, libc, deployment_target):
+ architecture, separator, suffix = target.partition("-")
+ host_architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get(
+ machine.lower(), machine.lower()
+ )
+ if (
+ not separator
+ or architecture not in ("aarch64", "x86_64")
+ or suffix not in TARGET_SYSTEMS
+ ):
+ raise ValueError(f"Unsupported native voice target: {target}")
+ if TARGET_SYSTEMS[suffix] != system or architecture != host_architecture:
+ raise ValueError("Use a native build host matching the requested target")
+ if system == "Linux" and libc != "glibc":
+ raise ValueError("Native Linux voice builds require glibc")
+ if system == "Darwin" and not re.fullmatch(
+ r"[0-9]+\.[0-9]+(?:\.[0-9]+)?", deployment_target or ""
+ ):
+ raise ValueError(
+ "Declare the macOS deployment target; do not inherit the host default"
+ )
+
+
+class NativeBuild:
+ def __init__(self, args, inherited_environment):
+ validate_target(
+ args.target,
+ platform.system(),
+ platform.machine(),
+ platform.libc_ver()[0],
+ args.deployment_target,
+ )
+ self.args = args
+ self.output = args.output.absolute()
+ self.prefix = self.output / "prefix"
+ self.tools = self.output / "tools"
+ self.windows = args.target.endswith("windows-msvc")
+ # Compiler drivers such as clang++ select link behavior from argv[0].
+ self.toolchain = {
+ name: getattr(args, name).absolute()
+ for name in ("cc", "cxx", "cmake", "make", "pkg_config", "shell")
+ }
+ self.bootstrap_make = (args.bootstrap_make or args.make).absolute()
+ for tool in (*self.toolchain.values(), self.bootstrap_make):
+ if not tool.is_file():
+ raise ValueError(f"Missing build tool: {tool}")
+ if self.windows and not all(
+ inherited_environment.get(key) for key in ("INCLUDE", "LIB")
+ ):
+ raise ValueError(
+ "Initialize the standard Visual Studio build environment first"
+ )
+ self.environment = {
+ key: value
+ for key, value in inherited_environment.items()
+ if key
+ in (
+ "HOME",
+ "TMPDIR",
+ "TMP",
+ "TEMP",
+ "SYSTEMROOT",
+ "SystemRoot",
+ "WINDIR",
+ "COMSPEC",
+ "INCLUDE",
+ "LIB",
+ "LIBPATH",
+ "HTTPS_PROXY",
+ "HTTP_PROXY",
+ "NO_PROXY",
+ )
+ }
+ paths = [
+ str(self.tools / "bin"),
+ *(str(p.parent) for p in self.toolchain.values()),
+ ]
+ paths += (
+ inherited_environment.get("PATH", "").split(os.pathsep)
+ if self.windows
+ else ["/usr/bin", "/bin", "/usr/sbin", "/sbin"]
+ )
+ self.environment.update(
+ {
+ "PATH": os.pathsep.join(dict.fromkeys(paths)),
+ "LANG": "C",
+ "LC_ALL": "C",
+ "PYTHONDONTWRITEBYTECODE": "1",
+ "CC": str(self.toolchain["cc"]),
+ "CXX": str(self.toolchain["cxx"]),
+ "PKG_CONFIG": str(self.toolchain["pkg_config"]),
+ "PKG_CONFIG_PATH": "",
+ "PKG_CONFIG_LIBDIR": os.pathsep.join(
+ str(self.prefix / p) for p in ("lib/pkgconfig", "share/pkgconfig")
+ ),
+ "CMAKE_PREFIX_PATH": str(self.prefix),
+ "NINJA": str(
+ self.tools / "bin" / ("ninja.exe" if self.windows else "ninja")
+ ),
+ }
+ )
+ self.cmake_platform = []
+ if args.target.endswith("apple-darwin"):
+ self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_target
+ self.cmake_platform = [
+ f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}"
+ ]
+ self.record = {
+ "target": args.target,
+ "deployment_target": args.deployment_target,
+ "steps": [],
+ }
+
+ def run(self, name, command, cwd=None, environment=None):
+ command = [str(part) for part in command]
+ step = {"name": name, "command": command}
+ self.record["steps"].append(step)
+ print(f"Building {name}", flush=True)
+ with (self.output / f"{name}.log").open("w", encoding="utf-8") as log:
+ result = subprocess.run(
+ command,
+ cwd=cwd or self.output,
+ env=environment or self.environment,
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ check=False,
+ )
+ step["exit_code"] = result.returncode
+ (self.output / "build-state.json").write_text(
+ json.dumps(self.record, indent=2) + "\n", encoding="utf-8"
+ )
+ result.check_returncode()
+
+ def cmake(self, name, options, *, bootstrap=False):
+ directory = self.output / "build" / name
+ prefix = self.tools if bootstrap else self.prefix
+ generator = (
+ ("NMake Makefiles" if self.windows else "Unix Makefiles")
+ if bootstrap
+ else "Ninja"
+ )
+ make = self.bootstrap_make if bootstrap else self.environment["NINJA"]
+ self.run(
+ name + "-configure",
+ [
+ self.toolchain["cmake"],
+ "-S",
+ self.sources[name],
+ "-B",
+ directory,
+ "-G",
+ generator,
+ f"-DCMAKE_MAKE_PROGRAM={make}",
+ "-DCMAKE_BUILD_TYPE=Release",
+ f"-DCMAKE_INSTALL_PREFIX={prefix}",
+ "-DCMAKE_INSTALL_LIBDIR=lib",
+ f"-DCMAKE_C_COMPILER={self.toolchain['cc']}",
+ f"-DCMAKE_CXX_COMPILER={self.toolchain['cxx']}",
+ f"-DCMAKE_PREFIX_PATH={self.prefix}",
+ "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF",
+ "-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF",
+ "-DCMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH=OFF",
+ "-DFETCHCONTENT_FULLY_DISCONNECTED=ON",
+ *self.cmake_platform,
+ *options,
+ ],
+ )
+ self.run(
+ name + "-build",
+ [
+ self.toolchain["cmake"],
+ "--build",
+ directory,
+ "--parallel",
+ self.args.jobs,
+ ],
+ )
+ self.run(name + "-install", [self.toolchain["cmake"], "--install", directory])
+
+ def meson(self, name, options):
+ directory = self.output / "build" / name
+ meson = [sys.executable, self.sources["meson"] / "meson.py"]
+ quote = subprocess.list2cmdline if self.windows else shlex.join
+ include = quote([f"{'/I' if self.windows else '-I'}{self.prefix / 'include'}"])
+ link = (
+ [f"/LIBPATH:{self.prefix / 'lib'}"]
+ if self.windows
+ else [f"-L{self.prefix / 'lib'}", f"-Wl,-rpath,{self.prefix / 'lib'}"]
+ )
+ self.environment.update(
+ {"CFLAGS": include, "CXXFLAGS": include, "LDFLAGS": quote(link)}
+ )
+ self.run(
+ name + "-configure",
+ [
+ *meson,
+ "setup",
+ directory,
+ self.sources[name],
+ f"--prefix={self.prefix}",
+ "--libdir=lib",
+ "--buildtype=release",
+ "--wrap-mode=nofallback",
+ "-Dauto_features=disabled",
+ "-Ddefault_library=shared",
+ *options,
+ ],
+ )
+ self.run(
+ name + "-build", [*meson, "compile", "-C", directory, "-j", self.args.jobs]
+ )
+ self.run(
+ name + "-install", [*meson, "install", "-C", directory, "--no-rebuild"]
+ )
+
+ def build(self):
+ self.output.mkdir()
+ manifest = MANIFEST.read_bytes()
+ prepare_sources(self.args.archives, self.output / "sources", manifest)
+ self.sources = {
+ s.name: self.output / "sources" / s.root for s in load_sources(manifest)
+ }
+ self.record["manifest_sha256"] = hashlib.sha256(manifest).hexdigest()
+ self.record["tools"] = {
+ name: str(path) for name, path in self.toolchain.items()
+ }
+ self.record["python"] = sys.version
+ self.run("cmake-version", [self.toolchain["cmake"], "--version"])
+ self.run("pkg-config-version", [self.toolchain["pkg_config"], "--version"])
+ self.cmake("ninja", ["-DBUILD_TESTING=OFF"], bootstrap=True)
+ self.cmake(
+ "zlib",
+ [
+ "-DZLIB_BUILD_TESTING=OFF",
+ "-DZLIB_BUILD_SHARED=ON",
+ "-DZLIB_BUILD_STATIC=OFF",
+ ],
+ )
+ self.cmake(
+ "pcre2",
+ [
+ "-DBUILD_SHARED_LIBS=ON",
+ "-DBUILD_STATIC_LIBS=OFF",
+ "-DPCRE2_BUILD_TESTS=OFF",
+ "-DPCRE2_BUILD_PCRE2GREP=OFF",
+ "-DPCRE2_SUPPORT_LIBZ=OFF",
+ "-DPCRE2_SUPPORT_LIBBZ2=OFF",
+ "-DPCRE2_SUPPORT_LIBREADLINE=OFF",
+ "-DPCRE2_SUPPORT_LIBEDIT=OFF",
+ ],
+ )
+ ffi_build = self.output / "build/libffi"
+ ffi_build.mkdir()
+ environment = self.environment.copy()
+ if self.windows:
+ wrapper = shlex.quote((self.sources["libffi"] / "msvcc.sh").as_posix())
+ architecture = (
+ "-m64" if self.args.target.startswith("x86_64-") else "-marm64"
+ )
+ environment.update(
+ {
+ "CC": f"{wrapper} {architecture}",
+ "CXX": f"{wrapper} {architecture}",
+ "LD": "link",
+ "CPP": "cl -nologo -EP",
+ "CXXCPP": "cl -nologo -EP",
+ "CPPFLAGS": "-DFFI_BUILDING_DLL",
+ }
+ )
+ self.run(
+ "libffi-configure",
+ [
+ self.toolchain["shell"],
+ (self.sources["libffi"] / "configure").as_posix(),
+ f"--prefix={self.prefix.as_posix()}",
+ "--enable-shared",
+ "--disable-static",
+ "--disable-docs",
+ ],
+ cwd=ffi_build,
+ environment=environment,
+ )
+ self.run(
+ "libffi-build",
+ [self.toolchain["make"], f"-j{self.args.jobs}"],
+ cwd=ffi_build,
+ environment=environment,
+ )
+ self.run(
+ "libffi-install",
+ [self.toolchain["make"], "install"],
+ cwd=ffi_build,
+ environment=environment,
+ )
+ self.cmake(
+ "opus",
+ [
+ "-DOPUS_BUILD_SHARED_LIBRARY=ON",
+ "-DOPUS_BUILD_TESTING=OFF",
+ "-DOPUS_BUILD_PROGRAMS=OFF",
+ ],
+ )
+ self.meson("proxy-libintl", [])
+ self.meson(
+ "glib",
+ [
+ "-Dtests=false",
+ "-Dinstalled_tests=false",
+ "-Dnls=disabled",
+ "-Ddocumentation=false",
+ "-Dintrospection=disabled",
+ "-Dlibmount=disabled",
+ "-Dselinux=disabled",
+ "-Dxattr=false",
+ ],
+ )
+ self.meson(
+ "gstreamer",
+ [
+ "-Dregistry=false",
+ "-Doption-parsing=false",
+ "-Dtracer_hooks=false",
+ "-Dgst_parse=false",
+ "-Dtools=disabled",
+ "-Dptp-helper=disabled",
+ ],
+ )
+ self.meson(
+ "gst-plugins-base",
+ [
+ "-Dapp=enabled",
+ "-Daudioconvert=enabled",
+ "-Daudioresample=enabled",
+ "-Dopus=enabled",
+ "-Dgl=disabled",
+ ],
+ )
+ self.meson("gst-plugins-good", ["-Drtp=enabled", "-Drtpmanager=enabled"])
+ (self.output / "built.json").write_text(
+ json.dumps(self.record, indent=2) + "\n", encoding="utf-8"
+ )
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ for name in (
+ "archives",
+ "output",
+ "cc",
+ "cxx",
+ "cmake",
+ "make",
+ "pkg-config",
+ "shell",
+ ):
+ parser.add_argument(f"--{name}", type=Path, required=True)
+ parser.add_argument(
+ "--bootstrap-make",
+ type=Path,
+ help="NMake on Windows; defaults to --make elsewhere",
+ )
+ parser.add_argument("--target", required=True)
+ parser.add_argument(
+ "--deployment-target",
+ help="Required on macOS; use the existing supported release minimum",
+ )
+ parser.add_argument("--jobs", type=int, default=8)
+ args = parser.parse_args()
+ if sys.version_info < (3, 12) or args.jobs < 1:
+ parser.error("Python 3.12+ and a positive --jobs value are required")
+ NativeBuild(args, os.environ).build()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/third_party/voice/test_build_native.py b/third_party/voice/test_build_native.py
new file mode 100644
index 0000000000..bf04af8a2c
--- /dev/null
+++ b/third_party/voice/test_build_native.py
@@ -0,0 +1,182 @@
+"""Check native build preconditions and subprocess failure propagation."""
+
+import json
+import os
+from pathlib import Path
+import platform
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+from build_native import NativeBuild, validate_target
+
+
+class NativeBuildTests(unittest.TestCase):
+ def setUp(self):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name)
+ machine = platform.machine().lower()
+ architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get(machine, machine)
+ suffix = {
+ "Darwin": "apple-darwin",
+ "Linux": "unknown-linux-gnu",
+ "Windows": "pc-windows-msvc",
+ }[platform.system()]
+ self.args = SimpleNamespace(
+ target=f"{architecture}-{suffix}",
+ deployment_target="11.0",
+ output=self.root / "build output",
+ archives=self.root / "archives",
+ cc=Path(sys.executable),
+ cxx=Path(sys.executable),
+ cmake=Path(sys.executable),
+ make=Path(sys.executable),
+ pkg_config=Path(sys.executable),
+ shell=Path(sys.executable),
+ bootstrap_make=None,
+ jobs=2,
+ )
+ self.environment = {
+ **os.environ,
+ "INCLUDE": "fixture include",
+ "LIB": "fixture lib",
+ }
+
+ def test_cli_entrypoint_imports_its_sibling_with_safe_path_enabled(self):
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-P",
+ str(Path(__file__).with_name("build_native.py")),
+ "--help",
+ ],
+ cwd=self.root,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("--archives", result.stdout)
+
+ def test_rejects_cross_host_and_musl_builds(self):
+ for target, system, machine, libc in [
+ ("x86_64-pc-windows-msvc", "Darwin", "x86_64", ""),
+ ("aarch64-apple-darwin", "Darwin", "x86_64", ""),
+ ("x86_64-unknown-linux-gnu", "Linux", "x86_64", "musl"),
+ ("x86_64-unknown-linux-musl", "Linux", "x86_64", "musl"),
+ ]:
+ with self.subTest(target=target, system=system):
+ with self.assertRaises(ValueError):
+ validate_target(target, system, machine, libc, "11.0")
+
+ def test_requires_explicit_macos_deployment_target(self):
+ with self.assertRaisesRegex(ValueError, "Declare the macOS deployment target"):
+ validate_target("aarch64-apple-darwin", "Darwin", "arm64", "", None)
+
+ def test_missing_tool_does_not_create_output(self):
+ self.args.cc = self.root / "missing compiler"
+ with self.assertRaisesRegex(ValueError, "Missing build tool"):
+ NativeBuild(self.args, self.environment)
+ self.assertFalse(self.args.output.exists())
+
+ @unittest.skipIf(os.name == "nt", "Exercises the Unix clang++ driver symlink")
+ def test_bootstrap_links_cpp_runtime_through_compiler_symlink(self):
+ tools = {name: shutil.which(name) for name in ("clang", "cmake", "make")}
+ if not all(tools.values()):
+ self.skipTest("Requires clang, CMake and make")
+ compiler = self.root / "clang++"
+ compiler.symlink_to(Path(tools["clang"]).resolve())
+ self.args.cc = Path(tools["clang"])
+ self.args.cxx = compiler
+ self.args.cmake = Path(tools["cmake"])
+ self.args.make = Path(tools["make"])
+ source = self.root / "cpp-source"
+ source.mkdir()
+ (source / "CMakeLists.txt").write_text(
+ "cmake_minimum_required(VERSION 3.15)\n"
+ "project(cpp_driver LANGUAGES CXX)\n"
+ "add_executable(cpp_driver main.cpp)\n"
+ "install(TARGETS cpp_driver DESTINATION bin)\n"
+ )
+ (source / "main.cpp").write_text(
+ '#include <iostream>\nint main() { std::cout << "linked"; }\n'
+ )
+ build = NativeBuild(self.args, self.environment)
+ build.output.mkdir()
+ build.sources = {"cpp-driver": source}
+ build.cmake("cpp-driver", [], bootstrap=True)
+ result = subprocess.run(
+ [build.tools / "bin" / "cpp_driver"],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ self.assertEqual(result.stdout, "linked")
+
+ def test_build_refuses_existing_output_before_reading_sources(self):
+ self.args.output.mkdir()
+ (self.args.output / "keep").write_text("untouched")
+ with self.assertRaises(FileExistsError):
+ NativeBuild(self.args, self.environment).build()
+ self.assertEqual(
+ {p.name: p.read_text() for p in self.args.output.iterdir()},
+ {"keep": "untouched"},
+ )
+
+ def test_failure_retains_log_and_state_without_completion_marker(self):
+ build = NativeBuild(self.args, self.environment)
+ build.output.mkdir()
+ command = [
+ sys.executable,
+ "-c",
+ "print('synthetic failure'); raise SystemExit(23)",
+ ]
+ with self.assertRaises(subprocess.CalledProcessError) as error:
+ build.run("failure", command)
+ self.assertEqual(error.exception.returncode, 23)
+ self.assertEqual(
+ (build.output / "failure.log").read_text().strip(), "synthetic failure"
+ )
+ self.assertEqual(
+ json.loads((build.output / "build-state.json").read_text()),
+ {
+ "target": self.args.target,
+ "deployment_target": "11.0",
+ "steps": [{"name": "failure", "command": command, "exit_code": 23}],
+ },
+ )
+ self.assertFalse((build.output / "built.json").exists())
+
+ def test_ambient_native_discovery_variables_are_not_inherited(self):
+ inherited = {
+ **self.environment,
+ "PKG_CONFIG_PATH": "/ambient",
+ "CMAKE_PREFIX_PATH": "/ambient",
+ "CPATH": "/ambient",
+ "CFLAGS": "-I/ambient",
+ "LDFLAGS": "-L/ambient",
+ }
+ environment = NativeBuild(self.args, inherited).environment
+ self.assertFalse(any("/ambient" in value for value in environment.values()))
+ self.assertEqual(environment["PKG_CONFIG_PATH"], "")
+
+ def test_meson_receives_private_link_inputs_with_spaces(self):
+ build = NativeBuild(self.args, self.environment)
+ build.sources = {"meson": self.root / "meson", "glib": self.root / "glib"}
+ with patch.object(build, "run"):
+ build.meson("glib", [])
+ if build.windows:
+ self.assertEqual(
+ build.environment["LDFLAGS"], f'"/LIBPATH:{build.prefix / "lib"}"'
+ )
+ else:
+ self.assertEqual(
+ shlex.split(build.environment["LDFLAGS"]),
+ [f"-L{build.prefix / 'lib'}", f"-Wl,-rpath,{build.prefix / 'lib'}"],
+ )