Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add a Bazel target for native voice dependencies (#43111)

Benjamin Carlssona31c18ab7a5d
diff --git a/.bazelrc b/.bazelrc
index 2ae73f52b7..0aec7b8953 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -83,6 +83,8 @@ common --jobs=30
# configurations include this group; CI configs override TestRunner below
# when tests must remain local on their runner.
common:remote --strategy=remote
+# Native Mac voice actions need the local Apple tools; Linux remains remote.
+common:macos --strategy=VoiceNativePrefix=remote,sandboxed,local
common:remote --extra_execution_platforms=//:rbe
common:remote --jobs=800
# TODO(team): Evaluate if this actually helps, zbarsky is not sure, everything seems bottlenecked on `core` either way.
diff --git a/MODULE.bazel b/MODULE.bazel
index 7c3b278833..ff5bbf21af 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -110,6 +110,10 @@ osx.frameworks(names = [
"AVFAudio",
"AVRouting",
"CFNetwork",
+ "Cocoa",
+ "CoreData",
+ "PrintCore",
+ "Symbols",
"FontServices",
"AudioUnit",
"CoreAudio",
@@ -157,6 +161,13 @@ single_version_override(
version = "0.15.1",
)
+bazel_dep(name = "rules_python", version = "1.7.0")
+
+voice_python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+voice_python.defaults(python_version = "3.11")
+voice_python.toolchain(python_version = "3.12")
+use_repo(voice_python, "python_3_12")
+
# Use the existing pinned Make build for foreign_cc consumers, including the
# native voice pkg-config bootstrap, instead of a PATH-provided Make.
register_toolchains("@rules_foreign_cc//toolchains:built_make_toolchain")
@@ -491,6 +502,17 @@ include("//bazel/modules:wine.MODULE.bazel")
voice_sources = use_extension("//third_party/voice:extensions.bzl", "voice_sources")
use_repo(
voice_sources,
+ "voice_archive_glib",
+ "voice_archive_gst_plugins_base",
+ "voice_archive_gst_plugins_good",
+ "voice_archive_gstreamer",
+ "voice_archive_libffi",
+ "voice_archive_meson",
+ "voice_archive_ninja",
+ "voice_archive_opus",
+ "voice_archive_pcre2",
+ "voice_archive_proxy_libintl",
+ "voice_archive_zlib",
"voice_glib",
"voice_gst_plugins_base",
"voice_gst_plugins_good",
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel
index e98c672033..87ea515b98 100644
--- a/third_party/voice/BUILD.bazel
+++ b/third_party/voice/BUILD.bazel
@@ -1,4 +1,5 @@
load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain")
+load(":native.bzl", "native_prefix")
load(":pkg_config.bzl", "pkg_config")
# All native build consumers must receive the same pinned source manifest.
@@ -49,6 +50,14 @@ filegroup(
visibility = ["//visibility:public"],
)
+filegroup(
+ name = "native_recipe",
+ srcs = [
+ "build_native.py",
+ ":source_inputs",
+ ],
+)
+
alias(
name = "pkg_config_linker",
actual = select({
@@ -103,3 +112,68 @@ native_tool_toolchain(
tags = ["manual"],
target = ":pkg_config",
)
+
+filegroup(
+ name = "archives",
+ srcs = [
+ "@voice_archive_glib//file",
+ "@voice_archive_gst_plugins_base//file",
+ "@voice_archive_gst_plugins_good//file",
+ "@voice_archive_gstreamer//file",
+ "@voice_archive_libffi//file",
+ "@voice_archive_meson//file",
+ "@voice_archive_ninja//file",
+ "@voice_archive_opus//file",
+ "@voice_archive_pcre2//file",
+ "@voice_archive_proxy_libintl//file",
+ "@voice_archive_zlib//file",
+ ],
+ tags = ["manual"],
+)
+
+# Configure probes execute target binaries: constrain each action's host too.
+_NATIVE_PLATFORMS = [
+ ("macos", "aarch64", "apple-darwin"),
+ ("macos", "x86_64", "apple-darwin"),
+ ("linux", "aarch64", "unknown-linux-gnu"),
+ ("linux", "x86_64", "unknown-linux-gnu"),
+]
+
+[
+ config_setting(
+ name = os + "_" + cpu,
+ constraint_values = [
+ "@platforms//os:" + os,
+ "@platforms//cpu:" + cpu,
+ ] + (["@llvm//constraints/libc:gnu.2.28"] if os == "linux" else []),
+ )
+ for os, cpu, _ in _NATIVE_PLATFORMS
+]
+
+[
+ native_prefix(
+ name = "native_prefix_" + os + "_" + cpu,
+ archives = [":archives"],
+ exec_compatible_with = [
+ "@platforms//os:" + os,
+ "@platforms//cpu:" + cpu,
+ ],
+ tags = ["manual"],
+ target = cpu + "-" + suffix,
+ target_compatible_with = select({
+ ":" + os + "_" + cpu: [],
+ "//conditions:default": ["@platforms//:incompatible"],
+ }),
+ )
+ for os, cpu, suffix in _NATIVE_PLATFORMS
+]
+
+alias(
+ name = "native_prefix",
+ actual = select({
+ ":" + os + "_" + cpu: ":native_prefix_" + os + "_" + cpu
+ for os, cpu, _ in _NATIVE_PLATFORMS
+ }),
+ tags = ["manual"],
+ visibility = ["//visibility:public"],
+)
diff --git a/third_party/voice/README.md b/third_party/voice/README.md
index 7bf06470ea..a4b7499e58 100644
--- a/third_party/voice/README.md
+++ b/third_party/voice/README.md
@@ -65,8 +65,8 @@ and use `pkg-config --define-prefix` for libffi/PCRE2/zlib metadata too. Only th
required native metadata is exported; capture Opus uses `opusic-sys`. Native
library loader paths are not changed by SDK export. These build inputs do not replace the
separate runtime projection and are never copied into users' Codex packages.
-Cargo/Bazel native providers, final helper linkage and moved-package execution
-remain separate integration work; exporting an SDK does not enable voice.
+Final helper linkage and moved-package execution remain separate integration
+work; exporting an SDK does not enable voice.
The Rust GStreamer bindings also link GLib's GIO library. Its pkg-config metadata
is included in the SDK, and runtime preparation treats its loader identity as an
@@ -76,6 +76,25 @@ transitive imports must satisfy the same private-library and system-import check
## Native build recipe
+`bazel build //third_party/voice:native_prefix` runs this same recipe with
+Bazel-declared source archives, C/C++ compiler and SDK flags, LLVM ranlib,
+CMake, Make, pkg-config and Python. It supports matching native macOS and Linux
+GNU 2.28 toolchain targets on x64/ARM64; Windows continues to use the standalone
+recipe below. The macOS minimum comes from the selected CcToolchain. This manual
+target is not part of ordinary wildcard CLI builds.
+Mac libffi partial links require Apple's `/usr/bin/ld` from Command Line Tools;
+the LLVM compiler and normal final-link flags remain in use.
+
+The root registers the existing pinned built-Make toolchain for all foreign_cc
+Make consumers, including pkg-config's bootstrap, instead of selecting host Make.
+The recipe still needs the host's standard Unix utilities and `/bin/bash`;
+it is not a fully hermetic build.
+
+The target exports raw `prefix.tar` and a `built.json` receipt (the `receipt`
+output group). These are build inputs, not a relocatable SDK or runtime package.
+No Rust GStreamer build-script annotations or helper loader-path changes are
+provided by this target. Upstream build failure logs appear in the action output.
+
`build_native.py` runs the unmodified upstream build systems in a new output
directory, using the same archives. Unix builds accept repeated `--c-flag=...`,
`--cxx-flag=...`, `--link-flag=...` and optional `--ar`/`--ranlib` inputs.
diff --git a/third_party/voice/bazel_native.py b/third_party/voice/bazel_native.py
new file mode 100644
index 0000000000..c51eb058ca
--- /dev/null
+++ b/third_party/voice/bazel_native.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Run the existing native recipe with Bazel-declared inputs; export a raw prefix."""
+
+import json
+import os
+from pathlib import Path
+import shutil
+import sys
+import tarfile
+import tempfile
+from types import SimpleNamespace
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from build_native import NativeBuild
+
+
+def main():
+ config = json.loads(Path(sys.argv[1]).read_text())
+ root = str(Path.cwd())
+ # Toolchain flags may name execroot-relative SDK files even though upstream
+ # build systems change directory. Substitute a literal marker, never a shell.
+ config = json.loads(json.dumps(config).replace("@VOICE_EXECROOT@", root))
+ with tempfile.TemporaryDirectory(prefix="voice-native-") as temporary:
+ temporary = Path(temporary)
+ archives = temporary / "archives"
+ archives.mkdir()
+ for archive in config.pop("archives"):
+ source = Path(archive)
+ shutil.copyfile(source, archives / source.name)
+ prefix = Path(config.pop("prefix")).absolute()
+ receipt = Path(config.pop("receipt")).absolute()
+ ld = Path(config.pop("ld")).absolute()
+ for name in (
+ "cc",
+ "cxx",
+ "ar",
+ "ranlib",
+ "cmake",
+ "make",
+ "pkg_config",
+ "shell",
+ ):
+ config[name] = Path(config[name]).absolute()
+ args = SimpleNamespace(
+ **config,
+ archives=archives,
+ output=temporary / "build",
+ bootstrap_make=None,
+ )
+ builder = NativeBuild(args, os.environ)
+ # Libtool probes the raw linker separately from compiler link flags.
+ builder.environment["LD"] = str(ld)
+ try:
+ builder.build()
+ except Exception:
+ # Preserve the failing upstream diagnostic in the Bazel action log.
+ if builder.record["steps"]:
+ log = builder.output / (builder.record["steps"][-1]["name"] + ".log")
+ if log.is_file():
+ print(log.read_text(errors="replace"), file=sys.stderr)
+ raise
+ with tarfile.open(prefix, "w") as archive:
+ archive.add(builder.prefix, arcname=".")
+ shutil.copyfile(builder.output / "built.json", receipt)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.py
index 183043516d..29d875874f 100644
--- a/third_party/voice/build_native.py
+++ b/third_party/voice/build_native.py
@@ -379,6 +379,9 @@ class NativeBuild:
}
)
else:
+ if self.args.target.endswith("apple-darwin"):
+ # Libtool's partial links need -r, which ld64.lld does not support.
+ environment["CC"] += " -fuse-ld=/usr/bin/ld"
for name in ("ar", "ranlib"):
if name in self.toolchain:
path = str(self.toolchain[name])
@@ -408,6 +411,13 @@ class NativeBuild:
+ "\n"
)
environment[name] = f"@{response}"
+ if self.args.target.endswith("unknown-linux-gnu"):
+ # Libtool otherwise drops Clang's declared CRT/runtime options.
+ environment["AM_LTLDFLAGS"] = shlex.join(
+ argument
+ for flag in self.flags["LDFLAGS"]
+ for argument in ("-Xcompiler", flag)
+ )
self.run(
"libffi-configure",
[
diff --git a/third_party/voice/extensions.bzl b/third_party/voice/extensions.bzl
index 38aa4401f0..5d209ac7ff 100644
--- a/third_party/voice/extensions.bzl
+++ b/third_party/voice/extensions.bzl
@@ -1,6 +1,6 @@
"""Use standard Bazel archives for the standalone builder's pinned sources."""
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
_BUILD_FILE = """
filegroup(
@@ -13,6 +13,12 @@ filegroup(
def _voice_sources_impl(module_ctx):
manifest = json.decode(module_ctx.read(Label("//third_party/voice:sources.json")))
for source in manifest["sources"]:
+ http_file(
+ name = "voice_archive_" + source["name"].replace("-", "_"),
+ urls = [source["url"]],
+ sha256 = source["sha256"],
+ downloaded_file_path = source["archive"],
+ )
http_archive(
name = "voice_" + source["name"].replace("-", "_"),
urls = [source["url"]],
diff --git a/third_party/voice/native.bzl b/third_party/voice/native.bzl
new file mode 100644
index 0000000000..b70c6b8cc2
--- /dev/null
+++ b/third_party/voice/native.bzl
@@ -0,0 +1,109 @@
+"""Build-only native voice prefix using the same recipe as release packaging."""
+
+load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
+load("@rules_cc//cc/common:cc_common.bzl", "cc_common")
+load("@rules_foreign_cc//foreign_cc/private:cc_toolchain_util.bzl", "absolutize_path_in_str", "get_env_vars", "get_flags_info", "get_tools_info")
+load("@rules_python//python:py_runtime_info.bzl", "PyRuntimeInfo")
+
+def _native_prefix_impl(ctx):
+ cc = find_cpp_toolchain(ctx)
+ features = cc_common.configure_features(
+ ctx = ctx,
+ cc_toolchain = cc,
+ requested_features = ctx.features,
+ unsupported_features = ctx.disabled_features,
+ )
+ runtime = cc.static_runtime_lib(feature_configuration = features)
+ tools = get_tools_info(ctx)
+ flags = get_flags_info(ctx)
+ minimums = [flag.removeprefix("-mmacosx-version-min=") for flag in flags.cc if flag.startswith("-mmacosx-version-min=")]
+ if ctx.attr.target.endswith("apple-darwin") and len(minimums) != 1:
+ fail("The native prefix requires one macOS minimum from the CcToolchain")
+ python = ctx.attr._python[PyRuntimeInfo]
+ if not python.interpreter:
+ fail("The native prefix requires a declared Python interpreter")
+ prefix = ctx.actions.declare_file(ctx.label.name + "/prefix.tar")
+ receipt = ctx.actions.declare_file(ctx.label.name + "/built.json")
+ config = ctx.actions.declare_file(ctx.label.name + ".json")
+ values = {
+ "archives": [file.path for file in ctx.files.archives],
+ "prefix": prefix.path,
+ "receipt": receipt.path,
+ "target": ctx.attr.target,
+ "deployment_target": minimums[0] if minimums else None,
+ "jobs": 8,
+ "cc": tools.cc,
+ "cxx": tools.cxx,
+ "ar": tools.cxx_linker_static,
+ "ranlib": ctx.executable._ranlib.path,
+ "ld": ctx.executable._ld.path,
+ "shell": "/bin/bash",
+ "pkg_config": ctx.file._pkg_config.path,
+ }
+ inputs = [cc.all_files, python.files, runtime, ctx.attr._pkg_config.files]
+ for name in ("cmake", "make"):
+ tool = ctx.toolchains["@rules_foreign_cc//toolchains:" + name + "_toolchain"].data
+ if not tool.target:
+ fail("The native prefix requires a declared " + name + " tool")
+ values[name] = tool.path
+
+ # Prebuilt tools expose a package-relative path; built tools already
+ # include their tree artifact. Match the upstream tool-access convention.
+ for file in tool.target.files.to_list():
+ if file.path.endswith("/" + tool.path):
+ values[name] = file.path
+ break
+ inputs.append(tool.target.files)
+ for name, arguments in {
+ "c_flag": flags.cc,
+ "cxx_flag": flags.cxx,
+ # Upstream selects executable vs shared output. Retain their common
+ # driver/SDK flags without forcing one output kind on configure probes.
+ "link_flag": [flag for flag in flags.cxx_linker_shared if flag in flags.cxx_linker_executable] + [file.path for file in runtime.to_list()],
+ }.items():
+ values[name] = [absolutize_path_in_str(ctx.workspace_name, "@VOICE_EXECROOT@/", flag) for flag in arguments]
+ ctx.actions.write(config, json.encode(values))
+ ctx.actions.run(
+ executable = python.interpreter,
+ arguments = [ctx.file._driver.path, config.path],
+ inputs = depset(
+ [config, ctx.file._driver, python.interpreter] + ctx.files.archives + ctx.files._recipe,
+ transitive = inputs,
+ ),
+ tools = [
+ ctx.attr._ranlib[DefaultInfo].files_to_run,
+ ctx.attr._ld[DefaultInfo].files_to_run,
+ ],
+ outputs = [prefix, receipt],
+ env = get_env_vars(ctx),
+ execution_requirements = {
+ "no-remote-exec": "1",
+ "no-remote-cache": "1",
+ } if ctx.attr.target.endswith("apple-darwin") else {},
+ mnemonic = "VoiceNativePrefix",
+ progress_message = "Building native voice prefix for " + ctx.attr.target,
+ )
+ return [
+ DefaultInfo(files = depset([prefix])),
+ OutputGroupInfo(receipt = depset([receipt])),
+ ]
+
+native_prefix = rule(
+ implementation = _native_prefix_impl,
+ attrs = {
+ "archives": attr.label_list(allow_files = True, mandatory = True),
+ "target": attr.string(mandatory = True),
+ "_driver": attr.label(default = "//third_party/voice:bazel_native.py", allow_single_file = True),
+ "_recipe": attr.label(default = "//third_party/voice:native_recipe"),
+ "_python": attr.label(default = "@python_3_12//:py3_runtime", cfg = "exec"),
+ "_ranlib": attr.label(default = "@llvm//tools:llvm-ranlib", executable = True, allow_files = True, cfg = "exec"),
+ "_ld": attr.label(default = "//third_party/voice:pkg_config_linker", executable = True, allow_files = True, cfg = "exec"),
+ "_pkg_config": attr.label(default = "//third_party/voice:pkg_config", allow_single_file = True, cfg = "exec"),
+ },
+ fragments = ["cpp"],
+ toolchains = [
+ "@bazel_tools//tools/cpp:toolchain_type",
+ "@rules_foreign_cc//toolchains:cmake_toolchain",
+ "@rules_foreign_cc//toolchains:make_toolchain",
+ ],
+)