Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Link Unix Bazel bindings against the prepared voice runtime (#43117)

Benjamin Carlssonf5a71ff40a71
diff --git a/MODULE.bazel b/MODULE.bazel
index ff5bbf21af..e656eab604 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -647,6 +647,94 @@ crate.annotation(
inject_repo(crate, "alsa_lib")
+# Keep upstream pkg-config version probes, but link only the prepared runtime.
+# system-deps applies these path/flag overrides after a successful version probe.
+[
+ crate.annotation(
+ build_script_data = [
+ "//third_party/voice:native_link",
+ "//third_party/voice:native_sdk",
+ ],
+ build_script_env = {
+ "PKG_CONFIG": "$(execpath @@//third_party/voice:pkg_config)",
+ "PKG_CONFIG_LIBDIR": "$(execpath @@//third_party/voice:native_sdk)/lib/pkgconfig",
+ "PKG_CONFIG_PATH": "",
+ } | {
+ "SYSTEM_DEPS_" + key + "_SEARCH_NATIVE": "$(execpath @@//third_party/voice:native_link)/.."
+ for key in keys
+ } | {
+ # GStreamer's .pc files add an absolute rpath; CcInfo supplies the
+ # relative Bazel runpath instead. Library/version checks still run.
+ "SYSTEM_DEPS_" + key + "_LDFLAGS": ""
+ for key in keys
+ if key.startswith("GSTREAMER_")
+ },
+ build_script_tools = ["//third_party/voice:pkg_config"],
+ crate = name,
+ gen_build_script = "on",
+ version = version,
+ deps = ["//third_party/voice:native_link"],
+ )
+ for name, version, keys in [
+ (
+ "glib-sys",
+ "0.22.8",
+ [
+ "GLIB_2_0",
+ "GOBJECT_2_0",
+ ],
+ ),
+ (
+ "gobject-sys",
+ "0.22.6",
+ ["GOBJECT_2_0"],
+ ),
+ (
+ "gio-sys",
+ "0.22.8",
+ ["GIO_2_0"],
+ ),
+ (
+ "gstreamer-sys",
+ "0.25.2",
+ ["GSTREAMER_1_0"],
+ ),
+ (
+ "gstreamer-base-sys",
+ "0.25.3",
+ ["GSTREAMER_BASE_1_0"],
+ ),
+ (
+ "gstreamer-app-sys",
+ "0.25.0",
+ ["GSTREAMER_APP_1_0"],
+ ),
+ (
+ "gstreamer-audio-sys",
+ "0.25.3",
+ ["GSTREAMER_AUDIO_1_0"],
+ ),
+ (
+ "glib-sys",
+ "0.22.9",
+ [
+ "GLIB_2_0",
+ "GOBJECT_2_0",
+ ],
+ ),
+ (
+ "gobject-sys",
+ "0.22.9",
+ ["GOBJECT_2_0"],
+ ),
+ (
+ "gio-sys",
+ "0.22.9",
+ ["GIO_2_0"],
+ ),
+ ]
+]
+
bazel_dep(name = "v8", version = "15.0.245.2")
archive_override(
module_name = "v8",
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel
index 6c807508a0..7a98b0ec58 100644
--- a/third_party/voice/BUILD.bazel
+++ b/third_party/voice/BUILD.bazel
@@ -1,5 +1,6 @@
load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain")
load(":native.bzl", "native_prefix")
+load(":native_link.bzl", "native_link")
load(":pkg_config.bzl", "pkg_config")
load(":runtime.bzl", "native_runtime")
@@ -82,6 +83,7 @@ pkg_config(
configure_options = [
"--with-internal-glib",
"--disable-shared",
+ "--enable-define-prefix",
],
copts = ["-Wno-int-conversion"],
env = {
@@ -105,6 +107,7 @@ pkg_config(
# Internal GLib needs no pkg-config. Avoid bootstrapping another copy first;
# the explicit failing command above forbids falling back to a host copy.
toolchain = "@rules_foreign_cc//toolchains:preinstalled_pkgconfig_toolchain",
+ visibility = ["//visibility:public"],
)
native_tool_toolchain(
@@ -206,3 +209,35 @@ alias(
tags = ["manual"],
visibility = ["//visibility:public"],
)
+
+filegroup(
+ name = "native_sdk",
+ srcs = [":native_runtime"],
+ output_group = "sdk",
+ tags = ["manual"],
+ visibility = ["//visibility:public"],
+)
+
+[
+ native_link(
+ name = "native_link_" + os + "_" + cpu,
+ runtime = ":native_runtime_" + os + "_" + cpu,
+ tags = ["manual"],
+ target = cpu + "-" + suffix,
+ target_compatible_with = select({
+ ":" + os + "_" + cpu: [],
+ "//conditions:default": ["@platforms//:incompatible"],
+ }),
+ )
+ for os, cpu, suffix in _NATIVE_PLATFORMS
+]
+
+alias(
+ name = "native_link",
+ actual = select({
+ ":" + os + "_" + cpu: ":native_link_" + os + "_" + cpu
+ for os, cpu, _ in _NATIVE_PLATFORMS
+ }),
+ tags = ["manual"],
+ visibility = ["//visibility:public"],
+)
diff --git a/third_party/voice/README.md b/third_party/voice/README.md
index 7488eef434..d04f070633 100644
--- a/third_party/voice/README.md
+++ b/third_party/voice/README.md
@@ -195,6 +195,19 @@ This manual target requires the same host inspection/signing tools as the standa
platform preparer. It does not link Rust, change Windows builds, assemble a CLI
package, or enable voice. The prepared runtime is the input to those later steps.
+The `native_sdk` output exports the existing development SDK from that same
+inspected build. Unix Bazel Rust bindings keep their upstream pkg-config version
+checks, restricted to this SDK and the declared pkg-config executable. The
+supported `system-deps` search-path override directs linking to `native_link`'s
+prepared libraries; its GStreamer linker-flag override removes the SDK's absolute
+rpath. Standard CcInfo supplies relative Bazel runpaths and explicit runfiles.
+Canonical ABI names and development aliases stay together, including transitive
+native dependencies. No host library fallback or version-probe bypass is used.
+Plugins and their manifest are exported beside those same canonical libraries;
+bindings and plugin imports must resolve to one physical copy of each library.
+Cargo still consumes an explicitly supplied SDK; Windows MSVC and final installed
+helper loader paths remain separate packaging steps. This does not enable voice.
+
## Private Windows runtime preparation
`windows_runtime.py` takes the same arguments for x64/ARM64 MSVC build prefixes.
diff --git a/third_party/voice/native_link.bzl b/third_party/voice/native_link.bzl
new file mode 100644
index 0000000000..c3931bd2d7
--- /dev/null
+++ b/third_party/voice/native_link.bzl
@@ -0,0 +1,104 @@
+"""Expose prepared native libraries to standard CcInfo linking and runfiles."""
+
+load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
+load("@rules_cc//cc/common:cc_common.bzl", "cc_common")
+load("@rules_cc//cc/common:cc_info.bzl", "CcInfo")
+
+# ABI filenames from the pinned native sources. Missing outputs fail the build.
+_ABI_VERSIONS = {
+ "ffi": "8",
+ "gio-2.0": "0",
+ "glib-2.0": "0",
+ "gmodule-2.0": "0",
+ "gobject-2.0": "0",
+ "gstapp-1.0": "0",
+ "gstaudio-1.0": "0",
+ "gstbase-1.0": "0",
+ "gstnet-1.0": "0",
+ "gstpbutils-1.0": "0",
+ "gstreamer-1.0": "0",
+ "gstrtp-1.0": "0",
+ "gsttag-1.0": "0",
+ "gstvideo-1.0": "0",
+ "intl": "8",
+ "opus": "0",
+ "pcre2-8": "0",
+ "z": "1",
+}
+
+def _native_link_impl(ctx):
+ runtime = ctx.file.runtime
+ versions = dict(_ABI_VERSIONS)
+ macos = ctx.attr.target.endswith("apple-darwin")
+ if not macos:
+ versions["gstallocators-1.0"] = "0"
+ locator = ctx.actions.declare_directory(ctx.label.name + "/lib/search-path")
+ originals, aliases, libraries = [], [], []
+ arguments = [locator.path]
+ cc = find_cpp_toolchain(ctx)
+ features = cc_common.configure_features(
+ ctx = ctx,
+ cc_toolchain = cc,
+ requested_features = ctx.features,
+ unsupported_features = ctx.disabled_features,
+ )
+ for name, version in versions.items():
+ filename = "lib" + name + ("." + version + ".dylib" if macos else ".so." + version)
+ alias = "lib" + name + (".dylib" if macos else ".so")
+ library = ctx.actions.declare_file(ctx.label.name + "/lib/" + filename)
+ development = ctx.actions.declare_file(ctx.label.name + "/lib/" + alias)
+ originals.append(library)
+ aliases.append(development)
+ arguments.extend([runtime.path + "/lib/" + filename, library.path])
+ arguments.extend([runtime.path + "/lib/" + filename, development.path])
+ libraries.append(cc_common.create_library_to_link(
+ actions = ctx.actions,
+ feature_configuration = features,
+ cc_toolchain = cc,
+ dynamic_library = library,
+ # @loader_path/$ORIGIN dependencies must stay beside one another.
+ dynamic_library_symlink_path = "voice/" + ctx.label.name + "/" + filename,
+ ))
+ payloads = []
+ paths = ["runtime.json"] + [
+ ("plugins/libgst" + plugin + ".dylib" if macos else "lib/gstreamer-1.0/libgst" + plugin + ".so")
+ for plugin in "app audioconvert audioresample coreelements opus rtp rtpmanager".split(" ")
+ ]
+ for path in paths:
+ payload = ctx.actions.declare_file(ctx.label.name + "/" + path)
+ payloads.append(payload)
+ arguments.extend([runtime.path + "/" + path, payload.path])
+ ctx.actions.run_shell(
+ inputs = [runtime],
+ outputs = [locator] + originals + aliases + payloads,
+ arguments = arguments,
+ command = 'set -eu; /bin/mkdir -p "$1"; shift; while [ "$#" -gt 0 ]; do /bin/mkdir -p "${2%/*}"; /bin/cp "$1" "$2"; shift 2; done',
+ mnemonic = "VoiceNativeLinkInputs",
+ )
+ linker = cc_common.create_linker_input(
+ owner = ctx.label,
+ user_link_flags = depset([
+ "-Wl,-rpath," + ("@loader_path/../lib" if macos else "$ORIGIN/../lib"),
+ ]),
+ libraries = depset(libraries),
+ additional_inputs = depset([locator] + aliases),
+ )
+ return [
+ CcInfo(linking_context = cc_common.create_linking_context(linker_inputs = depset([linker]))),
+ DefaultInfo(
+ # A real directory permits unambiguous $(execpath :native_link)/..
+ # in build-script settings; a marker file followed by /.. would fail.
+ files = depset([locator]),
+ runfiles = ctx.runfiles(files = [locator] + originals + aliases + payloads + [lib.dynamic_library for lib in libraries]),
+ ),
+ ]
+
+native_link = rule(
+ implementation = _native_link_impl,
+ attrs = {
+ "runtime": attr.label(mandatory = True, allow_single_file = True),
+ "target": attr.string(mandatory = True),
+ },
+ fragments = ["cpp"],
+ toolchains = ["@bazel_tools//tools/cpp:toolchain_type"],
+)
diff --git a/third_party/voice/prepare_built_runtime.py b/third_party/voice/prepare_built_runtime.py
index 9a070e479b..c773ffbfb0 100644
--- a/third_party/voice/prepare_built_runtime.py
+++ b/third_party/voice/prepare_built_runtime.py
@@ -15,9 +15,10 @@ import tempfile
sys.path.insert(0, str(Path(__file__).resolve().parent))
from runtime import digest
+from sdk import export_sdk
-def prepare_built(prefix, build_receipt, status, target, output):
+def prepare_built(prefix, build_receipt, status, target, output, *, sdk_output=None):
prefix = prefix.resolve(strict=True)
if build_receipt.stat().st_size > 1024 * 1024 or status.stat().st_size > 65536:
raise ValueError("native build metadata exceeds limits")
@@ -86,15 +87,19 @@ def prepare_built(prefix, build_receipt, status, target, output):
)
)
platform.project(prefix, receipts, target, output)
+ if sdk_output is not None:
+ export_sdk(prefix, receipts, target, sdk_output)
-def prepare_archive(archive, build_receipt, status, target, output):
+def prepare_archive(archive, build_receipt, status, target, output, *, sdk_output=None):
# Archives preserve native aliases through Bazel's cache and sandbox links.
with tempfile.TemporaryDirectory(prefix="voice-prefix-") as temporary:
prefix = Path(temporary)
with tarfile.open(archive) as source:
source.extractall(prefix, filter="data")
- prepare_built(prefix, build_receipt, status, target, output)
+ prepare_built(
+ prefix, build_receipt, status, target, output, sdk_output=sdk_output
+ )
if __name__ == "__main__":
@@ -102,12 +107,23 @@ if __name__ == "__main__":
for name in ("prefix", "build-receipt", "status", "output"):
parser.add_argument("--" + name, type=Path, required=True)
parser.add_argument("--target", required=True)
+ parser.add_argument("--sdk-output", type=Path)
args = parser.parse_args()
# Executors may leave the TreeArtifact absent or create an empty directory.
try:
args.output.rmdir()
except FileNotFoundError:
pass
+ if args.sdk_output is not None:
+ try:
+ args.sdk_output.rmdir()
+ except FileNotFoundError:
+ pass
prepare_archive(
- args.prefix, args.build_receipt, args.status, args.target, args.output
+ args.prefix,
+ args.build_receipt,
+ args.status,
+ args.target,
+ args.output,
+ sdk_output=args.sdk_output,
)
diff --git a/third_party/voice/runtime.bzl b/third_party/voice/runtime.bzl
index 3334e3ffd7..4c52fc5a6a 100644
--- a/third_party/voice/runtime.bzl
+++ b/third_party/voice/runtime.bzl
@@ -7,6 +7,7 @@ def _native_runtime_impl(ctx):
prefix = ctx.attr.prefix[DefaultInfo].files.to_list()[0]
receipt = ctx.attr.prefix[OutputGroupInfo].receipt.to_list()[0]
output = ctx.actions.declare_directory(ctx.label.name)
+ sdk = ctx.actions.declare_directory(ctx.label.name + "_sdk")
ctx.actions.run(
executable = python.interpreter,
arguments = [
@@ -21,18 +22,23 @@ def _native_runtime_impl(ctx):
ctx.attr.target,
"--output",
output.path,
+ "--sdk-output",
+ sdk.path,
],
inputs = depset(
[prefix, receipt, ctx.info_file, ctx.file._driver, python.interpreter] + ctx.files._preparers,
transitive = [python.files],
),
- outputs = [output],
+ outputs = [output, sdk],
env = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"},
execution_requirements = {"no-remote-exec": "1", "no-remote-cache": "1"} if ctx.attr.target.endswith("apple-darwin") else {},
mnemonic = "VoiceNativeRuntime",
progress_message = "Preparing private voice runtime for " + ctx.attr.target,
)
- return [DefaultInfo(files = depset([output]))]
+ return [
+ DefaultInfo(files = depset([output])),
+ OutputGroupInfo(sdk = depset([sdk])),
+ ]
native_runtime = rule(
implementation = _native_runtime_impl,
diff --git a/third_party/voice/test_prepare_built_runtime.py b/third_party/voice/test_prepare_built_runtime.py
index 6aa51d0884..e9bc654cdd 100644
--- a/third_party/voice/test_prepare_built_runtime.py
+++ b/third_party/voice/test_prepare_built_runtime.py
@@ -4,6 +4,7 @@ import json
from pathlib import Path
import sys
import subprocess
+import shutil
import tarfile
import tempfile
import unittest
@@ -13,6 +14,7 @@ import macos_runtime
from prepare_built_runtime import prepare_archive, prepare_built
from runtime import digest
import test_macos_runtime
+import test_sdk
class BuiltRuntimeTests(unittest.TestCase):
@@ -124,6 +126,11 @@ class BuiltRuntimeTests(unittest.TestCase):
fixture = test_macos_runtime.RuntimeTests("runTest")
self.addCleanup(fixture.doCleanups)
fixture.setUp()
+ sdk_fixture = test_sdk.SdkTests("runTest")
+ self.addCleanup(sdk_fixture.doCleanups)
+ sdk_fixture.setUp()
+ for relative in ("include", "lib/glib-2.0", "lib/pkgconfig"):
+ shutil.copytree(sdk_fixture.prefix / relative, fixture.prefix / relative)
self.receipt.write_text(json.dumps({**self.build, "target": fixture.target}))
library = next((fixture.prefix / "lib").glob("*.dylib"))
(library.parent / "development-alias.dylib").symlink_to(library.name)
@@ -137,6 +144,9 @@ class BuiltRuntimeTests(unittest.TestCase):
output = self.root / f"runtime-{state}"
if state == "empty":
output.mkdir()
+ sdk_output = self.root / f"sdk-{state}"
+ if state == "empty":
+ sdk_output.mkdir()
subprocess.run(
[
sys.executable,
@@ -151,10 +161,28 @@ class BuiltRuntimeTests(unittest.TestCase):
fixture.target,
"--output",
str(output),
+ "--sdk-output",
+ str(sdk_output),
],
check=True,
)
+ sdk_manifest = json.loads((sdk_output / "sdk.json").read_text())
+ self.assertEqual(
+ {
+ record["path"]: record["sha256"]
+ for record in sdk_manifest["files"]
+ },
+ {
+ path.relative_to(sdk_output).as_posix(): digest(path)
+ for path in sdk_output.rglob("*")
+ if path.is_file() and path.name != "sdk.json"
+ },
+ )
manifest = json.loads((output / "runtime.json").read_text())
+ self.assertEqual(
+ (sdk_manifest["sourceCommit"], sdk_manifest["target"]),
+ (manifest["sourceCommit"], manifest["target"]),
+ )
self.assertEqual(manifest["sourceCommit"], "a" * 40)
self.assertEqual(manifest["target"], fixture.target)
self.assertTrue(