Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add receipt-verified native voice SDK export (#43099)

Benjamin Carlsson5d358057152d
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel
index 1a6c678b8b..85feb73574 100644
--- a/third_party/voice/BUILD.bazel
+++ b/third_party/voice/BUILD.bazel
@@ -39,6 +39,7 @@ filegroup(
"macos_runtime.py",
"package_runtime.py",
"runtime.py",
+ "sdk.py",
"windows_runtime.py",
":source_inputs",
],
diff --git a/third_party/voice/README.md b/third_party/voice/README.md
index 50df282e25..61b3281c6f 100644
--- a/third_party/voice/README.md
+++ b/third_party/voice/README.md
@@ -50,6 +50,24 @@ a shared Opus build with Rust consumers or a reduced dependency count.
Rust `opus` 0.4.0 is available through Socket. Adding Rust transport dependencies
and establishing a shared Opus build remain separate integration work.
+## Native development inputs
+
+The private native CI job also emits `sdk.tar.gz` from the same inspected prefix.
+It contains headers (including the target's GLib configuration), development
+library names, import/static libraries and pkg-config metadata. `sdk.json`
+records the target, source commit, pinned manifest and every exported file hash.
+Shared-library bytes must match the native inspection receipt; other development
+files are hashed during export. This is provenance, not an authenticity check.
+
+Meson generates relocatable pkg-config metadata using its standard option.
+Consumers must restrict `PKG_CONFIG_LIBDIR` to the SDK, clear `PKG_CONFIG_PATH`,
+and use `pkg-config --define-prefix` for libffi/PCRE2/zlib metadata too. Only the
+required native metadata is exported; capture Opus uses `opusic-sys`. Native
+library loader paths are not changed by SDK export. These build inputs do not replace the
+separate runtime projection and are never copied into users' Codex packages.
+Cargo/Bazel native providers, final helper linkage and moved-package execution
+remain separate integration work; exporting an SDK does not enable voice.
+
## Native build recipe
`build_native.py` runs the unmodified upstream build systems in a new output
diff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.py
index adb75b6e4a..3ae5669557 100644
--- a/third_party/voice/build_native.py
+++ b/third_party/voice/build_native.py
@@ -248,6 +248,7 @@ class NativeBuild:
"--wrap-mode=nofallback",
"-Dauto_features=disabled",
"-Ddefault_library=shared",
+ "-Dpkgconfig.relocatable=true",
*options,
],
)
diff --git a/third_party/voice/sdk.py b/third_party/voice/sdk.py
new file mode 100644
index 0000000000..bc7f05037b
--- /dev/null
+++ b/third_party/voice/sdk.py
@@ -0,0 +1,140 @@
+"""Export development inputs from the same inspected build as the private runtime.
+
+SDKs are build inputs, never installed user payloads. Their receipt records byte
+identity and provenance, not authenticity or approval. Native loader relocation
+and final helper linking remain the runtime preparer and consumer's jobs.
+"""
+
+import json
+from pathlib import Path
+import re
+import shutil
+
+from runtime import digest
+
+MODULES = (
+ "glib-2.0",
+ "gobject-2.0",
+ "gmodule-no-export-2.0",
+ "gstreamer-1.0",
+ "gstreamer-base-1.0",
+ "gstreamer-app-1.0",
+ "gstreamer-audio-1.0",
+ "gstreamer-tag-1.0",
+)
+
+
+def export_sdk(prefix: Path, receipts: Path, target: str, output: Path):
+ prefix, receipts = prefix.resolve(strict=True), receipts.resolve(strict=True)
+ output = output.absolute()
+ if (
+ output.exists()
+ or output.is_symlink()
+ or any(output.resolve().is_relative_to(root) for root in (prefix, receipts))
+ ):
+ raise ValueError("SDK output must be fresh and outside the inputs")
+ if not re.fullmatch(
+ r"(?:aarch64|x86_64)-(?:apple-darwin|unknown-linux-gnu|pc-windows-msvc)",
+ target,
+ ):
+ raise ValueError("unsupported SDK target")
+ ci = json.loads((receipts / "ci.json").read_text())
+ source_hash = digest(Path(__file__).with_name("sources.json"))
+ if (
+ ci.get("target") != target
+ or ci.get("build_complete") is not True
+ or ci.get("inspection_complete") is not True
+ or ci.get("manifest_sha256") != source_hash
+ or not re.fullmatch(r"[0-9a-f]{40}", ci.get("commit", ""))
+ ):
+ raise ValueError("SDK build receipt does not match the sources and target")
+ inventory = json.loads((receipts / "inspection/binaries.json").read_text())
+ if not 1 <= len(inventory) <= 128:
+ raise ValueError("unexpected native inventory size")
+ binaries = {}
+ for record in inventory:
+ name = record["path"].replace("\\", "/")
+ if not re.fullmatch(r"(?:lib(?:/gstreamer-1\.0)?|bin)/[A-Za-z0-9_+.-]+", name):
+ raise ValueError("invalid native inventory path")
+ path = prefix / name
+ if (
+ name in binaries
+ or path.is_symlink()
+ or not path.resolve(strict=True).is_relative_to(prefix)
+ or record["target"] != target
+ or digest(path) != record["sha256"]
+ ):
+ raise ValueError("native SDK library does not match its receipt")
+ binaries[name] = record["sha256"]
+ selected = []
+ names = set()
+ size = 0
+ for path in sorted(prefix.rglob("*")):
+ name = path.relative_to(prefix).as_posix()
+ if path.is_dir():
+ if path.is_symlink():
+ raise ValueError("SDK input directories must not be links")
+ continue
+ if not (
+ name.startswith("include/")
+ or name == "lib/glib-2.0/include/glibconfig.h"
+ or name
+ in {
+ f"lib/pkgconfig/{module}.pc"
+ for module in (*MODULES, "libffi", "libpcre2-8", "zlib")
+ }
+ or re.fullmatch(
+ r"lib/[A-Za-z0-9_+.-]+\.(?:a|lib|dylib|so(?:\.[0-9]+)*)", name
+ )
+ ):
+ continue
+ source = path.resolve(strict=True)
+ if not source.is_relative_to(prefix) or not source.is_file():
+ raise ValueError("SDK inputs must be files inside the prefix")
+ if name.casefold() in names:
+ raise ValueError("SDK paths collide across platforms")
+ names.add(name.casefold())
+ size += source.stat().st_size
+ if len(selected) >= 4096 or size > 512 * 1024 * 1024:
+ raise ValueError("SDK inputs exceed the size limit")
+ source_name = source.relative_to(prefix).as_posix()
+ expected = digest(source)
+ if re.search(r"\.(?:dylib|so(?:\.[0-9]+)*)$", name):
+ if binaries.get(source_name) != expected:
+ raise ValueError("SDK shared library is missing from the receipt")
+ selected.append((name, source, expected))
+ for module in MODULES:
+ path = prefix / f"lib/pkgconfig/{module}.pc"
+ if path.relative_to(prefix).as_posix().casefold() not in names or not re.search(
+ r"^prefix=\$\{pcfiledir\}/\.\./\.\.$", path.read_text(), re.MULTILINE
+ ):
+ raise ValueError("rebuild the SDK with relocatable Meson pkg-config files")
+ if "lib/glib-2.0/include/glibconfig.h" not in names:
+ raise ValueError("SDK is missing the target's GLib configuration header")
+ output.mkdir()
+ try:
+ files = []
+ for name, source, expected in selected:
+ destination = output / name
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(source, destination)
+ if digest(destination) != expected:
+ raise ValueError("SDK input changed while copying")
+ files.append({"path": name, "sha256": expected})
+ (output / "sdk.json").write_text(
+ json.dumps(
+ {
+ "schemaVersion": 1,
+ "target": target,
+ "sourceCommit": ci["commit"],
+ "sourceManifestSha256": source_hash,
+ "files": files,
+ },
+ indent=2,
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ except BaseException:
+ shutil.rmtree(output)
+ raise
diff --git a/third_party/voice/test_sdk.py b/third_party/voice/test_sdk.py
new file mode 100644
index 0000000000..e31d543d04
--- /dev/null
+++ b/third_party/voice/test_sdk.py
@@ -0,0 +1,153 @@
+"""Exercise exported bytes, development aliases and moved pkg-config inputs."""
+
+import json
+import os
+from pathlib import Path
+import shlex
+import shutil
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from runtime import digest
+from sdk import MODULES, export_sdk
+
+
+class SdkTests(unittest.TestCase):
+ def setUp(self):
+ temporary = tempfile.TemporaryDirectory(prefix="voice SDK ")
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name)
+ self.prefix = self.root / "prefix"
+ self.receipts = self.root / "receipts"
+ self.output = self.root / "sdk"
+ self.target = "aarch64-apple-darwin"
+ for name in ("include/glib-2.0", "lib/glib-2.0/include", "lib/pkgconfig"):
+ (self.prefix / name).mkdir(parents=True)
+ (self.prefix / "include/glib-2.0/glib.h").write_text("/* SDK fixture */")
+ (self.prefix / "lib/glib-2.0/include/glibconfig.h").write_text("/* target */")
+ self.library = self.prefix / "lib/libfixture.0.dylib"
+ self.library.write_bytes(b"receipt-verified fixture bytes")
+ for module in (*MODULES, "zlib"):
+ (self.prefix / f"lib/pkgconfig/{module}.pc").write_text(
+ "prefix=${pcfiledir}/../..\n"
+ f"Name: {module}\nDescription: SDK fixture\nVersion: 1.0\n"
+ "Cflags: -I${prefix}/include\nLibs: -L${prefix}/lib -lfixture\n"
+ )
+ audio = self.prefix / "lib/pkgconfig/gstreamer-audio-1.0.pc"
+ audio.write_text(
+ audio.read_text() + "Requires.private: gstreamer-tag-1.0, zlib\n"
+ )
+ (self.receipts / "inspection").mkdir(parents=True)
+ self.ci = {
+ "target": self.target,
+ "build_complete": True,
+ "inspection_complete": True,
+ "manifest_sha256": digest(Path(__file__).with_name("sources.json")),
+ "commit": "a" * 40,
+ }
+ (self.receipts / "ci.json").write_text(json.dumps(self.ci))
+ (self.receipts / "inspection/binaries.json").write_text(
+ json.dumps(
+ [
+ {
+ "path": "lib/libfixture.0.dylib",
+ "target": self.target,
+ "sha256": digest(self.library),
+ }
+ ]
+ )
+ )
+
+ def test_exported_bytes_and_receipt_survive_move_without_original(self):
+ export_sdk(self.prefix, self.receipts, self.target, self.output)
+ moved = self.root / "moved SDK"
+ self.output.rename(moved)
+ shutil.rmtree(self.prefix)
+ manifest = json.loads((moved / "sdk.json").read_text())
+ actual = {
+ path.relative_to(moved).as_posix(): digest(path)
+ for path in moved.rglob("*")
+ if path.is_file() and path.name != "sdk.json"
+ }
+ self.assertEqual(
+ manifest,
+ {
+ "schemaVersion": 1,
+ "target": self.target,
+ "sourceCommit": "a" * 40,
+ "sourceManifestSha256": self.ci["manifest_sha256"],
+ "files": [
+ {"path": name, "sha256": actual[name]} for name in sorted(actual)
+ ],
+ },
+ )
+ pkg_config = os.environ.get("VOICE_PKG_CONFIG") or shutil.which("pkg-config")
+ if not pkg_config:
+ self.skipTest("pkg-config required for the moved development-input probe")
+ result = subprocess.check_output(
+ [
+ pkg_config,
+ "--define-prefix",
+ "--cflags",
+ "--libs",
+ "gstreamer-audio-1.0",
+ ],
+ env={
+ **os.environ,
+ "PKG_CONFIG_PATH": "",
+ "PKG_CONFIG_LIBDIR": str(moved / "lib/pkgconfig"),
+ },
+ text=True,
+ ).strip()
+ self.assertEqual(
+ shlex.split(result),
+ [f"-I{moved.as_posix()}/include", f"-L{moved.as_posix()}/lib", "-lfixture"],
+ )
+
+ def test_native_mutation_and_wrong_target_are_rejected(self):
+ with self.assertRaisesRegex(ValueError, "sources and target"):
+ export_sdk(self.prefix, self.receipts, "x86_64-apple-darwin", self.output)
+ self.library.write_bytes(b"changed")
+ with self.assertRaisesRegex(ValueError, "does not match"):
+ export_sdk(self.prefix, self.receipts, self.target, self.output)
+ self.assertFalse(self.output.exists())
+
+ def test_development_alias_is_materialized_and_escape_is_rejected(self):
+ alias = self.prefix / "lib/libfixture.dylib"
+ try:
+ alias.symlink_to(self.library.name)
+ except OSError:
+ self.skipTest("host cannot create the SDK's development symlink")
+ export_sdk(self.prefix, self.receipts, self.target, self.output)
+ self.assertFalse((self.output / "lib/libfixture.dylib").is_symlink())
+ self.assertEqual(
+ (self.output / "lib/libfixture.dylib").read_bytes(),
+ self.library.read_bytes(),
+ )
+ shutil.rmtree(self.output)
+ alias.unlink()
+ alias.symlink_to(self.receipts / "ci.json")
+ with self.assertRaisesRegex(ValueError, "inside the prefix"):
+ export_sdk(self.prefix, self.receipts, self.target, self.output)
+
+ def test_nonrelocatable_metadata_and_failed_copy_leave_no_output(self):
+ metadata = self.prefix / "lib/pkgconfig/gstreamer-1.0.pc"
+ original = metadata.read_text()
+ metadata.write_text(original.replace("${pcfiledir}/../..", "/old/build"))
+ with self.assertRaisesRegex(ValueError, "rebuild the SDK"):
+ export_sdk(self.prefix, self.receipts, self.target, self.output)
+ metadata.write_text(original)
+ with patch("sdk.shutil.copy2", side_effect=OSError("copy failed")):
+ with self.assertRaises(OSError):
+ export_sdk(self.prefix, self.receipts, self.target, self.output)
+ self.assertFalse(self.output.exists())
+
+ def test_output_cannot_overwrite_or_nest_inside_inputs(self):
+ for output in (self.prefix, self.prefix / "sdk", self.receipts / "sdk"):
+ with (
+ self.subTest(output=output),
+ self.assertRaisesRegex(ValueError, "fresh"),
+ ):
+ export_sdk(self.prefix, self.receipts, self.target, output)