Lightweight coding agent that runs in your terminal
Rust
Add receipt-verified native voice SDK export (#43099)
Benjamin Carlsson5d358057152d
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazelindex 1a6c678b8b..85feb73574 100644--- a/third_party/voice/BUILD.bazel+++ b/third_party/voice/BUILD.bazel@@ -39,6 +39,7 @@ filegroup("macos_runtime.py","package_runtime.py","runtime.py",+ "sdk.py","windows_runtime.py",":source_inputs",],diff --git a/third_party/voice/README.md b/third_party/voice/README.mdindex 50df282e25..61b3281c6f 100644--- a/third_party/voice/README.md+++ b/third_party/voice/README.md@@ -50,6 +50,24 @@ a shared Opus build with Rust consumers or a reduced dependency count.Rust `opus` 0.4.0 is available through Socket. Adding Rust transport dependenciesand establishing a shared Opus build remain separate integration work.+## Native development inputs++The private native CI job also emits `sdk.tar.gz` from the same inspected prefix.+It contains headers (including the target's GLib configuration), development+library names, import/static libraries and pkg-config metadata. `sdk.json`+records the target, source commit, pinned manifest and every exported file hash.+Shared-library bytes must match the native inspection receipt; other development+files are hashed during export. This is provenance, not an authenticity check.++Meson generates relocatable pkg-config metadata using its standard option.+Consumers must restrict `PKG_CONFIG_LIBDIR` to the SDK, clear `PKG_CONFIG_PATH`,+and use `pkg-config --define-prefix` for libffi/PCRE2/zlib metadata too. Only the+required native metadata is exported; capture Opus uses `opusic-sys`. Native+library loader paths are not changed by SDK export. These build inputs do not replace the+separate runtime projection and are never copied into users' Codex packages.+Cargo/Bazel native providers, final helper linkage and moved-package execution+remain separate integration work; exporting an SDK does not enable voice.+## Native build recipe`build_native.py` runs the unmodified upstream build systems in a new outputdiff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.pyindex adb75b6e4a..3ae5669557 100644--- a/third_party/voice/build_native.py+++ b/third_party/voice/build_native.py@@ -248,6 +248,7 @@ class NativeBuild:"--wrap-mode=nofallback","-Dauto_features=disabled","-Ddefault_library=shared",+ "-Dpkgconfig.relocatable=true",*options,],)diff --git a/third_party/voice/sdk.py b/third_party/voice/sdk.pynew file mode 100644index 0000000000..bc7f05037b--- /dev/null+++ b/third_party/voice/sdk.py@@ -0,0 +1,140 @@+"""Export development inputs from the same inspected build as the private runtime.++SDKs are build inputs, never installed user payloads. Their receipt records byte+identity and provenance, not authenticity or approval. Native loader relocation+and final helper linking remain the runtime preparer and consumer's jobs.+"""++import json+from pathlib import Path+import re+import shutil++from runtime import digest++MODULES = (+ "glib-2.0",+ "gobject-2.0",+ "gmodule-no-export-2.0",+ "gstreamer-1.0",+ "gstreamer-base-1.0",+ "gstreamer-app-1.0",+ "gstreamer-audio-1.0",+ "gstreamer-tag-1.0",+)+++def export_sdk(prefix: Path, receipts: Path, target: str, output: Path):+ prefix, receipts = prefix.resolve(strict=True), receipts.resolve(strict=True)+ output = output.absolute()+ if (+ output.exists()+ or output.is_symlink()+ or any(output.resolve().is_relative_to(root) for root in (prefix, receipts))+ ):+ raise ValueError("SDK output must be fresh and outside the inputs")+ if not re.fullmatch(+ r"(?:aarch64|x86_64)-(?:apple-darwin|unknown-linux-gnu|pc-windows-msvc)",+ target,+ ):+ raise ValueError("unsupported SDK target")+ ci = json.loads((receipts / "ci.json").read_text())+ source_hash = digest(Path(__file__).with_name("sources.json"))+ if (+ ci.get("target") != target+ or ci.get("build_complete") is not True+ or ci.get("inspection_complete") is not True+ or ci.get("manifest_sha256") != source_hash+ or not re.fullmatch(r"[0-9a-f]{40}", ci.get("commit", ""))+ ):+ raise ValueError("SDK build receipt does not match the sources and target")+ inventory = json.loads((receipts / "inspection/binaries.json").read_text())+ if not 1 <= len(inventory) <= 128:+ raise ValueError("unexpected native inventory size")+ binaries = {}+ for record in inventory:+ name = record["path"].replace("\\", "/")+ if not re.fullmatch(r"(?:lib(?:/gstreamer-1\.0)?|bin)/[A-Za-z0-9_+.-]+", name):+ raise ValueError("invalid native inventory path")+ path = prefix / name+ if (+ name in binaries+ or path.is_symlink()+ or not path.resolve(strict=True).is_relative_to(prefix)+ or record["target"] != target+ or digest(path) != record["sha256"]+ ):+ raise ValueError("native SDK library does not match its receipt")+ binaries[name] = record["sha256"]+ selected = []+ names = set()+ size = 0+ for path in sorted(prefix.rglob("*")):+ name = path.relative_to(prefix).as_posix()+ if path.is_dir():+ if path.is_symlink():+ raise ValueError("SDK input directories must not be links")+ continue+ if not (+ name.startswith("include/")+ or name == "lib/glib-2.0/include/glibconfig.h"+ or name+ in {+ f"lib/pkgconfig/{module}.pc"+ for module in (*MODULES, "libffi", "libpcre2-8", "zlib")+ }+ or re.fullmatch(+ r"lib/[A-Za-z0-9_+.-]+\.(?:a|lib|dylib|so(?:\.[0-9]+)*)", name+ )+ ):+ continue+ source = path.resolve(strict=True)+ if not source.is_relative_to(prefix) or not source.is_file():+ raise ValueError("SDK inputs must be files inside the prefix")+ if name.casefold() in names:+ raise ValueError("SDK paths collide across platforms")+ names.add(name.casefold())+ size += source.stat().st_size+ if len(selected) >= 4096 or size > 512 * 1024 * 1024:+ raise ValueError("SDK inputs exceed the size limit")+ source_name = source.relative_to(prefix).as_posix()+ expected = digest(source)+ if re.search(r"\.(?:dylib|so(?:\.[0-9]+)*)$", name):+ if binaries.get(source_name) != expected:+ raise ValueError("SDK shared library is missing from the receipt")+ selected.append((name, source, expected))+ for module in MODULES:+ path = prefix / f"lib/pkgconfig/{module}.pc"+ if path.relative_to(prefix).as_posix().casefold() not in names or not re.search(+ r"^prefix=\$\{pcfiledir\}/\.\./\.\.$", path.read_text(), re.MULTILINE+ ):+ raise ValueError("rebuild the SDK with relocatable Meson pkg-config files")+ if "lib/glib-2.0/include/glibconfig.h" not in names:+ raise ValueError("SDK is missing the target's GLib configuration header")+ output.mkdir()+ try:+ files = []+ for name, source, expected in selected:+ destination = output / name+ destination.parent.mkdir(parents=True, exist_ok=True)+ shutil.copy2(source, destination)+ if digest(destination) != expected:+ raise ValueError("SDK input changed while copying")+ files.append({"path": name, "sha256": expected})+ (output / "sdk.json").write_text(+ json.dumps(+ {+ "schemaVersion": 1,+ "target": target,+ "sourceCommit": ci["commit"],+ "sourceManifestSha256": source_hash,+ "files": files,+ },+ indent=2,+ )+ + "\n",+ encoding="utf-8",+ )+ except BaseException:+ shutil.rmtree(output)+ raisediff --git a/third_party/voice/test_sdk.py b/third_party/voice/test_sdk.pynew file mode 100644index 0000000000..e31d543d04--- /dev/null+++ b/third_party/voice/test_sdk.py@@ -0,0 +1,153 @@+"""Exercise exported bytes, development aliases and moved pkg-config inputs."""++import json+import os+from pathlib import Path+import shlex+import shutil+import subprocess+import tempfile+import unittest+from unittest.mock import patch++from runtime import digest+from sdk import MODULES, export_sdk+++class SdkTests(unittest.TestCase):+ def setUp(self):+ temporary = tempfile.TemporaryDirectory(prefix="voice SDK ")+ self.addCleanup(temporary.cleanup)+ self.root = Path(temporary.name)+ self.prefix = self.root / "prefix"+ self.receipts = self.root / "receipts"+ self.output = self.root / "sdk"+ self.target = "aarch64-apple-darwin"+ for name in ("include/glib-2.0", "lib/glib-2.0/include", "lib/pkgconfig"):+ (self.prefix / name).mkdir(parents=True)+ (self.prefix / "include/glib-2.0/glib.h").write_text("/* SDK fixture */")+ (self.prefix / "lib/glib-2.0/include/glibconfig.h").write_text("/* target */")+ self.library = self.prefix / "lib/libfixture.0.dylib"+ self.library.write_bytes(b"receipt-verified fixture bytes")+ for module in (*MODULES, "zlib"):+ (self.prefix / f"lib/pkgconfig/{module}.pc").write_text(+ "prefix=${pcfiledir}/../..\n"+ f"Name: {module}\nDescription: SDK fixture\nVersion: 1.0\n"+ "Cflags: -I${prefix}/include\nLibs: -L${prefix}/lib -lfixture\n"+ )+ audio = self.prefix / "lib/pkgconfig/gstreamer-audio-1.0.pc"+ audio.write_text(+ audio.read_text() + "Requires.private: gstreamer-tag-1.0, zlib\n"+ )+ (self.receipts / "inspection").mkdir(parents=True)+ self.ci = {+ "target": self.target,+ "build_complete": True,+ "inspection_complete": True,+ "manifest_sha256": digest(Path(__file__).with_name("sources.json")),+ "commit": "a" * 40,+ }+ (self.receipts / "ci.json").write_text(json.dumps(self.ci))+ (self.receipts / "inspection/binaries.json").write_text(+ json.dumps(+ [+ {+ "path": "lib/libfixture.0.dylib",+ "target": self.target,+ "sha256": digest(self.library),+ }+ ]+ )+ )++ def test_exported_bytes_and_receipt_survive_move_without_original(self):+ export_sdk(self.prefix, self.receipts, self.target, self.output)+ moved = self.root / "moved SDK"+ self.output.rename(moved)+ shutil.rmtree(self.prefix)+ manifest = json.loads((moved / "sdk.json").read_text())+ actual = {+ path.relative_to(moved).as_posix(): digest(path)+ for path in moved.rglob("*")+ if path.is_file() and path.name != "sdk.json"+ }+ self.assertEqual(+ manifest,+ {+ "schemaVersion": 1,+ "target": self.target,+ "sourceCommit": "a" * 40,+ "sourceManifestSha256": self.ci["manifest_sha256"],+ "files": [+ {"path": name, "sha256": actual[name]} for name in sorted(actual)+ ],+ },+ )+ pkg_config = os.environ.get("VOICE_PKG_CONFIG") or shutil.which("pkg-config")+ if not pkg_config:+ self.skipTest("pkg-config required for the moved development-input probe")+ result = subprocess.check_output(+ [+ pkg_config,+ "--define-prefix",+ "--cflags",+ "--libs",+ "gstreamer-audio-1.0",+ ],+ env={+ **os.environ,+ "PKG_CONFIG_PATH": "",+ "PKG_CONFIG_LIBDIR": str(moved / "lib/pkgconfig"),+ },+ text=True,+ ).strip()+ self.assertEqual(+ shlex.split(result),+ [f"-I{moved.as_posix()}/include", f"-L{moved.as_posix()}/lib", "-lfixture"],+ )++ def test_native_mutation_and_wrong_target_are_rejected(self):+ with self.assertRaisesRegex(ValueError, "sources and target"):+ export_sdk(self.prefix, self.receipts, "x86_64-apple-darwin", self.output)+ self.library.write_bytes(b"changed")+ with self.assertRaisesRegex(ValueError, "does not match"):+ export_sdk(self.prefix, self.receipts, self.target, self.output)+ self.assertFalse(self.output.exists())++ def test_development_alias_is_materialized_and_escape_is_rejected(self):+ alias = self.prefix / "lib/libfixture.dylib"+ try:+ alias.symlink_to(self.library.name)+ except OSError:+ self.skipTest("host cannot create the SDK's development symlink")+ export_sdk(self.prefix, self.receipts, self.target, self.output)+ self.assertFalse((self.output / "lib/libfixture.dylib").is_symlink())+ self.assertEqual(+ (self.output / "lib/libfixture.dylib").read_bytes(),+ self.library.read_bytes(),+ )+ shutil.rmtree(self.output)+ alias.unlink()+ alias.symlink_to(self.receipts / "ci.json")+ with self.assertRaisesRegex(ValueError, "inside the prefix"):+ export_sdk(self.prefix, self.receipts, self.target, self.output)++ def test_nonrelocatable_metadata_and_failed_copy_leave_no_output(self):+ metadata = self.prefix / "lib/pkgconfig/gstreamer-1.0.pc"+ original = metadata.read_text()+ metadata.write_text(original.replace("${pcfiledir}/../..", "/old/build"))+ with self.assertRaisesRegex(ValueError, "rebuild the SDK"):+ export_sdk(self.prefix, self.receipts, self.target, self.output)+ metadata.write_text(original)+ with patch("sdk.shutil.copy2", side_effect=OSError("copy failed")):+ with self.assertRaises(OSError):+ export_sdk(self.prefix, self.receipts, self.target, self.output)+ self.assertFalse(self.output.exists())++ def test_output_cannot_overwrite_or_nest_inside_inputs(self):+ for output in (self.prefix, self.prefix / "sdk", self.receipts / "sdk"):+ with (+ self.subTest(output=output),+ self.assertRaisesRegex(ValueError, "fresh"),+ ):+ export_sdk(self.prefix, self.receipts, self.target, output)