Lightweight coding agent that runs in your terminal
Rust
Add pinned native voice source preparation (#41884)
Benjamin Carlsson379d50be35d3
diff --git a/MODULE.bazel b/MODULE.bazelindex 7737e2e6d4..65d617755a 100644--- a/MODULE.bazel+++ b/MODULE.bazel@@ -451,6 +451,22 @@ new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.include("//bazel/modules:wine.MODULE.bazel")+voice_sources = use_extension("//third_party/voice:extensions.bzl", "voice_sources")+use_repo(+ voice_sources,+ "voice_glib",+ "voice_gst_plugins_base",+ "voice_gst_plugins_good",+ "voice_gstreamer",+ "voice_libffi",+ "voice_meson",+ "voice_ninja",+ "voice_opus",+ "voice_pcre2",+ "voice_proxy_libintl",+ "voice_zlib",+)+new_local_repository(name = "v8_targets",build_file = "//third_party/v8:BUILD.bazel",diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazelnew file mode 100644index 0000000000..817a9d67f8--- /dev/null+++ b/third_party/voice/BUILD.bazel@@ -0,0 +1,29 @@+# All native build consumers must receive the same pinned source manifest.+filegroup(+ name = "source_inputs",+ srcs = [+ "prepare_sources.py",+ "sources.json",+ ],+ visibility = ["//visibility:public"],+)++# Fetch and unpack only when this source target is requested by a build.+filegroup(+ name = "sources",+ srcs = [+ "@voice_glib//:sources",+ "@voice_gst_plugins_base//:sources",+ "@voice_gst_plugins_good//:sources",+ "@voice_gstreamer//:sources",+ "@voice_libffi//:sources",+ "@voice_meson//:sources",+ "@voice_ninja//:sources",+ "@voice_opus//:sources",+ "@voice_pcre2//:sources",+ "@voice_proxy_libintl//:sources",+ "@voice_zlib//:sources",+ ],+ tags = ["manual"],+ visibility = ["//visibility:public"],+)diff --git a/third_party/voice/README.md b/third_party/voice/README.mdnew file mode 100644index 0000000000..90a853529c--- /dev/null+++ b/third_party/voice/README.md@@ -0,0 +1,48 @@+# Native voice source inputs++This stage pins and prepares sources for a privately bundled, GStreamer-based+audio runtime, including its native dependencies and build tools. It does not+compile native libraries, link them into Codex or enable voice.++`sources.json` records the versions, URLs and SHA-256 digests of 11 archives:++| Purpose | Sources |+| --- | --- |+| GStreamer framework and plugin sources | `gstreamer`, `gst-plugins-base`, `gst-plugins-good` |+| Supporting native libraries | `glib`, `libffi`, `pcre2`, `zlib`, `proxy-libintl` |+| Audio codec | `opus` |+| Build tools, not runtime libraries | `meson`, `ninja` |++GLib also includes `gvdb` in its archive; it is recorded without a separate fetch.+These inputs do not include the complete platform toolchain for native builds.++Bazel uses standard `http_archive` rules to fetch, verify, unpack and cache the+archives from that manifest. Run from the repository root:++```sh+bazel build //third_party/voice:sources+```++For offline preparation without Bazel, use Python 3.12 or newer and an existing+archive directory whose filenames match the manifest:++```sh+python3 third_party/voice/prepare_sources.py --archives /path/to/archives --output /path/to/new-sources+python3 -m unittest discover -s third_party/voice -p 'test_*.py'+```++The adapter verifies archive digests and bounds, then extracts with Python's+`tarfile` data filter. It preserves links where supported and copies their archive+targets when link creation is unavailable, including on Windows. It refuses an+existing output directory and cleans up incomplete output. `prepared.json` records+successful preparation, not the integrity of later edits to the extracted tree.++Ordinary CLI builds do not run either path. The Bazel `:sources` target is+`manual`, so wildcard builds do not fetch these archives. `:source_inputs`+exports the manifest and adapter for standalone consumers; `:sources` exposes+extracted archives with Bazel build metadata. Neither target compiles libraries.++Checksums establish input identity, not security or license approval. Native+compilation, final Cargo/Bazel linking, installed packages, minimum OS support+and duplex audio validation remain separate stages. These inputs do not establish+a shared Opus build with Rust consumers or a reduced dependency count.diff --git a/third_party/voice/extensions.bzl b/third_party/voice/extensions.bzlnew file mode 100644index 0000000000..38aa4401f0--- /dev/null+++ b/third_party/voice/extensions.bzl@@ -0,0 +1,27 @@+"""Use standard Bazel archives for the standalone builder's pinned sources."""++load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")++_BUILD_FILE = """+filegroup(+ name = "sources",+ srcs = glob(["**"]),+ visibility = ["//visibility:public"],+)+"""++def _voice_sources_impl(module_ctx):+ manifest = json.decode(module_ctx.read(Label("//third_party/voice:sources.json")))+ for source in manifest["sources"]:+ http_archive(+ name = "voice_" + source["name"].replace("-", "_"),+ urls = [source["url"]],+ sha256 = source["sha256"],+ strip_prefix = source["root"],+ # Ninja's pinned codeload URL has no archive filename extension.+ type = "tar.gz" if source["name"] == "ninja" else "",+ build_file_content = _BUILD_FILE,+ )+ return module_ctx.extension_metadata(reproducible = True)++voice_sources = module_extension(implementation = _voice_sources_impl)diff --git a/third_party/voice/prepare_sources.py b/third_party/voice/prepare_sources.pynew file mode 100644index 0000000000..247d1d3e7f--- /dev/null+++ b/third_party/voice/prepare_sources.py@@ -0,0 +1,130 @@+#!/usr/bin/env python3+"""Prepare pinned native voice sources offline; never execute archive contents."""++import argparse+from dataclasses import dataclass+import hashlib+import io+import json+from pathlib import Path, PurePosixPath+import re+import shutil+import sys+import tarfile++MANIFEST = Path(__file__).with_name("sources.json")+MAX_ARCHIVE_BYTES = 64 * 1024 * 1024+MAX_SOURCE_BYTES = 512 * 1024 * 1024+MAX_MEMBERS = 100_000+++@dataclass(frozen=True)+class Source:+ name: str+ version: str+ role: str+ archive: str+ root: str+ url: str+ sha256: str+ provenance: str+++def load_sources(manifest: bytes) -> list[Source]:+ document = json.loads(manifest)+ if document["schema_version"] != 1:+ raise ValueError("Unsupported native source manifest version")+ sources = [Source(**entry) for entry in document["sources"]]+ seen = set()+ for source in sources:+ for field in (source.name, source.archive, source.root):+ if not re.fullmatch(r"[a-zA-Z0-9_-][a-zA-Z0-9_.-]*", field):+ raise ValueError(f"Invalid native source identifier: {field!r}")+ if not re.fullmatch(r"[0-9a-f]{64}", source.sha256):+ raise ValueError(f"Invalid SHA-256 for {source.name}")+ for kind, value in (+ ("name", source.name),+ ("archive", source.archive),+ ("root", source.root),+ ):+ key = (kind, value.casefold())+ if key in seen:+ raise ValueError(f"Duplicate native source {kind}: {value}")+ seen.add(key)+ if not sources:+ raise ValueError("Native source manifest is empty")+ return sources+++def extract_source(source: Source, archives: Path, destination: Path) -> None:+ # Use one immutable byte snapshot for both verification and extraction.+ with (archives / source.archive).open("rb") as archive:+ data = archive.read(MAX_ARCHIVE_BYTES + 1)+ if len(data) > MAX_ARCHIVE_BYTES:+ raise ValueError(f"Archive exceeds size limit: {source.name}")+ if hashlib.sha256(data).hexdigest() != source.sha256:+ raise ValueError(f"SHA-256 mismatch: {source.name}")+ with tarfile.open(fileobj=io.BytesIO(data)) as archive:+ members = []+ size = 0+ for member in archive:+ path = PurePosixPath(member.name)+ if path.parts[:1] != (source.root,) or ".." in path.parts:+ raise ValueError(f"Invalid archive path: {member.name!r}")+ size += member.size+ members.append(member)+ if size > MAX_SOURCE_BYTES or len(members) > MAX_MEMBERS:+ raise ValueError(f"Expanded source exceeds limits: {source.name}")+ if not members:+ raise ValueError(f"Empty source archive: {source.name}")+ # Bound Python's fallback copies when Windows cannot create archive links.+ links = sum(member.issym() or member.islnk() for member in members)+ if size + links * max(member.size for member in members) > MAX_SOURCE_BYTES:+ raise ValueError(f"Expanded source exceeds limits: {source.name}")+ archive.extractall(destination, members=members, filter="data")+++def prepare_sources(archives: Path, output: Path, manifest: bytes) -> None:+ sources = load_sources(manifest)+ # Claim a new directory exclusively; never reuse or overwrite an earlier build.+ output.mkdir()+ try:+ for source in sources:+ extract_source(source, archives, output)+ receipt = {+ "schema_version": 1,+ "manifest_sha256": hashlib.sha256(manifest).hexdigest(),+ "sources": {+ source.name: {"root": source.root, "sha256": source.sha256}+ for source in sources+ },+ }+ (output / "sources.json").write_bytes(manifest)+ # This is the completion marker; failures never leave a valid receipt.+ (output / "prepared.json").write_text(+ json.dumps(receipt, indent=2) + "\n", encoding="utf-8"+ )+ except BaseException:+ shutil.rmtree(output)+ raise+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument(+ "--archives",+ required=True,+ type=Path,+ help="Directory containing the pinned archives",+ )+ parser.add_argument(+ "--output", required=True, type=Path, help="New directory for verified sources"+ )+ args = parser.parse_args()+ if sys.version_info < (3, 12):+ parser.error("Native source preparation requires Python 3.12 or newer")+ prepare_sources(args.archives, args.output, MANIFEST.read_bytes())+++if __name__ == "__main__":+ main()diff --git a/third_party/voice/sources.json b/third_party/voice/sources.jsonnew file mode 100644index 0000000000..3d10b9adac--- /dev/null+++ b/third_party/voice/sources.json@@ -0,0 +1,124 @@+{+ "schema_version": 1,+ "sources": [+ {+ "name": "meson",+ "version": "1.12.0",+ "role": "build-tool",+ "archive": "meson-1.12.0.tar",+ "root": "meson-1.12.0",+ "url": "https://github.com/mesonbuild/meson/releases/download/1.12.0/meson-1.12.0.tar.gz",+ "sha256": "88afe0c20e52030218924ac37d0c81c59b4b5f3ae3752c8c6d7470c7d365886c",+ "provenance": "GitHub release asset digest"+ },+ {+ "name": "ninja",+ "version": "1.13.2",+ "role": "build-tool",+ "archive": "ninja-1.13.2.tar",+ "root": "ninja-3441b633c2fe2c494e958780ba0f4227b1327634",+ "url": "https://codeload.github.com/ninja-build/ninja/tar.gz/3441b633c2fe2c494e958780ba0f4227b1327634",+ "sha256": "bccc6197cd8c3ac2a439e26d6bf41506fe49c430cf3d593269a15379f24266ee",+ "provenance": "GitHub v1.13.2 tag -> commit3441b633c2fe2c494e958780ba0f4227b1327634; archive SHA recorded after fetch"+ },+ {+ "name": "libffi",+ "version": "3.8.0",+ "role": "native-library",+ "archive": "libffi-3.8.0.tar",+ "root": "libffi-3.8.0",+ "url": "https://github.com/libffi/libffi/releases/download/v3.8.0/libffi-3.8.0.tar.gz",+ "sha256": "7da3e2d9a171eb0a038f592ecad3ff2bb2550f3496d87b3b29ad0cf4430c0db4",+ "provenance": "GitHub release asset digest"+ },+ {+ "name": "pcre2",+ "version": "10.47",+ "role": "native-library",+ "archive": "pcre2-10.47.tar",+ "root": "pcre2-10.47",+ "url": "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.47/pcre2-10.47.tar.gz",+ "sha256": "c08ae2388ef333e8403e670ad70c0a11f1eed021fd88308d7e02f596fcd9dc16",+ "provenance": "GitHub release asset digest"+ },+ {+ "name": "zlib",+ "version": "1.3.2",+ "role": "native-library",+ "archive": "zlib-1.3.2.tar",+ "root": "zlib-1.3.2",+ "url": "https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz",+ "sha256": "bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16",+ "provenance": "GitHub release asset digest"+ },+ {+ "name": "opus",+ "version": "1.6.1",+ "role": "native-library",+ "archive": "opus-1.6.1.tar",+ "root": "opus-1.6.1",+ "url": "https://downloads.xiph.org/releases/opus/opus-1.6.1.tar.gz",+ "sha256": "6ffcb593207be92584df15b32466ed64bbec99109f007c82205f0194572411a1",+ "provenance": "https://downloads.xiph.org/releases/opus/SHA256SUMS.txt"+ },+ {+ "name": "glib",+ "version": "2.88.3",+ "role": "native-library",+ "archive": "glib-2.88.3.tar",+ "root": "glib-2.88.3",+ "url": "https://download.gnome.org/sources/glib/2.88/glib-2.88.3.tar.xz",+ "sha256": "ab24d24e698dfa1e408b7bcdb508f4aafc906185a8b8ce72fdf79bbbdc9b383b",+ "provenance": "https://download.gnome.org/sources/glib/2.88/glib-2.88.3.sha256sum"+ },+ {+ "name": "proxy-libintl",+ "version": "0.5",+ "role": "native-library",+ "archive": "proxy-libintl-0.5.tar",+ "root": "proxy-libintl-0.5",+ "url": "https://github.com/frida/proxy-libintl/archive/refs/tags/0.5.tar.gz",+ "sha256": "f7a1cbd7579baaf575c66f9d99fb6295e9b0684a28b095967cfda17857595303",+ "provenance": "GStreamer1.28.6 upstream wrap SHA; upstream tag commit33934de09af6a6627eb44e310a8079df009abdbb"+ },+ {+ "name": "gstreamer",+ "version": "1.28.6",+ "role": "native-library",+ "archive": "gstreamer-1.28.6.tar",+ "root": "gstreamer-1.28.6",+ "url": "https://gstreamer.freedesktop.org/src/gstreamer/gstreamer-1.28.6.tar.xz",+ "sha256": "62b6b9f0ad3147a6dd6420ac64a91180b14e990695bddd353b96041611d052ca",+ "provenance": "Official adjacent .sha256sum"+ },+ {+ "name": "gst-plugins-base",+ "version": "1.28.6",+ "role": "native-library",+ "archive": "gst-plugins-base-1.28.6.tar",+ "root": "gst-plugins-base-1.28.6",+ "url": "https://gstreamer.freedesktop.org/src/gst-plugins-base/gst-plugins-base-1.28.6.tar.xz",+ "sha256": "0ba699c7c6c66f4ba640be78cb38a24715add9683f3e3a199f5369dc5a4f04ac",+ "provenance": "Official adjacent .sha256sum"+ },+ {+ "name": "gst-plugins-good",+ "version": "1.28.6",+ "role": "native-library",+ "archive": "gst-plugins-good-1.28.6.tar",+ "root": "gst-plugins-good-1.28.6",+ "url": "https://gstreamer.freedesktop.org/src/gst-plugins-good/gst-plugins-good-1.28.6.tar.xz",+ "sha256": "b0c620a4b18b6ee931b4c43bbf1760d308666dc37f730a7e7f1ad327e59ce2df",+ "provenance": "Official adjacent .sha256sum"+ }+ ],+ "bundled_sources": [+ {+ "name": "gvdb",+ "included_by": "glib",+ "revision": "2b42fc75f09dbe1cd1057580b5782b08f2dcb400",+ "path": "subprojects/gvdb",+ "provenance": "Included in the pinned GLib release archive; no separate fetch"+ }+ ]+}diff --git a/third_party/voice/test_prepare_sources.py b/third_party/voice/test_prepare_sources.pynew file mode 100644index 0000000000..ea011414b2--- /dev/null+++ b/third_party/voice/test_prepare_sources.py@@ -0,0 +1,124 @@+"""Exercise source identity and extraction boundaries with synthetic archives."""++import hashlib+import io+import json+from pathlib import Path+import tarfile+import tempfile+import unittest+from unittest.mock import patch++import prepare_sources as preparation+++class PreparationTests(unittest.TestCase):+ def setUp(self):+ directory = tempfile.TemporaryDirectory()+ self.addCleanup(directory.cleanup)+ self.root = Path(directory.name)+ self.output = self.root / "prepared sources"++ def source(self, entries):+ archive_path = self.root / "fixture.tar"+ with tarfile.open(archive_path, "w") as archive:+ for name, kind, payload in entries:+ member = tarfile.TarInfo(name)+ member.type = kind+ if kind == tarfile.SYMTYPE:+ member.linkname = payload+ archive.addfile(member)+ else:+ member.size = len(payload)+ archive.addfile(member, io.BytesIO(payload))+ return {+ "name": "fixture",+ "version": "1",+ "role": "native-library",+ "archive": archive_path.name,+ "root": "fixture-1",+ "url": "https://example.invalid/fixture.tar",+ "sha256": hashlib.sha256(archive_path.read_bytes()).hexdigest(),+ "provenance": "Synthetic test input",+ }++ def prepare(self, source):+ manifest = json.dumps({"schema_version": 1, "sources": [source]}).encode()+ preparation.prepare_sources(self.root, self.output, manifest)+ return manifest++ def test_preserves_files_licenses_when_symlinks_are_unavailable(self):+ source = self.source(+ [+ ("fixture-1/LICENSES/license.txt", tarfile.REGTYPE, b"License text\n"),+ ("fixture-1/COPYING", tarfile.SYMTYPE, "LICENSES/license.txt"),+ ]+ )+ with patch("tarfile.os.symlink", side_effect=OSError("No symlink privilege")):+ manifest = self.prepare(source)+ files = {+ p.relative_to(self.output / source["root"]).as_posix(): p.read_bytes()+ for p in (self.output / source["root"]).rglob("*")+ if p.is_file()+ }+ self.assertEqual(+ files,+ {+ "LICENSES/license.txt": b"License text\n",+ "COPYING": b"License text\n",+ },+ )+ self.assertFalse(any(p.is_symlink() for p in self.output.rglob("*")))+ self.assertEqual((self.output / "sources.json").read_bytes(), manifest)+ self.assertEqual(+ json.loads((self.output / "prepared.json").read_text()),+ {+ "schema_version": 1,+ "manifest_sha256": hashlib.sha256(manifest).hexdigest(),+ "sources": {+ "fixture": {"root": "fixture-1", "sha256": source["sha256"]}+ },+ },+ )+ def test_rejects_changed_archive_before_extracting(self):+ source = self.source([("fixture-1/file", tarfile.REGTYPE, b"original")])+ (self.root / source["archive"]).write_bytes(b"changed")+ with self.assertRaisesRegex(ValueError, "SHA-256 mismatch"):+ self.prepare(source)+ self.assertFalse(self.output.exists())++ def test_filter_failure_cleans_incomplete_output(self):+ source = self.source(+ [+ ("fixture-1/file", tarfile.REGTYPE, b"safe"),+ ("fixture-1/link", tarfile.SYMTYPE, "../../outside"),+ ]+ )+ with self.assertRaises(tarfile.FilterError):+ self.prepare(source)+ self.assertFalse(self.output.exists())++ def test_materialized_links_cannot_bypass_expansion_limit(self):+ source = self.source(+ [+ ("fixture-1/file", tarfile.REGTYPE, b"123456"),+ ("fixture-1/link", tarfile.SYMTYPE, "file"),+ ]+ )+ with patch.object(preparation, "MAX_SOURCE_BYTES", 10):+ with self.assertRaisesRegex(+ ValueError, "Expanded source exceeds limits"+ ):+ self.prepare(source)+ self.assertFalse(self.output.exists())++ def test_preserves_existing_output(self):+ source = self.source([("fixture-1/file", tarfile.REGTYPE, b"data")])+ self.output.mkdir()+ (self.output / "keep").write_text("unchanged")+ with self.assertRaises(FileExistsError):+ self.prepare(source)+ self.assertEqual(+ {p.name: p.read_text() for p in self.output.iterdir()},+ {"keep": "unchanged"},+ )