Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add pinned native voice source preparation (#41884)

Benjamin Carlsson379d50be35d3
diff --git a/MODULE.bazel b/MODULE.bazel
index 7737e2e6d4..65d617755a 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -451,6 +451,22 @@ new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.
include("//bazel/modules:wine.MODULE.bazel")
+voice_sources = use_extension("//third_party/voice:extensions.bzl", "voice_sources")
+use_repo(
+ voice_sources,
+ "voice_glib",
+ "voice_gst_plugins_base",
+ "voice_gst_plugins_good",
+ "voice_gstreamer",
+ "voice_libffi",
+ "voice_meson",
+ "voice_ninja",
+ "voice_opus",
+ "voice_pcre2",
+ "voice_proxy_libintl",
+ "voice_zlib",
+)
+
new_local_repository(
name = "v8_targets",
build_file = "//third_party/v8:BUILD.bazel",
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel
new file mode 100644
index 0000000000..817a9d67f8
--- /dev/null
+++ b/third_party/voice/BUILD.bazel
@@ -0,0 +1,29 @@
+# All native build consumers must receive the same pinned source manifest.
+filegroup(
+ name = "source_inputs",
+ srcs = [
+ "prepare_sources.py",
+ "sources.json",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# Fetch and unpack only when this source target is requested by a build.
+filegroup(
+ name = "sources",
+ srcs = [
+ "@voice_glib//:sources",
+ "@voice_gst_plugins_base//:sources",
+ "@voice_gst_plugins_good//:sources",
+ "@voice_gstreamer//:sources",
+ "@voice_libffi//:sources",
+ "@voice_meson//:sources",
+ "@voice_ninja//:sources",
+ "@voice_opus//:sources",
+ "@voice_pcre2//:sources",
+ "@voice_proxy_libintl//:sources",
+ "@voice_zlib//:sources",
+ ],
+ tags = ["manual"],
+ visibility = ["//visibility:public"],
+)
diff --git a/third_party/voice/README.md b/third_party/voice/README.md
new file mode 100644
index 0000000000..90a853529c
--- /dev/null
+++ b/third_party/voice/README.md
@@ -0,0 +1,48 @@
+# Native voice source inputs
+
+This stage pins and prepares sources for a privately bundled, GStreamer-based
+audio runtime, including its native dependencies and build tools. It does not
+compile native libraries, link them into Codex or enable voice.
+
+`sources.json` records the versions, URLs and SHA-256 digests of 11 archives:
+
+| Purpose | Sources |
+| --- | --- |
+| GStreamer framework and plugin sources | `gstreamer`, `gst-plugins-base`, `gst-plugins-good` |
+| Supporting native libraries | `glib`, `libffi`, `pcre2`, `zlib`, `proxy-libintl` |
+| Audio codec | `opus` |
+| Build tools, not runtime libraries | `meson`, `ninja` |
+
+GLib also includes `gvdb` in its archive; it is recorded without a separate fetch.
+These inputs do not include the complete platform toolchain for native builds.
+
+Bazel uses standard `http_archive` rules to fetch, verify, unpack and cache the
+archives from that manifest. Run from the repository root:
+
+```sh
+bazel build //third_party/voice:sources
+```
+
+For offline preparation without Bazel, use Python 3.12 or newer and an existing
+archive directory whose filenames match the manifest:
+
+```sh
+python3 third_party/voice/prepare_sources.py --archives /path/to/archives --output /path/to/new-sources
+python3 -m unittest discover -s third_party/voice -p 'test_*.py'
+```
+
+The adapter verifies archive digests and bounds, then extracts with Python's
+`tarfile` data filter. It preserves links where supported and copies their archive
+targets when link creation is unavailable, including on Windows. It refuses an
+existing output directory and cleans up incomplete output. `prepared.json` records
+successful preparation, not the integrity of later edits to the extracted tree.
+
+Ordinary CLI builds do not run either path. The Bazel `:sources` target is
+`manual`, so wildcard builds do not fetch these archives. `:source_inputs`
+exports the manifest and adapter for standalone consumers; `:sources` exposes
+extracted archives with Bazel build metadata. Neither target compiles libraries.
+
+Checksums establish input identity, not security or license approval. Native
+compilation, final Cargo/Bazel linking, installed packages, minimum OS support
+and duplex audio validation remain separate stages. These inputs do not establish
+a shared Opus build with Rust consumers or a reduced dependency count.
diff --git a/third_party/voice/extensions.bzl b/third_party/voice/extensions.bzl
new file mode 100644
index 0000000000..38aa4401f0
--- /dev/null
+++ b/third_party/voice/extensions.bzl
@@ -0,0 +1,27 @@
+"""Use standard Bazel archives for the standalone builder's pinned sources."""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+_BUILD_FILE = """
+filegroup(
+ name = "sources",
+ srcs = glob(["**"]),
+ visibility = ["//visibility:public"],
+)
+"""
+
+def _voice_sources_impl(module_ctx):
+ manifest = json.decode(module_ctx.read(Label("//third_party/voice:sources.json")))
+ for source in manifest["sources"]:
+ http_archive(
+ name = "voice_" + source["name"].replace("-", "_"),
+ urls = [source["url"]],
+ sha256 = source["sha256"],
+ strip_prefix = source["root"],
+ # Ninja's pinned codeload URL has no archive filename extension.
+ type = "tar.gz" if source["name"] == "ninja" else "",
+ build_file_content = _BUILD_FILE,
+ )
+ return module_ctx.extension_metadata(reproducible = True)
+
+voice_sources = module_extension(implementation = _voice_sources_impl)
diff --git a/third_party/voice/prepare_sources.py b/third_party/voice/prepare_sources.py
new file mode 100644
index 0000000000..247d1d3e7f
--- /dev/null
+++ b/third_party/voice/prepare_sources.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Prepare pinned native voice sources offline; never execute archive contents."""
+
+import argparse
+from dataclasses import dataclass
+import hashlib
+import io
+import json
+from pathlib import Path, PurePosixPath
+import re
+import shutil
+import sys
+import tarfile
+
+MANIFEST = Path(__file__).with_name("sources.json")
+MAX_ARCHIVE_BYTES = 64 * 1024 * 1024
+MAX_SOURCE_BYTES = 512 * 1024 * 1024
+MAX_MEMBERS = 100_000
+
+
+@dataclass(frozen=True)
+class Source:
+ name: str
+ version: str
+ role: str
+ archive: str
+ root: str
+ url: str
+ sha256: str
+ provenance: str
+
+
+def load_sources(manifest: bytes) -> list[Source]:
+ document = json.loads(manifest)
+ if document["schema_version"] != 1:
+ raise ValueError("Unsupported native source manifest version")
+ sources = [Source(**entry) for entry in document["sources"]]
+ seen = set()
+ for source in sources:
+ for field in (source.name, source.archive, source.root):
+ if not re.fullmatch(r"[a-zA-Z0-9_-][a-zA-Z0-9_.-]*", field):
+ raise ValueError(f"Invalid native source identifier: {field!r}")
+ if not re.fullmatch(r"[0-9a-f]{64}", source.sha256):
+ raise ValueError(f"Invalid SHA-256 for {source.name}")
+ for kind, value in (
+ ("name", source.name),
+ ("archive", source.archive),
+ ("root", source.root),
+ ):
+ key = (kind, value.casefold())
+ if key in seen:
+ raise ValueError(f"Duplicate native source {kind}: {value}")
+ seen.add(key)
+ if not sources:
+ raise ValueError("Native source manifest is empty")
+ return sources
+
+
+def extract_source(source: Source, archives: Path, destination: Path) -> None:
+ # Use one immutable byte snapshot for both verification and extraction.
+ with (archives / source.archive).open("rb") as archive:
+ data = archive.read(MAX_ARCHIVE_BYTES + 1)
+ if len(data) > MAX_ARCHIVE_BYTES:
+ raise ValueError(f"Archive exceeds size limit: {source.name}")
+ if hashlib.sha256(data).hexdigest() != source.sha256:
+ raise ValueError(f"SHA-256 mismatch: {source.name}")
+ with tarfile.open(fileobj=io.BytesIO(data)) as archive:
+ members = []
+ size = 0
+ for member in archive:
+ path = PurePosixPath(member.name)
+ if path.parts[:1] != (source.root,) or ".." in path.parts:
+ raise ValueError(f"Invalid archive path: {member.name!r}")
+ size += member.size
+ members.append(member)
+ if size > MAX_SOURCE_BYTES or len(members) > MAX_MEMBERS:
+ raise ValueError(f"Expanded source exceeds limits: {source.name}")
+ if not members:
+ raise ValueError(f"Empty source archive: {source.name}")
+ # Bound Python's fallback copies when Windows cannot create archive links.
+ links = sum(member.issym() or member.islnk() for member in members)
+ if size + links * max(member.size for member in members) > MAX_SOURCE_BYTES:
+ raise ValueError(f"Expanded source exceeds limits: {source.name}")
+ archive.extractall(destination, members=members, filter="data")
+
+
+def prepare_sources(archives: Path, output: Path, manifest: bytes) -> None:
+ sources = load_sources(manifest)
+ # Claim a new directory exclusively; never reuse or overwrite an earlier build.
+ output.mkdir()
+ try:
+ for source in sources:
+ extract_source(source, archives, output)
+ receipt = {
+ "schema_version": 1,
+ "manifest_sha256": hashlib.sha256(manifest).hexdigest(),
+ "sources": {
+ source.name: {"root": source.root, "sha256": source.sha256}
+ for source in sources
+ },
+ }
+ (output / "sources.json").write_bytes(manifest)
+ # This is the completion marker; failures never leave a valid receipt.
+ (output / "prepared.json").write_text(
+ json.dumps(receipt, indent=2) + "\n", encoding="utf-8"
+ )
+ except BaseException:
+ shutil.rmtree(output)
+ raise
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "--archives",
+ required=True,
+ type=Path,
+ help="Directory containing the pinned archives",
+ )
+ parser.add_argument(
+ "--output", required=True, type=Path, help="New directory for verified sources"
+ )
+ args = parser.parse_args()
+ if sys.version_info < (3, 12):
+ parser.error("Native source preparation requires Python 3.12 or newer")
+ prepare_sources(args.archives, args.output, MANIFEST.read_bytes())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/third_party/voice/sources.json b/third_party/voice/sources.json
new file mode 100644
index 0000000000..3d10b9adac
--- /dev/null
+++ b/third_party/voice/sources.json
@@ -0,0 +1,124 @@
+{
+ "schema_version": 1,
+ "sources": [
+ {
+ "name": "meson",
+ "version": "1.12.0",
+ "role": "build-tool",
+ "archive": "meson-1.12.0.tar",
+ "root": "meson-1.12.0",
+ "url": "https://github.com/mesonbuild/meson/releases/download/1.12.0/meson-1.12.0.tar.gz",
+ "sha256": "88afe0c20e52030218924ac37d0c81c59b4b5f3ae3752c8c6d7470c7d365886c",
+ "provenance": "GitHub release asset digest"
+ },
+ {
+ "name": "ninja",
+ "version": "1.13.2",
+ "role": "build-tool",
+ "archive": "ninja-1.13.2.tar",
+ "root": "ninja-3441b633c2fe2c494e958780ba0f4227b1327634",
+ "url": "https://codeload.github.com/ninja-build/ninja/tar.gz/3441b633c2fe2c494e958780ba0f4227b1327634",
+ "sha256": "bccc6197cd8c3ac2a439e26d6bf41506fe49c430cf3d593269a15379f24266ee",
+ "provenance": "GitHub v1.13.2 tag -> commit3441b633c2fe2c494e958780ba0f4227b1327634; archive SHA recorded after fetch"
+ },
+ {
+ "name": "libffi",
+ "version": "3.8.0",
+ "role": "native-library",
+ "archive": "libffi-3.8.0.tar",
+ "root": "libffi-3.8.0",
+ "url": "https://github.com/libffi/libffi/releases/download/v3.8.0/libffi-3.8.0.tar.gz",
+ "sha256": "7da3e2d9a171eb0a038f592ecad3ff2bb2550f3496d87b3b29ad0cf4430c0db4",
+ "provenance": "GitHub release asset digest"
+ },
+ {
+ "name": "pcre2",
+ "version": "10.47",
+ "role": "native-library",
+ "archive": "pcre2-10.47.tar",
+ "root": "pcre2-10.47",
+ "url": "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.47/pcre2-10.47.tar.gz",
+ "sha256": "c08ae2388ef333e8403e670ad70c0a11f1eed021fd88308d7e02f596fcd9dc16",
+ "provenance": "GitHub release asset digest"
+ },
+ {
+ "name": "zlib",
+ "version": "1.3.2",
+ "role": "native-library",
+ "archive": "zlib-1.3.2.tar",
+ "root": "zlib-1.3.2",
+ "url": "https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz",
+ "sha256": "bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16",
+ "provenance": "GitHub release asset digest"
+ },
+ {
+ "name": "opus",
+ "version": "1.6.1",
+ "role": "native-library",
+ "archive": "opus-1.6.1.tar",
+ "root": "opus-1.6.1",
+ "url": "https://downloads.xiph.org/releases/opus/opus-1.6.1.tar.gz",
+ "sha256": "6ffcb593207be92584df15b32466ed64bbec99109f007c82205f0194572411a1",
+ "provenance": "https://downloads.xiph.org/releases/opus/SHA256SUMS.txt"
+ },
+ {
+ "name": "glib",
+ "version": "2.88.3",
+ "role": "native-library",
+ "archive": "glib-2.88.3.tar",
+ "root": "glib-2.88.3",
+ "url": "https://download.gnome.org/sources/glib/2.88/glib-2.88.3.tar.xz",
+ "sha256": "ab24d24e698dfa1e408b7bcdb508f4aafc906185a8b8ce72fdf79bbbdc9b383b",
+ "provenance": "https://download.gnome.org/sources/glib/2.88/glib-2.88.3.sha256sum"
+ },
+ {
+ "name": "proxy-libintl",
+ "version": "0.5",
+ "role": "native-library",
+ "archive": "proxy-libintl-0.5.tar",
+ "root": "proxy-libintl-0.5",
+ "url": "https://github.com/frida/proxy-libintl/archive/refs/tags/0.5.tar.gz",
+ "sha256": "f7a1cbd7579baaf575c66f9d99fb6295e9b0684a28b095967cfda17857595303",
+ "provenance": "GStreamer1.28.6 upstream wrap SHA; upstream tag commit33934de09af6a6627eb44e310a8079df009abdbb"
+ },
+ {
+ "name": "gstreamer",
+ "version": "1.28.6",
+ "role": "native-library",
+ "archive": "gstreamer-1.28.6.tar",
+ "root": "gstreamer-1.28.6",
+ "url": "https://gstreamer.freedesktop.org/src/gstreamer/gstreamer-1.28.6.tar.xz",
+ "sha256": "62b6b9f0ad3147a6dd6420ac64a91180b14e990695bddd353b96041611d052ca",
+ "provenance": "Official adjacent .sha256sum"
+ },
+ {
+ "name": "gst-plugins-base",
+ "version": "1.28.6",
+ "role": "native-library",
+ "archive": "gst-plugins-base-1.28.6.tar",
+ "root": "gst-plugins-base-1.28.6",
+ "url": "https://gstreamer.freedesktop.org/src/gst-plugins-base/gst-plugins-base-1.28.6.tar.xz",
+ "sha256": "0ba699c7c6c66f4ba640be78cb38a24715add9683f3e3a199f5369dc5a4f04ac",
+ "provenance": "Official adjacent .sha256sum"
+ },
+ {
+ "name": "gst-plugins-good",
+ "version": "1.28.6",
+ "role": "native-library",
+ "archive": "gst-plugins-good-1.28.6.tar",
+ "root": "gst-plugins-good-1.28.6",
+ "url": "https://gstreamer.freedesktop.org/src/gst-plugins-good/gst-plugins-good-1.28.6.tar.xz",
+ "sha256": "b0c620a4b18b6ee931b4c43bbf1760d308666dc37f730a7e7f1ad327e59ce2df",
+ "provenance": "Official adjacent .sha256sum"
+ }
+ ],
+ "bundled_sources": [
+ {
+ "name": "gvdb",
+ "included_by": "glib",
+ "revision": "2b42fc75f09dbe1cd1057580b5782b08f2dcb400",
+ "path": "subprojects/gvdb",
+ "provenance": "Included in the pinned GLib release archive; no separate fetch"
+ }
+ ]
+}
diff --git a/third_party/voice/test_prepare_sources.py b/third_party/voice/test_prepare_sources.py
new file mode 100644
index 0000000000..ea011414b2
--- /dev/null
+++ b/third_party/voice/test_prepare_sources.py
@@ -0,0 +1,124 @@
+"""Exercise source identity and extraction boundaries with synthetic archives."""
+
+import hashlib
+import io
+import json
+from pathlib import Path
+import tarfile
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import prepare_sources as preparation
+
+
+class PreparationTests(unittest.TestCase):
+ def setUp(self):
+ directory = tempfile.TemporaryDirectory()
+ self.addCleanup(directory.cleanup)
+ self.root = Path(directory.name)
+ self.output = self.root / "prepared sources"
+
+ def source(self, entries):
+ archive_path = self.root / "fixture.tar"
+ with tarfile.open(archive_path, "w") as archive:
+ for name, kind, payload in entries:
+ member = tarfile.TarInfo(name)
+ member.type = kind
+ if kind == tarfile.SYMTYPE:
+ member.linkname = payload
+ archive.addfile(member)
+ else:
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ return {
+ "name": "fixture",
+ "version": "1",
+ "role": "native-library",
+ "archive": archive_path.name,
+ "root": "fixture-1",
+ "url": "https://example.invalid/fixture.tar",
+ "sha256": hashlib.sha256(archive_path.read_bytes()).hexdigest(),
+ "provenance": "Synthetic test input",
+ }
+
+ def prepare(self, source):
+ manifest = json.dumps({"schema_version": 1, "sources": [source]}).encode()
+ preparation.prepare_sources(self.root, self.output, manifest)
+ return manifest
+
+ def test_preserves_files_licenses_when_symlinks_are_unavailable(self):
+ source = self.source(
+ [
+ ("fixture-1/LICENSES/license.txt", tarfile.REGTYPE, b"License text\n"),
+ ("fixture-1/COPYING", tarfile.SYMTYPE, "LICENSES/license.txt"),
+ ]
+ )
+ with patch("tarfile.os.symlink", side_effect=OSError("No symlink privilege")):
+ manifest = self.prepare(source)
+ files = {
+ p.relative_to(self.output / source["root"]).as_posix(): p.read_bytes()
+ for p in (self.output / source["root"]).rglob("*")
+ if p.is_file()
+ }
+ self.assertEqual(
+ files,
+ {
+ "LICENSES/license.txt": b"License text\n",
+ "COPYING": b"License text\n",
+ },
+ )
+ self.assertFalse(any(p.is_symlink() for p in self.output.rglob("*")))
+ self.assertEqual((self.output / "sources.json").read_bytes(), manifest)
+ self.assertEqual(
+ json.loads((self.output / "prepared.json").read_text()),
+ {
+ "schema_version": 1,
+ "manifest_sha256": hashlib.sha256(manifest).hexdigest(),
+ "sources": {
+ "fixture": {"root": "fixture-1", "sha256": source["sha256"]}
+ },
+ },
+ )
+ def test_rejects_changed_archive_before_extracting(self):
+ source = self.source([("fixture-1/file", tarfile.REGTYPE, b"original")])
+ (self.root / source["archive"]).write_bytes(b"changed")
+ with self.assertRaisesRegex(ValueError, "SHA-256 mismatch"):
+ self.prepare(source)
+ self.assertFalse(self.output.exists())
+
+ def test_filter_failure_cleans_incomplete_output(self):
+ source = self.source(
+ [
+ ("fixture-1/file", tarfile.REGTYPE, b"safe"),
+ ("fixture-1/link", tarfile.SYMTYPE, "../../outside"),
+ ]
+ )
+ with self.assertRaises(tarfile.FilterError):
+ self.prepare(source)
+ self.assertFalse(self.output.exists())
+
+ def test_materialized_links_cannot_bypass_expansion_limit(self):
+ source = self.source(
+ [
+ ("fixture-1/file", tarfile.REGTYPE, b"123456"),
+ ("fixture-1/link", tarfile.SYMTYPE, "file"),
+ ]
+ )
+ with patch.object(preparation, "MAX_SOURCE_BYTES", 10):
+ with self.assertRaisesRegex(
+ ValueError, "Expanded source exceeds limits"
+ ):
+ self.prepare(source)
+ self.assertFalse(self.output.exists())
+
+ def test_preserves_existing_output(self):
+ source = self.source([("fixture-1/file", tarfile.REGTYPE, b"data")])
+ self.output.mkdir()
+ (self.output / "keep").write_text("unchanged")
+ with self.assertRaises(FileExistsError):
+ self.prepare(source)
+ self.assertEqual(
+ {p.name: p.read_text() for p in self.output.iterdir()},
+ {"keep": "unchanged"},
+ )