Lightweight coding agent that runs in your terminal
Rust
Add GNU Linux voice runtime preparation (#42208)
Benjamin Carlsson8d01cd42faeb
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazelindex 33fac3e46a..4e6f3cb62c 100644--- a/third_party/voice/BUILD.bazel+++ b/third_party/voice/BUILD.bazel@@ -33,6 +33,7 @@ filegroup(srcs = ["assemble_package.py","build_native.py",+ "linux_runtime.py","macos_runtime.py","runtime.py",":source_inputs",diff --git a/third_party/voice/README.md b/third_party/voice/README.mdindex b60234138a..efc73f2ddb 100644--- a/third_party/voice/README.md+++ b/third_party/voice/README.md@@ -74,6 +74,12 @@ must still match the real native target. Native pkgconf relocates libffi's POSIXprefix metadata; CI rejects residual Cygwin paths. These are build prerequisites,not shipped runtime components or evidence of working voice.+CMake libraries use relative install runpaths (`$ORIGIN` on Linux and+`@loader_path` on Mac), with `@rpath` install names on Mac. Linux Meson links+use `$ORIGIN:$ORIGIN/..`, matching libraries in `lib/` and plugins in+`lib/gstreamer-1.0/`. Mac Meson and libffi still need packaging-time fixups;+these options do not make every Mac library relocatable at installation.+Outputs are under `prefix/`, build tools under `tools/`, and logs beside them.`build-state.json` records completed commands and failures; `built.json` existsonly when every build/install command succeeds. Failed builds retain their logs@@ -110,3 +116,18 @@ audio behavior. Dynamic-only dependencies, native helper linkage, LGPL notices,production signing/notarization, Windows/Linux loading and security approval remainseparate requirements. No microphone, device, plugin scanner or backend is startedby projection. Run its native relocation tests on macOS with Python 3.12 or newer.++## Private GNU Linux runtime preparation++`linux_runtime.py` takes the same prefix, receipts, target and output arguments.+It reads bounded ELF64 headers, segments and dynamic tables directly and accepts+x64/ARM64 GNU Linux libraries. The shared Python coordinator selects the seven+plugins and their declared dependencies without changing their bytes. The native+build must have emitted package-relative runpaths; older absolute paths are+rejected with a rebuild instruction. Output preserves `lib/gstreamer-1.0/` so+those relative paths remain valid. Loader audit/filter dependencies and+path-bearing imports are rejected. Native tests require Python 3.12, a C compiler+and `patchelf`; the latter constructs malformed inputs and is not needed during+preparation or shipped in the runtime. The output is development-only, uses the+host glibc, and does not establish musl or minimum-glibc support, dynamic-only+dependency closure, helper loading policy or working voice.diff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.pyindex d75cc1fc81..adb75b6e4a 100644--- a/third_party/voice/build_native.py+++ b/third_party/voice/build_native.py@@ -130,7 +130,14 @@ class NativeBuild:if args.target.endswith("apple-darwin"):self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_targetself.cmake_platform = [- f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}"+ f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}",+ "-DCMAKE_INSTALL_NAME_DIR=@rpath",+ "-DCMAKE_INSTALL_RPATH=@loader_path",+ ]+ elif not self.windows:+ self.cmake_platform = [+ "-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON",+ "-DCMAKE_INSTALL_RPATH=$ORIGIN",]self.record = {"target": args.target,@@ -223,6 +230,8 @@ class NativeBuild:if self.windowselse [f"-L{self.prefix / 'lib'}", f"-Wl,-rpath,{self.prefix / 'lib'}"])+ if self.args.target.endswith("unknown-linux-gnu"):+ link[-1] = "-Wl,-rpath,$ORIGIN:$ORIGIN/.."self.environment.update({"CFLAGS": include, "CXXFLAGS": include, "LDFLAGS": quote(link)})diff --git a/third_party/voice/linux_runtime.py b/third_party/voice/linux_runtime.pynew file mode 100644index 0000000000..124e5d40e1--- /dev/null+++ b/third_party/voice/linux_runtime.py@@ -0,0 +1,152 @@+"""Prepare verified GNU Linux audio libraries with package-relative loader paths."""++import argparse+import os+from pathlib import Path+import re+import struct+import sys++# Import only this script's siblings, including under PYTHONSAFEPATH.+sys.path.insert(0, str(Path(__file__).resolve().parent))+from runtime import Binary, PLUGINS, RuntimeFormat, prepare++SYSTEM_IMPORTS = frozenset(+ {+ "libc.so.6",+ "libm.so.6",+ "libdl.so.2",+ "libpthread.so.0",+ "librt.so.1",+ "libresolv.so.2",+ "ld-linux-x86-64.so.2",+ "ld-linux-aarch64.so.1",+ }+)+++def inspect(path, target):+ machine = {"x86_64-unknown-linux-gnu": 62, "aarch64-unknown-linux-gnu": 183}[target]+ with path.open("rb") as source:+ data = source.read(64 * 1024 * 1024 + 1)+ if not 64 <= len(data) <= 64 * 1024 * 1024:+ raise ValueError("invalid ELF file size")+ header = struct.unpack_from("<16sHHIQQQIHHHHHH", data)+ if header[0][:7] != b"\x7fELF\x02\x01\x01" or header[0][7] not in (0, 3):+ raise ValueError("expected a little-endian ELF64 library")+ if header[1:4] != (3, machine, 1) or header[8:10] != (64, 56):+ raise ValueError(f"expected a {target} shared library")+ phoff, count = header[5], header[10]+ if not 1 <= count <= 128 or phoff < 64 or phoff + count * 56 > len(data):+ raise ValueError("invalid ELF program headers")+ loads, dynamic = [], []+ for index in range(count):+ kind, _, offset, address, _, size, memory_size, _ = struct.unpack_from(+ "<IIQQQQQQ", data, phoff + index * 56+ )+ if size > memory_size or offset + size > len(data):+ raise ValueError("invalid ELF segment bounds")+ if kind == 1:+ loads.append((address, offset, size))+ elif kind == 2:+ dynamic.append((offset, size, address))+ elif kind == 3:+ raise ValueError("an ELF runtime library must not name an interpreter")+ if len(dynamic) != 1 or not 16 <= dynamic[0][1] <= 65536 or dynamic[0][1] % 16:+ raise ValueError("invalid ELF dynamic table")+ tags = {}+ offset, size, address = dynamic[0]+ mappings = [+ file_offset + address - start+ for start, file_offset, length in loads+ if start <= address and address + size <= start + length+ ]+ if mappings != [offset]:+ raise ValueError("ELF dynamic table must have one matching file-backed mapping")+ for cursor in range(offset, offset + size, 16):+ tag, value = struct.unpack_from("<qQ", data, cursor)+ if tag == 0:+ break+ # Auxiliary/filter libraries and audit modules are additional loader inputs.+ if tag in (0x7FFFFFFD, 0x7FFFFFFF, 0x6FFFFEFB, 0x6FFFFEFC):+ raise ValueError("unsupported ELF loader dependency")+ tags.setdefault(tag, []).append(value)+ else:+ raise ValueError("unterminated ELF dynamic table")+ if any(len(tags.get(tag, [])) != 1 for tag in (5, 10)):+ raise ValueError("invalid ELF dynamic string table")+ address, size = tags[5][0], tags[10][0]+ offsets = [+ offset + address - start+ for start, offset, length in loads+ if start <= address and address + size <= start + length+ ]+ if len(offsets) != 1 or not 1 <= size <= 1024 * 1024:+ raise ValueError("invalid ELF string table bounds")+ strings = data[offsets[0] : offsets[0] + size]+ values = {}+ for tag in (1, 14, 15, 29):+ values[tag] = []+ if tag != 1 and len(tags.get(tag, [])) > 1:+ raise ValueError("duplicate ELF loader metadata")+ for offset in tags.get(tag, []):+ end = strings.find(b"\0", offset)+ if not 0 <= offset < len(strings) or end < 0:+ raise ValueError("invalid ELF loader string")+ value = (+ os.fsdecode(strings[offset:end])+ if tag in (15, 29)+ else strings[offset:end].decode("ascii")+ )+ if tag in (1, 14) and not re.fullmatch(+ r"[A-Za-z0-9_+.-]+\.so(?:\.[0-9]+)*", value+ ):+ raise ValueError("ELF dependencies must be plain library names")+ values[tag].append(value)+ identity = values[14][0] if values[14] else path.name+ return Binary(+ identity,+ tuple(values[1]),+ tuple(f"{tag}={value}" for tag in (15, 29) for value in values[tag]),+ )+++def finalize_copy(destination, metadata, dependency_paths):+ allowed = {"29=$ORIGIN", "29=$ORIGIN:$ORIGIN/.."}+ needs_parent = destination.parent.name == "gstreamer-1.0"+ required = "29=$ORIGIN:$ORIGIN/.." if needs_parent else "29=$ORIGIN"+ if any(path not in allowed for path in metadata.rpaths) or (+ any(name not in SYSTEM_IMPORTS for name in metadata.imports)+ and required not in metadata.rpaths+ and "29=$ORIGIN:$ORIGIN/.." not in metadata.rpaths+ ):+ raise ValueError("rebuild native libraries with package-relative runtime paths")+ return metadata+++def project(prefix, receipts, target, output):+ if sys.platform != "linux" or target not in (+ "x86_64-unknown-linux-gnu",+ "aarch64-unknown-linux-gnu",+ ):+ raise ValueError(+ "runtime preparation requires GNU Linux and an explicit GNU target"+ )+ format = RuntimeFormat(+ tuple(Path(f"lib/gstreamer-1.0/libgst{name}.so") for name in sorted(PLUGINS)),+ SYSTEM_IMPORTS,+ inspect,+ finalize_copy,+ plugin_dir="lib/gstreamer-1.0",+ )+ prepare(prefix, receipts, target, output, format)+++if __name__ == "__main__":+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--prefix", type=Path, required=True)+ parser.add_argument("--receipts", type=Path, required=True)+ parser.add_argument("--target", required=True)+ parser.add_argument("--output", type=Path, required=True)+ args = parser.parse_args()+ project(args.prefix, args.receipts, args.target, args.output)diff --git a/third_party/voice/runtime.py b/third_party/voice/runtime.pyindex c50055e18b..cbc51855bb 100644--- a/third_party/voice/runtime.py+++ b/third_party/voice/runtime.py@@ -142,7 +142,7 @@ def prepare(prefix, receipts, target, output, format):for relative in sorted(selected):record, metadata = binaries[relative]destination = output / destinations[relative]- destination.parent.mkdir(exist_ok=True)+ destination.parent.mkdir(parents=True, exist_ok=True)shutil.copy2(prefix / relative, destination)if (digest(destination) != record["sha256"]diff --git a/third_party/voice/test_build_native.py b/third_party/voice/test_build_native.pyindex a0d1d54467..7da0ce2b56 100644--- a/third_party/voice/test_build_native.py+++ b/third_party/voice/test_build_native.py@@ -119,6 +119,42 @@ class NativeBuildTests(unittest.TestCase):)self.assertEqual(result.stdout, "linked")+ @unittest.skipIf(os.name == "nt", "Unix install paths; Windows layout is unchanged")+ def test_cmake_installs_relative_library_paths(self):+ for argument, tool in (+ ("cc", "cc"),+ ("cxx", "c++"),+ ("cmake", "cmake"),+ ("make", "make"),+ ):+ setattr(self.args, argument, Path(shutil.which(tool)))+ source = self.root / "library-source"+ source.mkdir()+ (source / "CMakeLists.txt").write_text(+ "cmake_minimum_required(VERSION 3.15)\nproject(relative_paths LANGUAGES C)\n"+ "add_library(fixture SHARED fixture.c)\ninstall(TARGETS fixture DESTINATION lib)\n"+ )+ (source / "fixture.c").write_text("int fixture(void) { return 42; }\n")+ build = NativeBuild(self.args, self.environment)+ build.output.mkdir()+ build.sources = {"fixture": source}+ build.cmake("fixture", [], bootstrap=True)+ if sys.platform == "darwin":+ from macos_runtime import inspect++ metadata = inspect(build.tools / "lib/libfixture.dylib", self.args.target)+ self.assertEqual(+ (metadata.identity, metadata.rpaths),+ ("@rpath/libfixture.dylib", ("@loader_path",)),+ )+ else:+ from linux_runtime import inspect++ metadata = inspect(build.tools / "lib/libfixture.so", self.args.target)+ self.assertEqual(+ (metadata.identity, metadata.rpaths), ("libfixture.so", ("29=$ORIGIN",))+ )+def test_build_refuses_existing_output_before_reading_sources(self):self.args.output.mkdir()(self.args.output / "keep").write_text("untouched")@@ -178,7 +214,12 @@ class NativeBuildTests(unittest.TestCase):else:self.assertEqual(shlex.split(build.environment["LDFLAGS"]),- [f"-L{build.prefix / 'lib'}", f"-Wl,-rpath,{build.prefix / 'lib'}"],+ [+ f"-L{build.prefix / 'lib'}",+ "-Wl,-rpath,$ORIGIN:$ORIGIN/.."+ if build.args.target.endswith("unknown-linux-gnu")+ else f"-Wl,-rpath,{build.prefix / 'lib'}",+ ],)def test_windows_paths_use_cygpath_and_propagate_failure(self):diff --git a/third_party/voice/test_linux_runtime.py b/third_party/voice/test_linux_runtime.pynew file mode 100644index 0000000000..7cf8a8c44f--- /dev/null+++ b/third_party/voice/test_linux_runtime.py@@ -0,0 +1,255 @@+"""Exercise real ELF relocation and rejection of unsafe native build inputs."""++import json+import os+from pathlib import Path+import platform+import shutil+import struct+import subprocess+import sys+import tempfile+import unittest+from unittest.mock import patch++from linux_runtime import inspect, project+from runtime import PLUGINS, digest++[email protected](sys.platform == "linux", "GNU Linux native runtime preparation")+class RuntimeTests(unittest.TestCase):+ def setUp(self):+ temporary = tempfile.TemporaryDirectory(prefix="native voice ")+ self.addCleanup(temporary.cleanup)+ self.root = Path(temporary.name)+ self.prefix, self.receipts, self.output = (+ self.root / name for name in ("input é", "receipts", "runtime")+ )+ self.target = (+ "aarch64" if platform.machine() == "aarch64" else "x86_64"+ ) + "-unknown-linux-gnu"+ (self.prefix / "lib/gstreamer-1.0").mkdir(parents=True)+ (self.receipts / "inspection").mkdir(parents=True)+ source = self.root / "fixture.c"+ source.write_text("int voice_fixture(void) { return 42; }\n")+ self.library = self.prefix / "lib/libfixture.so.1.2"+ subprocess.run(+ [+ "cc",+ "-shared",+ "-fPIC",+ str(source),+ "-o",+ str(self.library),+ "-Wl,-soname,libfixture.so.1",+ ],+ check=True,+ capture_output=True,+ )+ source.write_text(+ "extern int voice_fixture(void); int voice_plugin(void) { return voice_fixture(); }\n"+ )+ for name in PLUGINS:+ plugin = self.prefix / f"lib/gstreamer-1.0/libgst{name}.so"+ subprocess.run(+ [+ "cc",+ "-shared",+ "-fPIC",+ str(source),+ str(self.library),+ "-o",+ str(plugin),+ f"-Wl,-soname,{plugin.name}",+ "-Wl,-rpath,$ORIGIN:$ORIGIN/..",+ ],+ check=True,+ capture_output=True,+ )+ self.inventory_path = self.receipts / "inspection/binaries.json"+ self.inventory = [+ {+ "path": p.relative_to(self.prefix).as_posix(),+ "sha256": digest(p),+ "target": self.target,+ }+ for p in sorted(self.prefix.rglob("*.so*"))+ if p.is_file()+ ]+ self.inventory_path.write_text(json.dumps(self.inventory))+ (self.receipts / "ci.json").write_text(+ json.dumps(+ {+ "commit": "a" * 40,+ "target": self.target,+ "build_complete": True,+ "inspection_complete": True,+ "manifest_sha256": digest(Path(__file__).with_name("sources.json")),+ }+ )+ )++ def test_relocated_libraries_load_without_original_prefix(self):+ subprocess.run(+ [+ sys.executable,+ str(Path(__file__).with_name("linux_runtime.py").resolve()),+ "--prefix",+ str(self.prefix),+ "--receipts",+ str(self.receipts),+ "--target",+ self.target,+ "--output",+ str(self.output),+ ],+ check=True,+ capture_output=True,+ cwd=self.root,+ env={**os.environ, "PYTHONSAFEPATH": "1"},+ )+ self.assertEqual(+ {r["path"]: digest(self.prefix / r["path"]) for r in self.inventory},+ {r["path"]: r["sha256"] for r in self.inventory},+ )+ moved = self.root / "moved runtime"+ self.output.rename(moved)+ shutil.rmtree(self.prefix)+ environment = {k: v for k, v in os.environ.items() if not k.startswith("LD_")}+ result = subprocess.run(+ [+ sys.executable,+ "-c",+ "import ctypes,json,pathlib,sys; root=pathlib.Path(sys.argv[1]); "+ "manifest=json.loads((root/'runtime.json').read_text()); "+ "print([ctypes.CDLL(str(root/p)).voice_plugin() for p in manifest['plugins']])",+ str(moved),+ ],+ check=True,+ capture_output=True,+ text=True,+ env=environment,+ cwd=self.root,+ )+ self.assertEqual(json.loads(result.stdout), [42] * len(PLUGINS))+ manifest = json.loads((moved / "runtime.json").read_text())+ self.assertEqual(len(manifest["libraries"]), len(PLUGINS) + 1)+ for record in manifest["libraries"]:+ path = moved / record["path"]+ expected = (+ ("29=$ORIGIN:$ORIGIN/..",)+ if path.parent.name == "gstreamer-1.0"+ else ()+ )+ self.assertEqual(inspect(path, self.target).rpaths, expected)+ self.assertEqual(digest(path), record["sourceSha256"])+ self.assertEqual(record["sha256"], record["sourceSha256"])++ def test_absolute_build_paths_require_a_new_native_build(self):+ plugin = self.prefix / "lib/gstreamer-1.0/libgstapp.so"+ subprocess.run(+ ["patchelf", "--set-rpath", str(self.prefix / "lib"), str(plugin)],+ check=True,+ capture_output=True,+ )+ for record in self.inventory:+ record["sha256"] = digest(self.prefix / record["path"])+ self.inventory_path.write_text(json.dumps(self.inventory))+ with self.assertRaisesRegex(ValueError, "rebuild native libraries"):+ project(self.prefix, self.receipts, self.target, self.output)+ self.assertFalse(self.output.exists())++ def test_missing_dependency_and_digest_mismatch_leave_no_output(self):+ for records in (+ [r for r in self.inventory if "libfixture" not in r["path"]],+ [{**r, "sha256": "b" * 64} for r in self.inventory],+ ):+ with self.subTest(records=records), self.assertRaises(ValueError):+ self.inventory_path.write_text(json.dumps(records))+ project(self.prefix, self.receipts, self.target, self.output)+ self.assertFalse(self.output.exists())++ def test_path_import_is_not_treated_as_a_system_library(self):+ plugin = self.prefix / "lib/gstreamer-1.0/libgstapp.so"+ subprocess.run(+ [+ "patchelf",+ "--replace-needed",+ "libfixture.so.1",+ "outside\nlibc.so.6",+ str(plugin),+ ],+ check=True,+ capture_output=True,+ )+ with self.assertRaisesRegex(ValueError, "plain library names"):+ inspect(plugin, self.target)++ def test_malformed_header_and_segment_bounds_are_rejected(self):+ original = self.library.read_bytes()+ for offset, replacement in (+ (18, b"\x00\x00"),+ (32, struct.pack("<Q", len(original))),+ (56, b"\xff\xff"),+ ):+ data = bytearray(original)+ data[offset : offset + len(replacement)] = replacement+ self.library.write_bytes(data)+ with self.subTest(offset=offset), self.assertRaises(ValueError):+ inspect(self.library, self.target)++ def test_additional_loader_dependencies_are_rejected(self):+ data = bytearray(self.library.read_bytes())+ header = struct.unpack_from("<16sHHIQQQIHHHHHH", data)+ for index in range(header[10]):+ segment = struct.unpack_from("<IIQQQQQQ", data, header[5] + index * 56)+ if segment[0] == 2:+ struct.pack_into("<qQ", data, segment[2], 0x7FFFFFFF, 0)+ break+ self.library.write_bytes(data)+ with self.assertRaisesRegex(ValueError, "unsupported ELF loader"):+ inspect(self.library, self.target)++ def test_dynamic_table_mapping_is_checked_before_creating_output(self):+ # An existing intended runpath can make patchelf leave this malformed table untouched.+ subprocess.run(+ ["patchelf", "--set-rpath", "$ORIGIN", str(self.library)],+ check=True,+ capture_output=True,+ )+ original = self.library.read_bytes()+ header = struct.unpack_from("<16sHHIQQQIHHHHHH", original)+ dynamic_offset = next(+ header[5] + index * 56+ for index in range(header[10])+ if struct.unpack_from("<I", original, header[5] + index * 56)[0] == 2+ )+ address = struct.unpack_from("<Q", original, dynamic_offset + 16)[0]+ for invalid_address in (address + 8, 0x77770000):+ data = bytearray(original)+ struct.pack_into("<Q", data, dynamic_offset + 16, invalid_address)+ self.library.write_bytes(data)+ self.inventory_path.write_text(+ json.dumps(+ [+ {**record, "sha256": digest(self.prefix / record["path"])}+ for record in self.inventory+ ]+ )+ )+ with (+ self.subTest(address=invalid_address),+ self.assertRaisesRegex(ValueError, "dynamic table.*mapping"),+ ):+ project(self.prefix, self.receipts, self.target, self.output)+ self.assertFalse(self.output.exists())++ def test_failed_copy_removes_only_fresh_output(self):+ before = {r["path"]: digest(self.prefix / r["path"]) for r in self.inventory}+ with patch("runtime.shutil.copy2", side_effect=RuntimeError("copy failed")):+ with self.assertRaisesRegex(RuntimeError, "copy failed"):+ project(self.prefix, self.receipts, self.target, self.output)+ self.assertFalse(self.output.exists())+ self.assertEqual(+ before, {r["path"]: digest(self.prefix / r["path"]) for r in self.inventory}+ )