Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add GNU Linux voice runtime preparation (#42208)

Benjamin Carlsson8d01cd42faeb
diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel
index 33fac3e46a..4e6f3cb62c 100644
--- a/third_party/voice/BUILD.bazel
+++ b/third_party/voice/BUILD.bazel
@@ -33,6 +33,7 @@ filegroup(
srcs = [
"assemble_package.py",
"build_native.py",
+ "linux_runtime.py",
"macos_runtime.py",
"runtime.py",
":source_inputs",
diff --git a/third_party/voice/README.md b/third_party/voice/README.md
index b60234138a..efc73f2ddb 100644
--- a/third_party/voice/README.md
+++ b/third_party/voice/README.md
@@ -74,6 +74,12 @@ must still match the real native target. Native pkgconf relocates libffi's POSIX
prefix metadata; CI rejects residual Cygwin paths. These are build prerequisites,
not shipped runtime components or evidence of working voice.
+CMake libraries use relative install runpaths (`$ORIGIN` on Linux and
+`@loader_path` on Mac), with `@rpath` install names on Mac. Linux Meson links
+use `$ORIGIN:$ORIGIN/..`, matching libraries in `lib/` and plugins in
+`lib/gstreamer-1.0/`. Mac Meson and libffi still need packaging-time fixups;
+these options do not make every Mac library relocatable at installation.
+
Outputs are under `prefix/`, build tools under `tools/`, and logs beside them.
`build-state.json` records completed commands and failures; `built.json` exists
only when every build/install command succeeds. Failed builds retain their logs
@@ -110,3 +116,18 @@ audio behavior. Dynamic-only dependencies, native helper linkage, LGPL notices,
production signing/notarization, Windows/Linux loading and security approval remain
separate requirements. No microphone, device, plugin scanner or backend is started
by projection. Run its native relocation tests on macOS with Python 3.12 or newer.
+
+## Private GNU Linux runtime preparation
+
+`linux_runtime.py` takes the same prefix, receipts, target and output arguments.
+It reads bounded ELF64 headers, segments and dynamic tables directly and accepts
+x64/ARM64 GNU Linux libraries. The shared Python coordinator selects the seven
+plugins and their declared dependencies without changing their bytes. The native
+build must have emitted package-relative runpaths; older absolute paths are
+rejected with a rebuild instruction. Output preserves `lib/gstreamer-1.0/` so
+those relative paths remain valid. Loader audit/filter dependencies and
+path-bearing imports are rejected. Native tests require Python 3.12, a C compiler
+and `patchelf`; the latter constructs malformed inputs and is not needed during
+preparation or shipped in the runtime. The output is development-only, uses the
+host glibc, and does not establish musl or minimum-glibc support, dynamic-only
+dependency closure, helper loading policy or working voice.
diff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.py
index d75cc1fc81..adb75b6e4a 100644
--- a/third_party/voice/build_native.py
+++ b/third_party/voice/build_native.py
@@ -130,7 +130,14 @@ class NativeBuild:
if args.target.endswith("apple-darwin"):
self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_target
self.cmake_platform = [
- f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}"
+ f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}",
+ "-DCMAKE_INSTALL_NAME_DIR=@rpath",
+ "-DCMAKE_INSTALL_RPATH=@loader_path",
+ ]
+ elif not self.windows:
+ self.cmake_platform = [
+ "-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON",
+ "-DCMAKE_INSTALL_RPATH=$ORIGIN",
]
self.record = {
"target": args.target,
@@ -223,6 +230,8 @@ class NativeBuild:
if self.windows
else [f"-L{self.prefix / 'lib'}", f"-Wl,-rpath,{self.prefix / 'lib'}"]
)
+ if self.args.target.endswith("unknown-linux-gnu"):
+ link[-1] = "-Wl,-rpath,$ORIGIN:$ORIGIN/.."
self.environment.update(
{"CFLAGS": include, "CXXFLAGS": include, "LDFLAGS": quote(link)}
)
diff --git a/third_party/voice/linux_runtime.py b/third_party/voice/linux_runtime.py
new file mode 100644
index 0000000000..124e5d40e1
--- /dev/null
+++ b/third_party/voice/linux_runtime.py
@@ -0,0 +1,152 @@
+"""Prepare verified GNU Linux audio libraries with package-relative loader paths."""
+
+import argparse
+import os
+from pathlib import Path
+import re
+import struct
+import sys
+
+# Import only this script's siblings, including under PYTHONSAFEPATH.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from runtime import Binary, PLUGINS, RuntimeFormat, prepare
+
+SYSTEM_IMPORTS = frozenset(
+ {
+ "libc.so.6",
+ "libm.so.6",
+ "libdl.so.2",
+ "libpthread.so.0",
+ "librt.so.1",
+ "libresolv.so.2",
+ "ld-linux-x86-64.so.2",
+ "ld-linux-aarch64.so.1",
+ }
+)
+
+
+def inspect(path, target):
+ machine = {"x86_64-unknown-linux-gnu": 62, "aarch64-unknown-linux-gnu": 183}[target]
+ with path.open("rb") as source:
+ data = source.read(64 * 1024 * 1024 + 1)
+ if not 64 <= len(data) <= 64 * 1024 * 1024:
+ raise ValueError("invalid ELF file size")
+ header = struct.unpack_from("<16sHHIQQQIHHHHHH", data)
+ if header[0][:7] != b"\x7fELF\x02\x01\x01" or header[0][7] not in (0, 3):
+ raise ValueError("expected a little-endian ELF64 library")
+ if header[1:4] != (3, machine, 1) or header[8:10] != (64, 56):
+ raise ValueError(f"expected a {target} shared library")
+ phoff, count = header[5], header[10]
+ if not 1 <= count <= 128 or phoff < 64 or phoff + count * 56 > len(data):
+ raise ValueError("invalid ELF program headers")
+ loads, dynamic = [], []
+ for index in range(count):
+ kind, _, offset, address, _, size, memory_size, _ = struct.unpack_from(
+ "<IIQQQQQQ", data, phoff + index * 56
+ )
+ if size > memory_size or offset + size > len(data):
+ raise ValueError("invalid ELF segment bounds")
+ if kind == 1:
+ loads.append((address, offset, size))
+ elif kind == 2:
+ dynamic.append((offset, size, address))
+ elif kind == 3:
+ raise ValueError("an ELF runtime library must not name an interpreter")
+ if len(dynamic) != 1 or not 16 <= dynamic[0][1] <= 65536 or dynamic[0][1] % 16:
+ raise ValueError("invalid ELF dynamic table")
+ tags = {}
+ offset, size, address = dynamic[0]
+ mappings = [
+ file_offset + address - start
+ for start, file_offset, length in loads
+ if start <= address and address + size <= start + length
+ ]
+ if mappings != [offset]:
+ raise ValueError("ELF dynamic table must have one matching file-backed mapping")
+ for cursor in range(offset, offset + size, 16):
+ tag, value = struct.unpack_from("<qQ", data, cursor)
+ if tag == 0:
+ break
+ # Auxiliary/filter libraries and audit modules are additional loader inputs.
+ if tag in (0x7FFFFFFD, 0x7FFFFFFF, 0x6FFFFEFB, 0x6FFFFEFC):
+ raise ValueError("unsupported ELF loader dependency")
+ tags.setdefault(tag, []).append(value)
+ else:
+ raise ValueError("unterminated ELF dynamic table")
+ if any(len(tags.get(tag, [])) != 1 for tag in (5, 10)):
+ raise ValueError("invalid ELF dynamic string table")
+ address, size = tags[5][0], tags[10][0]
+ offsets = [
+ offset + address - start
+ for start, offset, length in loads
+ if start <= address and address + size <= start + length
+ ]
+ if len(offsets) != 1 or not 1 <= size <= 1024 * 1024:
+ raise ValueError("invalid ELF string table bounds")
+ strings = data[offsets[0] : offsets[0] + size]
+ values = {}
+ for tag in (1, 14, 15, 29):
+ values[tag] = []
+ if tag != 1 and len(tags.get(tag, [])) > 1:
+ raise ValueError("duplicate ELF loader metadata")
+ for offset in tags.get(tag, []):
+ end = strings.find(b"\0", offset)
+ if not 0 <= offset < len(strings) or end < 0:
+ raise ValueError("invalid ELF loader string")
+ value = (
+ os.fsdecode(strings[offset:end])
+ if tag in (15, 29)
+ else strings[offset:end].decode("ascii")
+ )
+ if tag in (1, 14) and not re.fullmatch(
+ r"[A-Za-z0-9_+.-]+\.so(?:\.[0-9]+)*", value
+ ):
+ raise ValueError("ELF dependencies must be plain library names")
+ values[tag].append(value)
+ identity = values[14][0] if values[14] else path.name
+ return Binary(
+ identity,
+ tuple(values[1]),
+ tuple(f"{tag}={value}" for tag in (15, 29) for value in values[tag]),
+ )
+
+
+def finalize_copy(destination, metadata, dependency_paths):
+ allowed = {"29=$ORIGIN", "29=$ORIGIN:$ORIGIN/.."}
+ needs_parent = destination.parent.name == "gstreamer-1.0"
+ required = "29=$ORIGIN:$ORIGIN/.." if needs_parent else "29=$ORIGIN"
+ if any(path not in allowed for path in metadata.rpaths) or (
+ any(name not in SYSTEM_IMPORTS for name in metadata.imports)
+ and required not in metadata.rpaths
+ and "29=$ORIGIN:$ORIGIN/.." not in metadata.rpaths
+ ):
+ raise ValueError("rebuild native libraries with package-relative runtime paths")
+ return metadata
+
+
+def project(prefix, receipts, target, output):
+ if sys.platform != "linux" or target not in (
+ "x86_64-unknown-linux-gnu",
+ "aarch64-unknown-linux-gnu",
+ ):
+ raise ValueError(
+ "runtime preparation requires GNU Linux and an explicit GNU target"
+ )
+ format = RuntimeFormat(
+ tuple(Path(f"lib/gstreamer-1.0/libgst{name}.so") for name in sorted(PLUGINS)),
+ SYSTEM_IMPORTS,
+ inspect,
+ finalize_copy,
+ plugin_dir="lib/gstreamer-1.0",
+ )
+ prepare(prefix, receipts, target, output, format)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--prefix", type=Path, required=True)
+ parser.add_argument("--receipts", type=Path, required=True)
+ parser.add_argument("--target", required=True)
+ parser.add_argument("--output", type=Path, required=True)
+ args = parser.parse_args()
+ project(args.prefix, args.receipts, args.target, args.output)
diff --git a/third_party/voice/runtime.py b/third_party/voice/runtime.py
index c50055e18b..cbc51855bb 100644
--- a/third_party/voice/runtime.py
+++ b/third_party/voice/runtime.py
@@ -142,7 +142,7 @@ def prepare(prefix, receipts, target, output, format):
for relative in sorted(selected):
record, metadata = binaries[relative]
destination = output / destinations[relative]
- destination.parent.mkdir(exist_ok=True)
+ destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(prefix / relative, destination)
if (
digest(destination) != record["sha256"]
diff --git a/third_party/voice/test_build_native.py b/third_party/voice/test_build_native.py
index a0d1d54467..7da0ce2b56 100644
--- a/third_party/voice/test_build_native.py
+++ b/third_party/voice/test_build_native.py
@@ -119,6 +119,42 @@ class NativeBuildTests(unittest.TestCase):
)
self.assertEqual(result.stdout, "linked")
+ @unittest.skipIf(os.name == "nt", "Unix install paths; Windows layout is unchanged")
+ def test_cmake_installs_relative_library_paths(self):
+ for argument, tool in (
+ ("cc", "cc"),
+ ("cxx", "c++"),
+ ("cmake", "cmake"),
+ ("make", "make"),
+ ):
+ setattr(self.args, argument, Path(shutil.which(tool)))
+ source = self.root / "library-source"
+ source.mkdir()
+ (source / "CMakeLists.txt").write_text(
+ "cmake_minimum_required(VERSION 3.15)\nproject(relative_paths LANGUAGES C)\n"
+ "add_library(fixture SHARED fixture.c)\ninstall(TARGETS fixture DESTINATION lib)\n"
+ )
+ (source / "fixture.c").write_text("int fixture(void) { return 42; }\n")
+ build = NativeBuild(self.args, self.environment)
+ build.output.mkdir()
+ build.sources = {"fixture": source}
+ build.cmake("fixture", [], bootstrap=True)
+ if sys.platform == "darwin":
+ from macos_runtime import inspect
+
+ metadata = inspect(build.tools / "lib/libfixture.dylib", self.args.target)
+ self.assertEqual(
+ (metadata.identity, metadata.rpaths),
+ ("@rpath/libfixture.dylib", ("@loader_path",)),
+ )
+ else:
+ from linux_runtime import inspect
+
+ metadata = inspect(build.tools / "lib/libfixture.so", self.args.target)
+ self.assertEqual(
+ (metadata.identity, metadata.rpaths), ("libfixture.so", ("29=$ORIGIN",))
+ )
+
def test_build_refuses_existing_output_before_reading_sources(self):
self.args.output.mkdir()
(self.args.output / "keep").write_text("untouched")
@@ -178,7 +214,12 @@ class NativeBuildTests(unittest.TestCase):
else:
self.assertEqual(
shlex.split(build.environment["LDFLAGS"]),
- [f"-L{build.prefix / 'lib'}", f"-Wl,-rpath,{build.prefix / 'lib'}"],
+ [
+ f"-L{build.prefix / 'lib'}",
+ "-Wl,-rpath,$ORIGIN:$ORIGIN/.."
+ if build.args.target.endswith("unknown-linux-gnu")
+ else f"-Wl,-rpath,{build.prefix / 'lib'}",
+ ],
)
def test_windows_paths_use_cygpath_and_propagate_failure(self):
diff --git a/third_party/voice/test_linux_runtime.py b/third_party/voice/test_linux_runtime.py
new file mode 100644
index 0000000000..7cf8a8c44f
--- /dev/null
+++ b/third_party/voice/test_linux_runtime.py
@@ -0,0 +1,255 @@
+"""Exercise real ELF relocation and rejection of unsafe native build inputs."""
+
+import json
+import os
+from pathlib import Path
+import platform
+import shutil
+import struct
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from linux_runtime import inspect, project
+from runtime import PLUGINS, digest
+
+
[email protected](sys.platform == "linux", "GNU Linux native runtime preparation")
+class RuntimeTests(unittest.TestCase):
+ def setUp(self):
+ temporary = tempfile.TemporaryDirectory(prefix="native voice ")
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name)
+ self.prefix, self.receipts, self.output = (
+ self.root / name for name in ("input é", "receipts", "runtime")
+ )
+ self.target = (
+ "aarch64" if platform.machine() == "aarch64" else "x86_64"
+ ) + "-unknown-linux-gnu"
+ (self.prefix / "lib/gstreamer-1.0").mkdir(parents=True)
+ (self.receipts / "inspection").mkdir(parents=True)
+ source = self.root / "fixture.c"
+ source.write_text("int voice_fixture(void) { return 42; }\n")
+ self.library = self.prefix / "lib/libfixture.so.1.2"
+ subprocess.run(
+ [
+ "cc",
+ "-shared",
+ "-fPIC",
+ str(source),
+ "-o",
+ str(self.library),
+ "-Wl,-soname,libfixture.so.1",
+ ],
+ check=True,
+ capture_output=True,
+ )
+ source.write_text(
+ "extern int voice_fixture(void); int voice_plugin(void) { return voice_fixture(); }\n"
+ )
+ for name in PLUGINS:
+ plugin = self.prefix / f"lib/gstreamer-1.0/libgst{name}.so"
+ subprocess.run(
+ [
+ "cc",
+ "-shared",
+ "-fPIC",
+ str(source),
+ str(self.library),
+ "-o",
+ str(plugin),
+ f"-Wl,-soname,{plugin.name}",
+ "-Wl,-rpath,$ORIGIN:$ORIGIN/..",
+ ],
+ check=True,
+ capture_output=True,
+ )
+ self.inventory_path = self.receipts / "inspection/binaries.json"
+ self.inventory = [
+ {
+ "path": p.relative_to(self.prefix).as_posix(),
+ "sha256": digest(p),
+ "target": self.target,
+ }
+ for p in sorted(self.prefix.rglob("*.so*"))
+ if p.is_file()
+ ]
+ self.inventory_path.write_text(json.dumps(self.inventory))
+ (self.receipts / "ci.json").write_text(
+ json.dumps(
+ {
+ "commit": "a" * 40,
+ "target": self.target,
+ "build_complete": True,
+ "inspection_complete": True,
+ "manifest_sha256": digest(Path(__file__).with_name("sources.json")),
+ }
+ )
+ )
+
+ def test_relocated_libraries_load_without_original_prefix(self):
+ subprocess.run(
+ [
+ sys.executable,
+ str(Path(__file__).with_name("linux_runtime.py").resolve()),
+ "--prefix",
+ str(self.prefix),
+ "--receipts",
+ str(self.receipts),
+ "--target",
+ self.target,
+ "--output",
+ str(self.output),
+ ],
+ check=True,
+ capture_output=True,
+ cwd=self.root,
+ env={**os.environ, "PYTHONSAFEPATH": "1"},
+ )
+ self.assertEqual(
+ {r["path"]: digest(self.prefix / r["path"]) for r in self.inventory},
+ {r["path"]: r["sha256"] for r in self.inventory},
+ )
+ moved = self.root / "moved runtime"
+ self.output.rename(moved)
+ shutil.rmtree(self.prefix)
+ environment = {k: v for k, v in os.environ.items() if not k.startswith("LD_")}
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ "import ctypes,json,pathlib,sys; root=pathlib.Path(sys.argv[1]); "
+ "manifest=json.loads((root/'runtime.json').read_text()); "
+ "print([ctypes.CDLL(str(root/p)).voice_plugin() for p in manifest['plugins']])",
+ str(moved),
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ env=environment,
+ cwd=self.root,
+ )
+ self.assertEqual(json.loads(result.stdout), [42] * len(PLUGINS))
+ manifest = json.loads((moved / "runtime.json").read_text())
+ self.assertEqual(len(manifest["libraries"]), len(PLUGINS) + 1)
+ for record in manifest["libraries"]:
+ path = moved / record["path"]
+ expected = (
+ ("29=$ORIGIN:$ORIGIN/..",)
+ if path.parent.name == "gstreamer-1.0"
+ else ()
+ )
+ self.assertEqual(inspect(path, self.target).rpaths, expected)
+ self.assertEqual(digest(path), record["sourceSha256"])
+ self.assertEqual(record["sha256"], record["sourceSha256"])
+
+ def test_absolute_build_paths_require_a_new_native_build(self):
+ plugin = self.prefix / "lib/gstreamer-1.0/libgstapp.so"
+ subprocess.run(
+ ["patchelf", "--set-rpath", str(self.prefix / "lib"), str(plugin)],
+ check=True,
+ capture_output=True,
+ )
+ for record in self.inventory:
+ record["sha256"] = digest(self.prefix / record["path"])
+ self.inventory_path.write_text(json.dumps(self.inventory))
+ with self.assertRaisesRegex(ValueError, "rebuild native libraries"):
+ project(self.prefix, self.receipts, self.target, self.output)
+ self.assertFalse(self.output.exists())
+
+ def test_missing_dependency_and_digest_mismatch_leave_no_output(self):
+ for records in (
+ [r for r in self.inventory if "libfixture" not in r["path"]],
+ [{**r, "sha256": "b" * 64} for r in self.inventory],
+ ):
+ with self.subTest(records=records), self.assertRaises(ValueError):
+ self.inventory_path.write_text(json.dumps(records))
+ project(self.prefix, self.receipts, self.target, self.output)
+ self.assertFalse(self.output.exists())
+
+ def test_path_import_is_not_treated_as_a_system_library(self):
+ plugin = self.prefix / "lib/gstreamer-1.0/libgstapp.so"
+ subprocess.run(
+ [
+ "patchelf",
+ "--replace-needed",
+ "libfixture.so.1",
+ "outside\nlibc.so.6",
+ str(plugin),
+ ],
+ check=True,
+ capture_output=True,
+ )
+ with self.assertRaisesRegex(ValueError, "plain library names"):
+ inspect(plugin, self.target)
+
+ def test_malformed_header_and_segment_bounds_are_rejected(self):
+ original = self.library.read_bytes()
+ for offset, replacement in (
+ (18, b"\x00\x00"),
+ (32, struct.pack("<Q", len(original))),
+ (56, b"\xff\xff"),
+ ):
+ data = bytearray(original)
+ data[offset : offset + len(replacement)] = replacement
+ self.library.write_bytes(data)
+ with self.subTest(offset=offset), self.assertRaises(ValueError):
+ inspect(self.library, self.target)
+
+ def test_additional_loader_dependencies_are_rejected(self):
+ data = bytearray(self.library.read_bytes())
+ header = struct.unpack_from("<16sHHIQQQIHHHHHH", data)
+ for index in range(header[10]):
+ segment = struct.unpack_from("<IIQQQQQQ", data, header[5] + index * 56)
+ if segment[0] == 2:
+ struct.pack_into("<qQ", data, segment[2], 0x7FFFFFFF, 0)
+ break
+ self.library.write_bytes(data)
+ with self.assertRaisesRegex(ValueError, "unsupported ELF loader"):
+ inspect(self.library, self.target)
+
+ def test_dynamic_table_mapping_is_checked_before_creating_output(self):
+ # An existing intended runpath can make patchelf leave this malformed table untouched.
+ subprocess.run(
+ ["patchelf", "--set-rpath", "$ORIGIN", str(self.library)],
+ check=True,
+ capture_output=True,
+ )
+ original = self.library.read_bytes()
+ header = struct.unpack_from("<16sHHIQQQIHHHHHH", original)
+ dynamic_offset = next(
+ header[5] + index * 56
+ for index in range(header[10])
+ if struct.unpack_from("<I", original, header[5] + index * 56)[0] == 2
+ )
+ address = struct.unpack_from("<Q", original, dynamic_offset + 16)[0]
+ for invalid_address in (address + 8, 0x77770000):
+ data = bytearray(original)
+ struct.pack_into("<Q", data, dynamic_offset + 16, invalid_address)
+ self.library.write_bytes(data)
+ self.inventory_path.write_text(
+ json.dumps(
+ [
+ {**record, "sha256": digest(self.prefix / record["path"])}
+ for record in self.inventory
+ ]
+ )
+ )
+ with (
+ self.subTest(address=invalid_address),
+ self.assertRaisesRegex(ValueError, "dynamic table.*mapping"),
+ ):
+ project(self.prefix, self.receipts, self.target, self.output)
+ self.assertFalse(self.output.exists())
+
+ def test_failed_copy_removes_only_fresh_output(self):
+ before = {r["path"]: digest(self.prefix / r["path"]) for r in self.inventory}
+ with patch("runtime.shutil.copy2", side_effect=RuntimeError("copy failed")):
+ with self.assertRaisesRegex(RuntimeError, "copy failed"):
+ project(self.prefix, self.receipts, self.target, self.output)
+ self.assertFalse(self.output.exists())
+ self.assertEqual(
+ before, {r["path"]: digest(self.prefix / r["path"]) for r in self.inventory}
+ )