Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: propagate the Ollama CUDA runtime to llama-server (replacement for #7563) (#11666)
Wasim Yousef Said828234f4ab52
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 36574443cc9..8787d26b27f 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -15446,9 +15446,17 @@ class LlamaCppBackend:]:if os.path.isdir(cuda_lib):lib_dirs.append(cuda_lib)++ # Vendored dirs go last: rescue only, never displace a runtime already found.+ from utils.llama_cpp_freshness import read_install_marker+ from utils.prebuilt.runtime_libs import vendored_cuda_runtime_dirs++ marker_binary = str(_resolve_llama_binary(binary))+ vendored_cuda_dirs = vendored_cuda_runtime_dirs(read_install_marker(marker_binary))existing_ld = env.get("LD_LIBRARY_PATH", "")- new_ld = ":".join(lib_dirs)- env["LD_LIBRARY_PATH"] = f"{new_ld}:{existing_ld}" if existing_ld else new_ld+ env["LD_LIBRARY_PATH"] = ":".join(+ path for path in [*lib_dirs, existing_ld, *vendored_cuda_dirs] if path+ )return envdiff --git a/studio/backend/core/inference/stt_ggml_sidecar.py b/studio/backend/core/inference/stt_ggml_sidecar.pyindex 7c3be289550..7643eb74002 100644--- a/studio/backend/core/inference/stt_ggml_sidecar.py+++ b/studio/backend/core/inference/stt_ggml_sidecar.py@@ -356,6 +356,7 @@ def _whisper_server_child_env(binary: str) -> dict[str, str]:repointed at a managed scratch dir (a downloaded binary must not see the realhome's token caches), co-located libs on the loader path, WSL system HIP firston WSL2 ROCm."""+ binary = str(Path(binary).resolve())env = scrub_env(os.environ)isolate_home(env, str(_managed_whisper_cpp_dir() / ".child_home"))bin_dir = str(Path(binary).parent)@@ -368,12 +369,28 @@ def _whisper_server_child_env(binary: str) -> dict[str, str]:for pattern in ("libggml-cuda.so*", "ggml-cuda*.dll")for path in bundle_dir.glob(pattern))+ vendored_cuda_dirs: list[str] = []if has_cuda_module:try:from utils.prebuilt.runtime_libs import python_runtime_dirscuda_runtime_dirs = python_runtime_dirs()except Exception:cuda_runtime_dirs = []+ try:+ from utils.llama_cpp_freshness import read_install_marker as read_llama_install_marker+ from utils.prebuilt.runtime_libs import vendored_cuda_runtime_dirs++ marker = _whisper_install_marker(binary)+ # Slim bundles hardlink CUDA from their paired llama.cpp install; its marker holds the runtime.+ linked_from = marker.get("linked_from") if isinstance(marker, dict) else None+ paired_marker = (+ read_llama_install_marker(linked_from)+ if isinstance(linked_from, str) and linked_from+ else None+ )+ vendored_cuda_dirs = vendored_cuda_runtime_dirs(paired_marker or marker)+ except Exception:+ vendored_cuda_dirs = []if sys.platform == "win32":var, lead = "PATH", [bin_dir, *cuda_runtime_dirs]elif sys.platform == "darwin":@@ -385,7 +402,7 @@ def _whisper_server_child_env(binary: str) -> dict[str, str]:lead = [*wsl_rocm, bin_dir, *cuda_runtime_dirs]env.setdefault("HSA_ENABLE_DXG_DETECTION", "1")existing = [p for p in env.get(var, "").split(os.pathsep) if p]- env[var] = os.pathsep.join(_dedupe_existing_dirs([*lead, *existing]))+ env[var] = os.pathsep.join(_dedupe_existing_dirs([*lead, *existing, *vendored_cuda_dirs]))return envdiff --git a/studio/backend/tests/test_llama_cpp_freshness.py b/studio/backend/tests/test_llama_cpp_freshness.pyindex e5fc64fde4d..776a2786ede 100644--- a/studio/backend/tests/test_llama_cpp_freshness.py+++ b/studio/backend/tests/test_llama_cpp_freshness.py@@ -162,6 +162,14 @@ def test_read_install_marker_handles_invalid_json(tmp_path):assert fr.read_install_marker(str(bin_path)) is None+def test_read_install_marker_handles_non_utf8(tmp_path):+ install_dir = tmp_path / "llama.cpp"+ install_dir.mkdir(parents = True)+ (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_bytes(b'{"runtime_line": "\xff\xfecuda13"}')+ bin_path = _fake_binary(install_dir, layout = "root")+ assert fr.read_install_marker(str(bin_path)) is None++@pytest.mark.parametrize("payload",["[]", '["cpu"]', '"cuda"', "123", "true", "null"],diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.pyindex 594053329c1..01b69cc2bd2 100644--- a/studio/backend/tests/test_llama_cpp_mtp_detection.py+++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py@@ -11,6 +11,7 @@ from __future__ import annotationsimport astimport inspect+import jsonimport osimport structimport sys@@ -843,6 +844,134 @@ def test_probe_server_capabilities_uses_binary_library_env(tmp_path, monkeypatchassert captured["env"]["LD_LIBRARY_PATH"] == "/already-there"+def _make_vendored_cuda_runtime(tmp_path: Path, runtime_line: str = "cuda13") -> tuple[Path, Path]:+ """A prebuilt install tree plus the private CUDA runtime its marker selects."""+ binary_dir = tmp_path / "llama.cpp" / "build" / "bin"+ binary_dir.mkdir(parents = True)+ marker = binary_dir.parent.parent / "UNSLOTH_PREBUILT_INFO.json"+ marker.write_text(json.dumps({"runtime_line": runtime_line}))++ major = runtime_line.removeprefix("cuda")+ runtime_dir = tmp_path / "ollama" / f"cuda_v{major}"+ runtime_dir.mkdir(parents = True)+ cudart_payload = runtime_dir / f"libcudart.so.{major}.0"+ cublas_payload = runtime_dir / f"libcublas.so.{major}.0"+ cudart_payload.write_bytes(b"")+ cublas_payload.write_bytes(b"")+ (runtime_dir / f"libcudart.so.{major}").symlink_to(cudart_payload.name)+ (runtime_dir / f"libcublas.so.{major}").symlink_to(cublas_payload.name)+ return binary_dir, runtime_dir++[email protected](not sys.platform.startswith("linux"), reason = "LD_LIBRARY_PATH is Linux-only")+def test_llama_server_env_appends_vendored_cuda_runtime(tmp_path, monkeypatch):+ """The private runtime lands on LD_LIBRARY_PATH, but behind every other source."""+ import utils.prebuilt.runtime_libs as runtime_libs+ from utils.llama_cpp_freshness import reset_caches++ binary_dir, runtime_dir = _make_vendored_cuda_runtime(tmp_path)+ binary = binary_dir / "llama-server"+ binary.write_bytes(b"")+ python_cuda_dir = tmp_path / "site-packages" / "nvidia" / "cublas" / "lib"+ python_cuda_dir.mkdir(parents = True)++ monkeypatch.setattr(+ "core.inference.llama_cpp.child_env_without_native_path_secret",+ lambda: {"LD_LIBRARY_PATH": "/already-there"},+ )+ monkeypatch.setattr("core.inference.llama_cpp._wsl_system_rocm_lib_dirs", lambda: [])+ monkeypatch.setattr(+ "core.inference.llama_cpp._native_linux_system_rocm_lib_dirs", lambda _binary_dir: []+ )+ monkeypatch.setattr(+ "glob.glob",+ lambda pattern: (+ [str(python_cuda_dir)] if pattern.replace("\\", "/").endswith("nvidia/cu*/lib") else []+ ),+ )++ def _loader_dirs(roots: tuple[tuple[Path, str], ...]) -> list[str]:+ monkeypatch.setattr(runtime_libs, "_VENDORED_CUDA_ROOTS", roots)+ monkeypatch.setattr(runtime_libs, "_loader_already_provides_runtime", lambda _major: False)+ reset_caches()+ env = LlamaCppBackend._llama_server_env_for_binary(str(binary))+ reset_caches()+ return env["LD_LIBRARY_PATH"].split(os.pathsep)++ absent = _loader_dirs(((tmp_path / "nonexistent", "cuda_v{major}"),))+ present = _loader_dirs(((tmp_path / "ollama", "cuda_v{major}"),))++ vendored = str(runtime_dir.resolve())+ assert vendored not in absent+ assert present == [*absent, vendored]+ assert present.index(str(binary_dir.resolve())) < present.index(vendored)+ assert present.index(str(python_cuda_dir)) < present.index(vendored)+ assert present.index("/already-there") < present.index(vendored)+++@_NEEDS_BASH[email protected](not sys.platform.startswith("linux"), reason = "LD_LIBRARY_PATH is Linux-only")+def test_llama_server_env_keeps_vendored_runtime_off_a_resolvable_runtime(tmp_path, monkeypatch):+ """A runtime the loader already finds is never displaced by the private one."""+ import utils.prebuilt.runtime_libs as runtime_libs+ from utils.llama_cpp_freshness import reset_caches++ binary_dir, runtime_dir = _make_vendored_cuda_runtime(tmp_path)+ binary = binary_dir / "llama-server"+ binary.write_bytes(b"")+ monkeypatch.setattr(+ "core.inference.llama_cpp.child_env_without_native_path_secret",+ lambda: {"LD_LIBRARY_PATH": "/already-there"},+ )+ monkeypatch.setattr("core.inference.llama_cpp._wsl_system_rocm_lib_dirs", lambda: [])+ monkeypatch.setattr(+ "core.inference.llama_cpp._native_linux_system_rocm_lib_dirs", lambda _binary_dir: []+ )+ monkeypatch.setattr("glob.glob", lambda _pattern: [])+ monkeypatch.setattr(+ runtime_libs, "_VENDORED_CUDA_ROOTS", ((tmp_path / "ollama", "cuda_v{major}"),)+ )+ monkeypatch.setattr(runtime_libs, "_loader_already_provides_runtime", lambda _major: True)++ reset_caches()+ env = LlamaCppBackend._llama_server_env_for_binary(str(binary))+ reset_caches()++ assert str(runtime_dir.resolve()) not in env["LD_LIBRARY_PATH"].split(os.pathsep)+++@_NEEDS_BASH[email protected](not sys.platform.startswith("linux"), reason = "LD_LIBRARY_PATH is Linux-only")+def test_llama_server_env_resolves_marker_through_binary_symlink(tmp_path, monkeypatch):+ import utils.prebuilt.runtime_libs as runtime_libs+ from utils.llama_cpp_freshness import reset_caches++ install_bin, runtime_dir = _make_vendored_cuda_runtime(tmp_path)+ target = install_bin / "llama-server"+ target.write_bytes(b"")+ external = tmp_path / "external"+ external.mkdir()+ symlink = external / "llama-server"+ symlink.symlink_to(target)++ monkeypatch.setattr("core.inference.llama_cpp.child_env_without_native_path_secret", dict)+ monkeypatch.setattr("core.inference.llama_cpp._wsl_system_rocm_lib_dirs", lambda: [])+ monkeypatch.setattr(+ "core.inference.llama_cpp._native_linux_system_rocm_lib_dirs", lambda _binary_dir: []+ )+ monkeypatch.setattr("glob.glob", lambda _pattern: [])+ monkeypatch.setattr(+ runtime_libs, "_VENDORED_CUDA_ROOTS", ((tmp_path / "ollama", "cuda_v{major}"),)+ )+ monkeypatch.setattr(runtime_libs, "_loader_already_provides_runtime", lambda _major: False)++ reset_caches()+ env = LlamaCppBackend._llama_server_env_for_binary(str(symlink))+ reset_caches()++ assert str(runtime_dir.resolve()) in env["LD_LIBRARY_PATH"].split(os.pathsep)++@_NEEDS_BASHdef test_probe_server_capabilities_does_not_disable_devices_off_macos(tmp_path, monkeypatch):fake = _make_fake_llama_server(diff --git a/studio/backend/tests/test_stt_ggml_sidecar.py b/studio/backend/tests/test_stt_ggml_sidecar.pyindex 8547c0df4e0..9595914ed87 100644--- a/studio/backend/tests/test_stt_ggml_sidecar.py+++ b/studio/backend/tests/test_stt_ggml_sidecar.py@@ -539,6 +539,156 @@ def test_child_env_omits_cuda_runtime_dirs_for_cpu_bundle(monkeypatch, tmp_path)assert called["n"] == 0+def _make_cuda_bundle(+ tmp_path,+ *,+ runtime_line = "cuda13",+ linked_from = None,+):+ """A CUDA whisper bundle whose install marker names a CUDA runtime line."""+ bindir = tmp_path / "bin"+ bindir.mkdir()+ (bindir / _SERVER_NAME).write_text("#!/bin/sh\n")+ (bindir / "libggml-cuda.so").write_text("")+ if runtime_line is not None or linked_from is not None:+ marker = {}+ if runtime_line is not None:+ marker["runtime_line"] = runtime_line+ if linked_from is not None:+ marker["linked_from"] = str(linked_from)+ (tmp_path / "UNSLOTH_WHISPER_PREBUILT_INFO.json").write_text(json.dumps(marker))+ return bindir++[email protected](not sys.platform.startswith("linux"), reason = "Linux loader path only")+def test_child_env_appends_vendored_cuda_runtime(monkeypatch, tmp_path):+ import utils.prebuilt.runtime_libs as rl+ from utils.whisper_cpp_freshness import reset_caches++ bindir = _make_cuda_bundle(tmp_path)+ vendored = tmp_path / "ollama" / "cuda_v13"+ vendored.mkdir(parents = True)+ (vendored / "libcudart.so.13").write_text("")+ (vendored / "libcublas.so.13").write_text("")+ wheel_dir = tmp_path / "nvidia" / "cuda_runtime" / "lib"+ wheel_dir.mkdir(parents = True)++ reset_caches()+ monkeypatch.setattr(rl, "python_runtime_dirs", lambda: [str(wheel_dir)])+ monkeypatch.setattr(rl, "_VENDORED_CUDA_ROOTS", ((tmp_path / "ollama", "cuda_v{major}"),))+ monkeypatch.setattr(rl, "_loader_already_provides_runtime", lambda _major: False)+ env = ggml_module._whisper_server_child_env(str(bindir / _SERVER_NAME))+ reset_caches()++ parts = env[_loader_path_var()].split(os.pathsep)+ assert parts[-1] == str(vendored.resolve()) # behind every other source+ assert parts.index(str(bindir.resolve())) < parts.index(str(vendored.resolve()))+ assert parts.index(str(wheel_dir.resolve())) < parts.index(str(vendored.resolve()))++[email protected](not sys.platform.startswith("linux"), reason = "Linux loader path only")+def test_child_env_keeps_vendored_runtime_off_a_resolvable_runtime(monkeypatch, tmp_path):+ import utils.prebuilt.runtime_libs as rl+ from utils.whisper_cpp_freshness import reset_caches++ bindir = _make_cuda_bundle(tmp_path)+ vendored = tmp_path / "ollama" / "cuda_v13"+ vendored.mkdir(parents = True)+ (vendored / "libcudart.so.13").write_text("")+ (vendored / "libcublas.so.13").write_text("")++ reset_caches()+ monkeypatch.setattr(rl, "python_runtime_dirs", lambda: [])+ monkeypatch.setattr(rl, "_VENDORED_CUDA_ROOTS", ((tmp_path / "ollama", "cuda_v{major}"),))+ monkeypatch.setattr(rl, "_loader_already_provides_runtime", lambda _major: True)+ env = ggml_module._whisper_server_child_env(str(bindir / _SERVER_NAME))+ reset_caches()++ assert str(vendored.resolve()) not in env[_loader_path_var()].split(os.pathsep)++[email protected](not sys.platform.startswith("linux"), reason = "Linux loader path only")+def test_slim_child_env_reads_vendored_cuda_runtime_from_paired_llama(monkeypatch, tmp_path):+ import utils.prebuilt.runtime_libs as rl+ from utils.llama_cpp_freshness import reset_caches as reset_llama_caches+ from utils.whisper_cpp_freshness import reset_caches as reset_whisper_caches++ llama_bin = tmp_path / "llama.cpp" / "build" / "bin"+ llama_bin.mkdir(parents = True)+ (tmp_path / "llama.cpp" / "UNSLOTH_PREBUILT_INFO.json").write_text(+ json.dumps({"runtime_line": "cuda13"})+ )+ bindir = _make_cuda_bundle(tmp_path, runtime_line = None, linked_from = llama_bin)+ vendored = tmp_path / "ollama" / "cuda_v13"+ vendored.mkdir(parents = True)+ (vendored / "libcudart.so.13").write_text("")+ (vendored / "libcublas.so.13").write_text("")++ reset_llama_caches()+ reset_whisper_caches()+ monkeypatch.setattr(rl, "python_runtime_dirs", lambda: [])+ monkeypatch.setattr(rl, "_VENDORED_CUDA_ROOTS", ((tmp_path / "ollama", "cuda_v{major}"),))+ monkeypatch.setattr(rl, "_loader_already_provides_runtime", lambda _major: False)+ env = ggml_module._whisper_server_child_env(str(bindir / _SERVER_NAME))+ reset_llama_caches()+ reset_whisper_caches()++ assert str(vendored.resolve()) in env[_loader_path_var()].split(os.pathsep)++[email protected](not sys.platform.startswith("linux"), reason = "Linux loader path only")+def test_child_env_omits_vendored_cuda_runtime_for_cpu_bundle(monkeypatch, tmp_path):+ import utils.prebuilt.runtime_libs as rl+ from utils.whisper_cpp_freshness import reset_caches++ bindir = tmp_path / "bin"+ bindir.mkdir()+ (bindir / _SERVER_NAME).write_text("#!/bin/sh\n")+ (tmp_path / "UNSLOTH_WHISPER_PREBUILT_INFO.json").write_text(+ json.dumps({"runtime_line": "cuda13"})+ )+ vendored = tmp_path / "ollama" / "cuda_v13"+ vendored.mkdir(parents = True)+ (vendored / "libcudart.so.13").write_text("")+ (vendored / "libcublas.so.13").write_text("")++ reset_caches()+ monkeypatch.setattr(rl, "_VENDORED_CUDA_ROOTS", ((tmp_path / "ollama", "cuda_v{major}"),))+ env = ggml_module._whisper_server_child_env(str(bindir / _SERVER_NAME))+ reset_caches()++ assert str(vendored.resolve()) not in env[_loader_path_var()].split(os.pathsep)++[email protected](not sys.platform.startswith("linux"), reason = "Linux loader path only")+def test_child_env_resolves_external_symlink_before_cuda_bundle_probe(monkeypatch, tmp_path):+ import utils.prebuilt.runtime_libs as rl+ from utils.whisper_cpp_freshness import reset_caches++ managed_root = tmp_path / "managed"+ managed_root.mkdir()+ bindir = _make_cuda_bundle(managed_root)+ external_dir = tmp_path / "external"+ external_dir.mkdir()+ alias = external_dir / _SERVER_NAME+ alias.symlink_to(bindir / _SERVER_NAME)+ vendored = tmp_path / "ollama" / "cuda_v13"+ vendored.mkdir(parents = True)+ (vendored / "libcudart.so.13").write_text("")+ (vendored / "libcublas.so.13").write_text("")++ monkeypatch.setattr(ggml_module, "_managed_whisper_cpp_dir", lambda: managed_root)+ monkeypatch.setattr(rl, "_VENDORED_CUDA_ROOTS", ((tmp_path / "ollama", "cuda_v{major}"),))+ monkeypatch.setattr(rl, "_loader_already_provides_runtime", lambda _major: False)+ reset_caches()+ env = ggml_module._whisper_server_child_env(str(alias))+ reset_caches()++ parts = env["LD_LIBRARY_PATH"].split(os.pathsep)+ assert str(bindir.resolve()) in parts+ assert parts[-1] == str(vendored.resolve())++def test_engine_unavailable_is_stt_unavailable():# Routes map SttUnavailableError to HTTP 501; the engine error must share it.assert issubclass(SttEngineUnavailableError, SttUnavailableError)diff --git a/studio/backend/tests/test_vendored_cuda_runtime.py b/studio/backend/tests/test_vendored_cuda_runtime.pynew file mode 100644index 00000000000..40b532e5b4b--- /dev/null+++ b/studio/backend/tests/test_vendored_cuda_runtime.py@@ -0,0 +1,457 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for rescuing a privately vendored CUDA runtime onto LD_LIBRARY_PATH."""++from __future__ import annotations++import os+import subprocess+import sys+import sysconfig++import pytest++from utils.prebuilt.runtime_libs import vendored_cuda_runtime_dirs++import utils.prebuilt.runtime_libs as runtime_libs++_REAL_LOADER_DEFAULT_LIB_DIRS = runtime_libs._LOADER_DEFAULT_LIB_DIRS+_REAL_LD_CACHE_ENTRIES = runtime_libs._ld_cache_entries+_REAL_LOADER_RESOLVES = runtime_libs._loader_resolves_sonames++pytestmark = pytest.mark.skipif(+ not sys.platform.startswith("linux"), reason = "the vendored roots are Linux paths"+)+++def _make_runtime(+ root,+ name: str,+ *,+ cudart: bool = True,+ cublas: bool = True,+ major = "13",+ major_soname_links: bool = True,+):+ directory = root / name+ directory.mkdir(parents = True)+ if cudart:+ payload = directory / f"libcudart.so.{major}.0.48"+ payload.write_bytes(b"")+ if major_soname_links:+ (directory / f"libcudart.so.{major}").symlink_to(payload.name)+ if cublas:+ payload = directory / f"libcublas.so.{major}.0.1"+ payload.write_bytes(b"")+ if major_soname_links:+ (directory / f"libcublas.so.{major}").symlink_to(payload.name)+ return directory+++def _roots(tmp_path):+ return ((tmp_path, "cuda_v{major}"),)++[email protected](autouse = True)+def _host_loader_unknown(monkeypatch):+ monkeypatch.setattr(runtime_libs, "_ld_cache_entries", lambda: None)+ monkeypatch.setattr(runtime_libs, "_LOADER_DEFAULT_LIB_DIRS", ())+ monkeypatch.setattr(runtime_libs, "_loader_resolves_sonames", lambda _sonames: False)+++def test_matches_the_marker_runtime_line(tmp_path):+ runtime_dir = _make_runtime(tmp_path, "cuda_v13")++ result = vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path))++ assert result == [str(runtime_dir.resolve())]+++def test_ignores_a_different_cuda_major(tmp_path):+ _make_runtime(tmp_path, "cuda_v13")+ _make_runtime(tmp_path, "cuda_v12", major = "12")++ result = vendored_cuda_runtime_dirs({"runtime_line": "cuda12"}, roots = _roots(tmp_path))++ assert result == [str((tmp_path / "cuda_v12").resolve())]+++def test_accepts_a_minor_qualified_directory(tmp_path):+ runtime_dir = _make_runtime(tmp_path, "cuda_v13.0")++ result = vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path))++ assert result == [str(runtime_dir.resolve())]+++def test_rejects_a_longer_major_that_shares_the_prefix(tmp_path):+ # cuda_v130 answers a cuda_v13* glob but is not CUDA 13.+ _make_runtime(tmp_path, "cuda_v130", major = "130")++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == []++[email protected]("missing", ["cudart", "cublas"])+def test_requires_a_complete_runtime(tmp_path, missing):+ _make_runtime(tmp_path, "cuda_v13", cudart = missing != "cudart", cublas = missing != "cublas")++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == []+++def test_rejects_payloads_without_major_soname_links(tmp_path):+ runtime_dir = _make_runtime(tmp_path, "cuda_v13", major_soname_links = False)++ assert (runtime_dir / "libcudart.so.13.0.48").is_file()+ assert (runtime_dir / "libcublas.so.13.0.1").is_file()+ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == []+++ hasattr(os, "geteuid") and os.geteuid() == 0,+ reason = "root can read mode-000 files despite missing permission bits",+)+def test_rejects_unreadable_major_soname_links(tmp_path):+ runtime_dir = _make_runtime(tmp_path, "cuda_v13")+ for soname in ("libcudart.so.13", "libcublas.so.13"):+ (runtime_dir / soname).resolve().chmod(0)+ assert not os.access(runtime_dir / soname, os.R_OK)++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == []+++def test_ignores_a_file_named_like_a_runtime_dir(tmp_path):+ (tmp_path / "cuda_v13").write_bytes(b"")++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == []+++ "marker",+ [+ None, # source build / unreadable or corrupt marker+ [], # valid JSON, wrong shape+ "cuda13",+ {}, # marker without a runtime line+ {"runtime_line": None},+ {"runtime_line": 13}, # not a string+ {"runtime_line": "cpu"},+ {"runtime_line": "vulkan"},+ ],+)+def test_yields_nothing_without_a_cuda_runtime_line(tmp_path, marker):+ _make_runtime(tmp_path, "cuda_v13")++ assert vendored_cuda_runtime_dirs(marker, roots = _roots(tmp_path)) == []+++def test_missing_root_is_not_an_error(tmp_path):+ roots = ((tmp_path / "nonexistent", "cuda_v{major}"),)++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = roots) == []+++def test_withholds_the_runtime_when_the_loader_already_finds_it(tmp_path, monkeypatch):+ import utils.prebuilt.runtime_libs as runtime_libs++ runtime_dir = _make_runtime(tmp_path, "cuda_v13")+ system_dir = tmp_path / "system-runtime"+ system_dir.mkdir()+ cudart = system_dir / "libcudart.so.13"+ cublas = system_dir / "libcublas.so.13"+ cudart.write_bytes(b"")+ cublas.write_bytes(b"")+ monkeypatch.setattr(+ runtime_libs,+ "_ld_cache_entries",+ lambda: (+ ("libcudart.so.13", "libc6,x86-64", str(cudart)),+ ("libcublas.so.13", "libc6,x86-64", str(cublas)),+ ),+ )+ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == []+ monkeypatch.setattr(+ runtime_libs,+ "_ld_cache_entries",+ lambda: (("libcudart.so.13", "libc6,x86-64", str(cudart)),),+ )+ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == [+ str(runtime_dir.resolve())+ ]+++def test_ignores_cache_entries_with_missing_targets(tmp_path, monkeypatch):+ import utils.prebuilt.runtime_libs as runtime_libs++ runtime_dir = _make_runtime(tmp_path, "cuda_v13")+ system_dir = tmp_path / "system-runtime"+ system_dir.mkdir()+ cudart = system_dir / "libcudart.so.13"+ cudart.write_bytes(b"")+ missing_cublas = system_dir / "libcublas.so.13"+ monkeypatch.setattr(+ runtime_libs,+ "_ld_cache_entries",+ lambda: (+ ("libcudart.so.13", "libc6,x86-64", str(cudart)),+ ("libcublas.so.13", "libc6,x86-64", str(missing_cublas)),+ ),+ )++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == [+ str(runtime_dir.resolve())+ ]+++def test_checks_loader_defaults_even_when_cache_is_readable(tmp_path, monkeypatch):+ import utils.prebuilt.runtime_libs as runtime_libs++ _make_runtime(tmp_path, "cuda_v13")+ default_dir = tmp_path / "loader-default"+ default_dir.mkdir()+ (default_dir / "libcudart.so.13").write_bytes(b"")+ (default_dir / "libcublas.so.13").write_bytes(b"")+ monkeypatch.setattr(runtime_libs, "_ld_cache_entries", lambda: ())+ monkeypatch.setattr(runtime_libs, "_LOADER_DEFAULT_LIB_DIRS", (str(default_dir),))++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == []+++ hasattr(os, "geteuid") and os.geteuid() == 0,+ reason = "root can read mode-000 files despite missing permission bits",+)+def test_unreadable_loader_default_files_do_not_block_the_vendored_rescue(tmp_path, monkeypatch):+ runtime_dir = _make_runtime(tmp_path, "cuda_v13")+ default_dir = tmp_path / "loader-default"+ default_dir.mkdir()+ system_libraries = [+ default_dir / "libcudart.so.13",+ default_dir / "libcublas.so.13",+ ]+ for library in system_libraries:+ library.write_bytes(b"inaccessible regular file")+ library.chmod(0)++ assert all(+ library.is_file() and not os.access(library, os.R_OK) for library in system_libraries+ )+ with pytest.raises(PermissionError):+ system_libraries[0].open("rb")++ monkeypatch.setattr(runtime_libs, "_ld_cache_entries", lambda: ())+ monkeypatch.setattr(runtime_libs, "_LOADER_DEFAULT_LIB_DIRS", (str(default_dir),))++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == [+ str(runtime_dir.resolve())+ ]+++def test_ignores_foreign_abi_cache_entries(tmp_path, monkeypatch):+ import platform++ if platform.machine().lower() not in {"x86_64", "amd64"}:+ pytest.skip("synthetic foreign-ABI cache fixture is x86-64-specific")++ runtime_dir = _make_runtime(tmp_path, "cuda_v13")+ monkeypatch.setattr(runtime_libs.shutil, "which", lambda _candidate: "/sbin/ldconfig")+ original_exists = runtime_libs.os.path.exists+ monkeypatch.setattr(+ runtime_libs.os.path,+ "exists",+ lambda path: True if path == "/sbin/ldconfig" else original_exists(path),+ )+ result = type(+ "Result",+ (),+ {+ "returncode": 0,+ "stdout": (+ "libcudart.so.13 (libc6,i686) => /usr/lib/i386-linux-gnu/libcudart.so.13\\n"+ "libcublas.so.13 (libc6,i686) => /usr/lib/i386-linux-gnu/libcublas.so.13\\n"+ ),+ },+ )()+ monkeypatch.setattr(runtime_libs.subprocess, "run", lambda *_args, **_kwargs: result)+ monkeypatch.setattr(runtime_libs, "_LOADER_DEFAULT_LIB_DIRS", ())++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == [+ str(runtime_dir.resolve())+ ]++[email protected](not sys.platform.startswith("linux"), reason = "Linux loader cache only")+def test_recognizes_aarch64_ldconfig_abi_spelling_case_insensitively(tmp_path, monkeypatch):+ import utils.prebuilt.runtime_libs as runtime_libs++ monkeypatch.setattr(runtime_libs, "_ld_cache_entries", _REAL_LD_CACHE_ENTRIES)++ runtime_dir = _make_runtime(tmp_path, "cuda_v13")+ system_dir = tmp_path / "system-cache"+ system_dir.mkdir()+ cudart = system_dir / "libcudart.so.13"+ cublas = system_dir / "libcublas.so.13"+ cudart.write_bytes(b"")+ cublas.write_bytes(b"")+ monkeypatch.setattr(runtime_libs.platform, "machine", lambda: "aarch64")+ monkeypatch.setattr(runtime_libs.shutil, "which", lambda _candidate: None)+ original_exists = runtime_libs.os.path.exists+ monkeypatch.setattr(+ runtime_libs.os.path,+ "exists",+ lambda path: True if path == "/sbin/ldconfig" else original_exists(path),+ )+ result = type(+ "Result",+ (),+ {+ "returncode": 0,+ "stdout": (+ f"libcudart.so.13 (libc6,AArch64) => {cudart}\n"+ f"libcublas.so.13 (libc6,AArch64) => {cublas}\n"+ ),+ },+ )()+ monkeypatch.setattr(runtime_libs.subprocess, "run", lambda *_args, **_kwargs: result)+ monkeypatch.setattr(runtime_libs, "_LOADER_DEFAULT_LIB_DIRS", ())++ assert runtime_libs._ld_cache_entries() == (+ ("libcudart.so.13", "libc6,aarch64", str(cudart)),+ ("libcublas.so.13", "libc6,aarch64", str(cublas)),+ )+ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == []+++def test_an_unreadable_cache_still_rescues_from_the_default_dirs(tmp_path, monkeypatch):+ import utils.prebuilt.runtime_libs as runtime_libs++ runtime_dir = _make_runtime(tmp_path, "cuda_v13")+ monkeypatch.setattr(runtime_libs, "_ld_cache_entries", lambda: None)+ default = tmp_path / "default"+ default.mkdir()+ monkeypatch.setattr(runtime_libs, "_LOADER_DEFAULT_LIB_DIRS", (str(default),))++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == [+ str(runtime_dir.resolve())+ ]+ (default / "libcudart.so.13").write_bytes(b"")+ (default / "libcublas.so.13").write_bytes(b"")++ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == []+++def test_probes_loader_when_ldconfig_cache_query_fails(monkeypatch):+ monkeypatch.setattr(runtime_libs, "_ld_cache_entries", lambda: None)+ monkeypatch.setattr(runtime_libs, "_loader_resolves_sonames", lambda _sonames: True)++ assert runtime_libs._loader_already_provides_runtime("13")+++def test_uses_the_dynamic_loader_cache_without_ldconfig(monkeypatch):+ sonames = ("libcudart.so.13", "libcublas.so.13")+ native_abis = runtime_libs._NATIVE_LOADER_ABIS.get(+ runtime_libs.platform.machine().lower(), frozenset()+ )+ cached = _REAL_LD_CACHE_ENTRIES()+ cache_paths = {+ soname: next(+ (+ path+ for cached_soname, abi, path in cached or ()+ if cached_soname == soname+ and abi in native_abis+ and os.path.isfile(path)+ and os.access(path, os.R_OK)+ and not any(+ os.path.dirname(path) == default_dir+ for default_dir in _REAL_LOADER_DEFAULT_LIB_DIRS+ )+ ),+ None,+ )+ for soname in sonames+ }+ if not all(cache_paths.values()):+ pytest.skip("host has no accessible CUDA 13 pair in a non-default loader-cache directory")++ env = os.environ.copy()+ env.pop("LD_LIBRARY_PATH", None)+ loader_probe = subprocess.run(+ [+ sys.executable,+ "-I",+ "-S",+ "-c",+ "import ctypes, sys; [ctypes.CDLL(name) for name in sys.argv[1:]]",+ *sonames,+ ],+ capture_output = True,+ env = env,+ timeout = 10,+ )+ if loader_probe.returncode:+ pytest.skip("dynamic loader cannot resolve the cached CUDA 13 pair on this host")++ monkeypatch.setattr(runtime_libs, "_loader_resolves_sonames", _REAL_LOADER_RESOLVES)+ monkeypatch.setattr(runtime_libs, "_ld_cache_entries", _REAL_LD_CACHE_ENTRIES)+ monkeypatch.setattr(runtime_libs.shutil, "which", lambda _candidate: None)+ original_exists = runtime_libs.os.path.exists+ monkeypatch.setattr(+ runtime_libs.os.path,+ "exists",+ lambda path: False+ if path in {"/sbin/ldconfig", "/usr/sbin/ldconfig"}+ else original_exists(path),+ )+ monkeypatch.setattr(runtime_libs, "_LOADER_DEFAULT_LIB_DIRS", ())++ assert runtime_libs._ld_cache_entries() is None+ assert runtime_libs._loader_already_provides_runtime("13")+++def test_the_multiarch_dirs_are_discovered_not_assumed(tmp_path, monkeypatch):+ import utils.prebuilt.runtime_libs as runtime_libs++ for name in ("lib/x86_64-linux-gnu", "usr/lib/x86_64-linux-gnu", "usr/lib/riscv64-linux-gnu"):+ (tmp_path / name).mkdir(parents = True)+ (tmp_path / "usr/lib/not-a-layout").mkdir(parents = True)+ monkeypatch.setattr(+ runtime_libs,+ "_MULTIARCH_LIB_GLOBS",+ (str(tmp_path / "lib/*-linux-gnu*"), str(tmp_path / "usr/lib/*-linux-gnu*")),+ )++ assert runtime_libs._multiarch_lib_dirs() == [+ str(tmp_path / "lib/x86_64-linux-gnu"),+ str(tmp_path / "usr/lib/riscv64-linux-gnu"),+ str(tmp_path / "usr/lib/x86_64-linux-gnu"),+ ]+++def test_the_default_dirs_cover_the_hosts_multiarch_paths():+ triple = sysconfig.get_config_var("MULTIARCH")+ if not triple:+ pytest.skip("this interpreter records no MULTIARCH triple")+ installed = [+ f"{prefix}/{triple}"+ for prefix in ("/lib", "/usr/lib")+ if os.path.isdir(f"{prefix}/{triple}")+ ]+ if not installed:+ pytest.skip(f"no {triple} directory is installed on this host")+ for directory in installed:+ assert directory in _REAL_LOADER_DEFAULT_LIB_DIRS++[email protected](not sys.platform.startswith("linux"), reason = "Linux loader path only")+def test_usr_local_is_not_assumed_without_a_readable_cache(tmp_path, monkeypatch):+ runtime_dir = _make_runtime(tmp_path, "cuda_v13")+ monkeypatch.setattr(runtime_libs, "_ld_cache_entries", lambda: None)++ assert not any(+ directory.rstrip("/") in {"/usr/local/lib", "/usr/local/lib64"}+ for directory in _REAL_LOADER_DEFAULT_LIB_DIRS+ )+ assert vendored_cuda_runtime_dirs({"runtime_line": "cuda13"}, roots = _roots(tmp_path)) == [+ str(runtime_dir.resolve())+ ]diff --git a/studio/backend/utils/prebuilt/freshness_flow.py b/studio/backend/utils/prebuilt/freshness_flow.pyindex d0458f0291d..5cbe6d12cd9 100644--- a/studio/backend/utils/prebuilt/freshness_flow.py+++ b/studio/backend/utils/prebuilt/freshness_flow.py@@ -45,7 +45,7 @@ def read_install_marker(if candidate.is_file():try:marker = json.loads(candidate.read_text(encoding = "utf-8"))- except (OSError, json.JSONDecodeError) as exc:+ except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:logger.debug(log_message, path = str(candidate), error = str(exc))marker = Noneelse:diff --git a/studio/backend/utils/prebuilt/runtime_libs.py b/studio/backend/utils/prebuilt/runtime_libs.pyindex f231b8a8a99..141f0d8745b 100644--- a/studio/backend/utils/prebuilt/runtime_libs.py+++ b/studio/backend/utils/prebuilt/runtime_libs.py@@ -1,19 +1,26 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""CUDA runtime dirs shipped inside Python wheels, for the STT sidecar's child env.+"""CUDA runtime dirs the loader does not find on its own (Python wheels, privately vendored dirs).-Kept in sync with install_llama_prebuilt.py's python_runtime_dirs; the backend-cannot import the studio/ installer scripts, so this small copy stays importable-with only the backend root on sys.path.+Kept in sync with install_llama_prebuilt.py's python_runtime_dirs and+prebuilt_core.py's linux_runtime_dirs_for_required_libraries; the backend cannot+import the studio/ installer scripts, so this small copy stays importable with+only the backend root on sys.path."""from __future__ import annotations+import glob+import os+import platform+import re+import shutilimport site+import subprocessimport sysfrom pathlib import Path-from typing import Iterable+from typing import Iterable, Optionaldef dedupe_existing_dirs(paths: Iterable[str | Path]) -> list[str]:@@ -36,6 +43,156 @@ def dedupe_existing_dirs(paths: Iterable[str | Path]) -> list[str]:return unique+# Debian multiarch dirs are built into ld.so's default search path, no ldconfig needed.+_MULTIARCH_LIB_GLOBS: tuple[str, ...] = ("/lib/*-linux-gnu*", "/usr/lib/*-linux-gnu*")+++def _multiarch_lib_dirs() -> list[str]:+ found: list[str] = []+ for pattern in _MULTIARCH_LIB_GLOBS:+ try:+ found.extend(sorted(glob.glob(pattern)))+ except OSError:+ continue+ return found+++_LOADER_DEFAULT_LIB_DIRS: tuple[str, ...] = (+ "/lib",+ "/lib64",+ "/usr/lib",+ "/usr/lib64",+ *_multiarch_lib_dirs(),+)+++def _ld_cache_entries() -> tuple[tuple[str, str, str], ...] | None:+ """(soname, ABI, path) entries, or None when ldconfig cannot be read."""+ for candidate in ("ldconfig", "/sbin/ldconfig", "/usr/sbin/ldconfig"):+ exe = shutil.which(candidate) if "/" not in candidate else candidate+ if not exe or not os.path.exists(exe):+ continue+ try:+ result = subprocess.run(+ [exe, "-p"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 10,+ )+ except (OSError, subprocess.SubprocessError):+ continue+ if result.returncode != 0:+ continue+ entries: list[tuple[str, str, str]] = []+ for line in (result.stdout or "").splitlines():+ match = re.match(r"\s*(\S+)\s+\(([^)]*)\)\s+=>\s+(\S+)", line)+ if match:+ soname, abi_text, path = match.groups()+ abi = abi_text.strip().lower()+ entries.append((soname, abi, path))+ return tuple(entries)+ return None+++def _loader_resolves_sonames(sonames: tuple[str, ...]) -> bool:+ """Probe in a child process so CUDA never loads into Studio."""+ script = "import ctypes, sys; [ctypes.CDLL(name) for name in sys.argv[1:]]"+ try:+ result = subprocess.run(+ [sys.executable, "-I", "-S", "-c", script, *sonames],+ capture_output = True,+ timeout = 10,+ )+ except (OSError, subprocess.SubprocessError):+ return False+ return result.returncode == 0+++_NATIVE_LOADER_ABIS: dict[str, frozenset[str]] = {+ "x86_64": frozenset({"libc6,x86-64", "libc6,x86-64-v2", "libc6,x86-64-v3", "libc6,x86-64-v4"}),+ "amd64": frozenset({"libc6,x86-64", "libc6,x86-64-v2", "libc6,x86-64-v3", "libc6,x86-64-v4"}),+ "i386": frozenset({"libc6,i686", "libc6,i586", "libc6,i486", "libc6,i386"}),+ "i686": frozenset({"libc6,i686", "libc6,i586", "libc6,i486", "libc6,i386"}),+ "aarch64": frozenset({"libc6,aarch64"}),+ "arm64": frozenset({"libc6,aarch64"}),+ "armv7l": frozenset({"libc6,armhf"}),+ "ppc64le": frozenset({"libc6,ppc64le"}),+ "s390x": frozenset({"libc6,s390x"}),+ "riscv64": frozenset({"libc6,riscv64"}),+}+++def _loader_already_provides_runtime(major: str) -> bool:+ """LD_LIBRARY_PATH outranks ld.so.cache and default dirs: never shadow a native pair they provide."""+ sonames = (f"libcudart.so.{major}", f"libcublas.so.{major}")+ cached = _ld_cache_entries()+ if cached is None and _loader_resolves_sonames(sonames):+ return True+ native_abis = _NATIVE_LOADER_ABIS.get(platform.machine().lower(), frozenset())+ for soname in sonames:+ cache_has_compatible = cached is not None and any(+ cached_soname == soname+ and abi in native_abis+ and os.path.isfile(path)+ and os.access(path, os.R_OK)+ for cached_soname, abi, path in cached+ )+ if cache_has_compatible:+ continue+ if not any(+ os.path.isfile(os.path.join(directory, soname))+ and os.access(os.path.join(directory, soname), os.R_OK)+ for directory in _LOADER_DEFAULT_LIB_DIRS+ ):+ return False+ return True+++_VENDORED_CUDA_ROOTS: tuple[tuple[Path, str], ...] = (+ (Path("/usr/local/lib/ollama"), "cuda_v{major}"),+)+++def vendored_cuda_runtime_dirs(+ marker: object, *, roots: Optional[Iterable[tuple[Path, str]]] = None+) -> list[str]:+ """Vendored dirs holding libcudart + libcublas for the marker's ``runtime_line`` major.++ Empty when the loader already finds that pair. Callers append the result LAST+ so it never displaces the runtime the build picked.+ """+ if not sys.platform.startswith("linux"):+ return []+ runtime_line = marker.get("runtime_line") if isinstance(marker, dict) else None+ match = re.fullmatch(r"cuda(\d+)", runtime_line if isinstance(runtime_line, str) else "")+ if match is None:+ return []+ major = match.group(1)+ if _loader_already_provides_runtime(major):+ return []+ found: list[Path] = []+ for root, prefix_template in _VENDORED_CUDA_ROOTS if roots is None else roots:+ prefix = prefix_template.format(major = major)+ # cuda_v130 must not answer a cuda_v13 glob; cuda_v13.0 still should.+ exact = re.compile(rf"{re.escape(prefix)}(?:[._-].*)?")+ try:+ found.extend(+ directory+ for directory in sorted(Path(root).glob(f"{prefix}*"))+ if exact.fullmatch(directory.name)+ and all(+ (directory / f"lib{name}.so.{major}").is_file()+ and os.access(directory / f"lib{name}.so.{major}", os.R_OK)+ for name in ("cudart", "cublas")+ )+ )+ except OSError:+ continue+ return dedupe_existing_dirs(found)++def python_runtime_dirs() -> list[str]:"""CUDA runtime dirs shipped inside Python wheels (torch + nvidia-* wheels)."""candidates: list[Path] = []