Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

llama.cpp prebuilt: accept upstream win-rocm bundles, run native arm64 under Rosetta, source-build on Jetson (#14373)

Daniel Han3594ed54db37
diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py
index 2a7e2ce790c..3ec46974fe9 100644
--- a/studio/install_llama_prebuilt.py
+++ b/studio/install_llama_prebuilt.py
@@ -821,6 +821,36 @@ def _upstream_arm64_cuda_allowed() -> bool:
return raw not in {"0", "false", "no", "off"}
+def _upstream_windows_hip_asset_names(release_tag: str, assets: "Iterable[str]") -> list[str]:
+ """Legacy win-hip-radeon first, then win-rocm-<ver> (current upstream name), newest first."""
+ names = list(assets)
+ legacy = f"llama-{release_tag}-bin-win-hip-radeon-x64.zip"
+ pattern = re.compile(
+ rf"llama-{re.escape(release_tag)}-bin-win-rocm-([0-9]+(?:\.[0-9]+)*)-x64\.zip"
+ )
+ versioned = sorted(
+ (
+ (tuple(int(p) for p in m.group(1).split(".")), name)
+ for name in names
+ if (m := pattern.fullmatch(name)) is not None
+ ),
+ reverse = True,
+ )
+ return ([legacy] if legacy in names else []) + [name for _version, name in versioned]
+
+
+def _windows_rocm_vulkan_fallback_allowed(host: HostInfo) -> bool:
+ """Vulkan behind HIP only with no HIP mask and no NVIDIA trace: it honours neither mask,
+ and following ROCm torch clears has_physical_nvidia while the card stays enumerable."""
+ return not (
+ host.has_physical_nvidia
+ or host.nvidia_smi
+ or host.physical_compute_caps
+ or host.compute_caps
+ or _hip_visible_device_mask_set()
+ )
+
+
def windows_cuda_arch_for_host(host: HostInfo) -> str:
"""CUDA prebuilt arch token for a Windows host: arm64 on Windows on ARM, else x64.
@@ -1328,19 +1358,32 @@ def direct_upstream_release_plan(
)
attempts[:] = _drop_blackwell_incapable_windows_cuda(selection_host, attempts)
elif host.has_rocm:
- hip_asset = f"llama-{release_tag}-bin-win-hip-radeon-x64.zip"
- hip_url = assets.get(hip_asset)
- if hip_url:
+ for hip_asset in _upstream_windows_hip_asset_names(release_tag, assets):
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = hip_asset,
- url = hip_url,
+ url = assets[hip_asset],
source_label = "upstream",
install_kind = "windows-hip",
)
)
+ # Vulkan before CPU if HIP is missing or fails validation; never past a HIP mask, which Vulkan ignores.
+ if _windows_rocm_vulkan_fallback_allowed(host):
+ vulkan_asset = f"llama-{release_tag}-bin-win-vulkan-x64.zip"
+ vulkan_url = assets.get(vulkan_asset)
+ if vulkan_url:
+ attempts.append(
+ AssetChoice(
+ repo = repo,
+ tag = release_tag,
+ name = vulkan_asset,
+ url = vulkan_url,
+ source_label = "upstream",
+ install_kind = "windows-vulkan",
+ )
+ )
# Intel or AMD-without-usable-ROCm: use the Vulkan prebuilt, matching the published
# branch and the Linux x86_64 one here. Reached when the caller pins a non-default
# --published-repo, so leaving it Intel-only sent a Windows AMD host to the CPU
@@ -2036,6 +2079,32 @@ def _mask_widened_sms(host: HostInfo) -> list[str] | None:
return normalize_compute_caps(physical + visible)
+_TEGRA_FALLBACK_LOGGED: list[bool] = []
+
+
[email protected]_cache(maxsize = 1)
+def _is_tegra() -> bool:
+ """Mirrors studio/backend/utils/tegra.py; never raises."""
+ try:
+ if os.path.exists("/etc/nv_tegra_release"):
+ return True
+ with open("/proc/device-tree/compatible", "rb") as fh:
+ return b"nvidia,tegra" in fh.read()
+ except OSError:
+ return False
+
+
+def _tegra_blocks_arm64_cuda(host: HostInfo) -> bool:
+ """Jetson below CUDA 13 cannot run the SBSA arm64 CUDA bundle; Thor (CUDA 13) and unknown drivers can."""
+ return (
+ host.is_linux
+ and host.is_arm64
+ and host.driver_cuda_version is not None
+ and host.driver_cuda_version[0] < 13
+ and _is_tegra()
+ )
+
+
def linux_cuda_choice_from_release(
host: HostInfo,
release: PublishedReleaseBundle,
@@ -2043,6 +2112,15 @@ def linux_cuda_choice_from_release(
selection_preamble: Iterable[str] = (),
host_sms: list[str] | None = None,
) -> LinuxCudaSelection | None:
+ if _tegra_blocks_arm64_cuda(host):
+ if not _TEGRA_FALLBACK_LOGGED:
+ _TEGRA_FALLBACK_LOGGED.append(True)
+ log(
+ "NVIDIA Jetson (Tegra) with CUDA "
+ f"{host.driver_cuda_version[0]}.{host.driver_cuda_version[1]}: the arm64 CUDA "
+ "bundle targets server ARM64, building from source instead"
+ )
+ return None
selection_preamble = list(selection_preamble)
# An explicit host_sms is the widened try, whose miss falls back rather than warns.
warn_uncovered = host_sms is None
@@ -3007,9 +3085,28 @@ def _windows_display_adapter_in_registry(needles: tuple[tuple[str, str], ...]) -
return False
+def _host_machine(system: str) -> str:
+ """platform.machine(), but arm64 under Rosetta: macos-x64 is built without Metal."""
+ machine = platform.machine().lower()
+ if system != "Darwin" or machine != "x86_64":
+ return machine
+ try:
+ result = subprocess.run(
+ ["sysctl", "-n", "hw.optional.arm64"],
+ capture_output = True,
+ text = True,
+ timeout = 5,
+ )
+ if result.returncode == 0 and result.stdout.strip() == "1":
+ return "arm64"
+ except Exception:
+ pass
+ return machine
+
+
def detect_host(*, probe_rocm_with_nvidia: bool = False) -> HostInfo:
system = platform.system()
- machine = platform.machine().lower()
+ machine = _host_machine(system)
is_windows = system == "Windows"
is_linux = system == "Linux"
is_macos = system == "Darwin"
@@ -4241,10 +4338,9 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice
if not _host_is_blackwell(host):
raise PrebuiltFallback("no compatible Windows CUDA asset was found")
- # AMD ROCm on Windows: try upstream HIP prebuilt, then fall back to CPU
+ # AMD ROCm on Windows: try upstream HIP prebuilt, then Vulkan, then CPU
if host.has_rocm:
- hip_name = f"llama-{llama_tag}-bin-win-hip-radeon-x64.zip"
- if hip_name in upstream_assets:
+ for hip_name in _upstream_windows_hip_asset_names(llama_tag, upstream_assets):
log(f"AMD ROCm detected on Windows -- trying upstream HIP prebuilt {hip_name}")
return AssetChoice(
repo = UPSTREAM_REPO,
@@ -4254,16 +4350,23 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice
source_label = "upstream",
install_kind = "windows-hip",
)
- log("AMD ROCm detected on Windows but no HIP prebuilt found -- falling back to CPU")
+ log("AMD ROCm detected on Windows but no HIP prebuilt found -- trying Vulkan, then CPU")
# Mirrors the Linux branch: without the AMD half a Strix Halo box with no ROCm
- # tooling took windows-cpu, where the same silicon takes Vulkan on Linux.
+ # tooling took windows-cpu, where the same silicon takes Vulkan on Linux. ROCm
+ # hosts reach here only without a HIP bundle.
if (
- (host.has_intel_gpu or host.has_amd_gpu_without_rocm)
- and not host.has_physical_nvidia
- and not host.has_rocm
- ):
- vendor = "Intel GPU" if host.has_intel_gpu else "AMD GPU without usable ROCm"
+ _windows_rocm_vulkan_fallback_allowed(host)
+ if host.has_rocm
+ else host.has_intel_gpu or host.has_amd_gpu_without_rocm
+ ) and not host.has_physical_nvidia:
+ vendor = (
+ "AMD ROCm"
+ if host.has_rocm
+ else "Intel GPU"
+ if host.has_intel_gpu
+ else "AMD GPU without usable ROCm"
+ )
vulkan_name = f"llama-{llama_tag}-bin-win-vulkan-x64.zip"
if vulkan_name in upstream_assets:
log(f"{vendor} detected on Windows -- using upstream Vulkan prebuilt {vulkan_name}")
@@ -7506,6 +7609,9 @@ def _fork_manifest_release_plans(
and driver_below_cuda_prebuilt_floor(host)
):
raise PrebuiltFallback(cuda_driver_floor_message(host)) from exc
+ # Same for Jetson: no release's arm64 CUDA bundle serves it.
+ if host.has_physical_nvidia and _tegra_blocks_arm64_cuda(host):
+ raise
log(
"published release skipped for install planning: "
f"{bundle.repo}@{bundle.release_tag} upstream_tag={resolved_tag} ({exc})"
@@ -9501,7 +9607,7 @@ def platform_only_host() -> HostInfo:
platform.mac_ver() rather than probed.
"""
system = platform.system()
- machine = platform.machine().lower()
+ machine = _host_machine(system)
is_macos = system == "Darwin"
return HostInfo(
system = system,
@@ -9713,6 +9819,8 @@ def _kept_install_covers_host(marker: "dict[str, Any] | None", host: HostInfo) -
marker = marker or {}
backend = marker_backend(marker)
if backend == "cuda":
+ if _tegra_blocks_arm64_cuda(host):
+ return False
line = marker.get("runtime_line")
if isinstance(line, str) and line.startswith("cuda") and host.driver_cuda_version:
lines = (
@@ -9808,7 +9916,9 @@ def reusable_existing_install(install_dir: Path, host: HostInfo) -> bool:
return True
# #12842: below the floor a CUDA prebuilt still answers --version but loads no kernel.
marker = load_prebuilt_metadata(install_dir) or {}
- if marker_backend(marker) == "cuda" and driver_below_cuda_prebuilt_floor(host):
+ if marker_backend(marker) == "cuda" and (
+ driver_below_cuda_prebuilt_floor(host) or _tegra_blocks_arm64_cuda(host)
+ ):
return False
return _existing_install_runs(install_dir, host)
@@ -10459,9 +10569,9 @@ def _windows_hip_gfx_targets(published_repo: str | None) -> frozenset[str]:
The combined floor above includes the FORK's windows-rocm bundles, and only the fork is
planned from a manifest: resolve_simple_install_release_plans() sends every other
--published-repo (ggml-org, but equally any mirror of upstream-standard assets) to
- direct_upstream_release_plan(), whose AMD branch offers win-hip-radeon then CPU and never
- Vulkan. Answering "supported" there for a fork-only arch would silently land it on
- HIP/CPU instead of the Vulkan bundle that would actually run, so gate on the fork rather
+ direct_upstream_release_plan(), whose AMD branch offers the upstream HIP bundle first and
+ Vulkan only behind it. Answering "supported" there for a fork-only arch would silently land
+ it on HIP instead of the Vulkan bundle that would actually run, so gate on the fork rather
than exempting one repo, mirroring that dispatch exactly, spelling included: an empty
value defaults to the fork, and a differently cased repo really does take the upstream
path and must be answered with upstream coverage."""
@@ -11544,13 +11654,20 @@ def select_backend_install(
cpu_mechanism = cpu_mechanism,
host = host,
)
+ # Up front: the fork's plans are lazy, so their Jetson rejection would surface outside the try below (#4856).
+ if route.backend == "cuda" and _tegra_blocks_arm64_cuda(route.host):
+ raise BackendUnavailable(
+ "no CUDA prebuilt runs on a Jetson before CUDA 13; source-building instead"
+ )
try:
requested_tag, release_plans = resolve_simple_install_release_plans(
llama_tag, route.host, route.published_repo, route.published_release_tag
)
except PrebuiltFallback as exc:
# #12842: a stored cuda choice must fall back to detection, not fail the update.
- if route.backend == "cuda" and driver_below_cuda_prebuilt_floor(route.host):
+ if route.backend == "cuda" and (
+ driver_below_cuda_prebuilt_floor(route.host) or _tegra_blocks_arm64_cuda(route.host)
+ ):
raise BackendUnavailable(str(exc)) from exc
raise
if route.rocm_fallback_host is not None:
diff --git a/tests/studio/install/test_llama_host_platform_edges.py b/tests/studio/install/test_llama_host_platform_edges.py
new file mode 100644
index 00000000000..ec0ba5951de
--- /dev/null
+++ b/tests/studio/install/test_llama_host_platform_edges.py
@@ -0,0 +1,466 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""llama.cpp prebuilt choice on three host edges, all I/O faked:
+Windows AMD against upstream's win-rocm-<ver> rename, Apple Silicon under Rosetta, and Jetson."""
+
+import builtins
+import hashlib
+import importlib.util
+import io
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+
+PACKAGE_ROOT = Path(__file__).resolve().parents[3]
+MODULE_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py"
+SPEC = importlib.util.spec_from_file_location("studio_install_llama_prebuilt_edges", MODULE_PATH)
+assert SPEC is not None and SPEC.loader is not None
+ILP = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = ILP
+SPEC.loader.exec_module(ILP)
+
+TAG = "b9900"
+
+
+def _host(**overrides):
+ system = overrides.pop("system", "Windows")
+ machine = overrides.pop("machine", "AMD64")
+ defaults = dict(
+ system = system,
+ machine = machine,
+ is_linux = system == "Linux",
+ is_windows = system == "Windows",
+ is_macos = system == "Darwin",
+ is_x86_64 = machine.lower() in {"x86_64", "amd64"},
+ is_arm64 = machine.lower() in {"arm64", "aarch64"},
+ nvidia_smi = None,
+ driver_cuda_version = None,
+ compute_caps = [],
+ visible_cuda_devices = None,
+ has_physical_nvidia = False,
+ has_usable_nvidia = False,
+ )
+ defaults.update(overrides)
+ return ILP.HostInfo(**defaults)
+
+
+def _release(names):
+ return {
+ "tag_name": TAG,
+ "assets": [
+ {
+ "name": n,
+ "browser_download_url": f"https://example.com/{n}",
+ "digest": "sha256:" + hashlib.sha256(n.encode()).hexdigest(),
+ }
+ for n in names
+ ],
+ }
+
+
+ROCM = f"llama-{TAG}-bin-win-rocm-10.0-x64.zip"
+ROCM_OLD = f"llama-{TAG}-bin-win-rocm-7.2-x64.zip"
+HIP_RADEON = f"llama-{TAG}-bin-win-hip-radeon-x64.zip"
+VULKAN = f"llama-{TAG}-bin-win-vulkan-x64.zip"
+CPU = f"llama-{TAG}-bin-win-cpu-x64.zip"
+
+
[email protected](autouse = True)
+def _no_hip_mask(monkeypatch):
+ for var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(var, raising = False)
+
+
+def _direct_names(names, host):
+ plan = ILP.direct_upstream_release_plan(_release(names), host, ILP.UPSTREAM_REPO, "latest")
+ return [a.name for a in plan.attempts]
+
+
+def _single_name(names, host, monkeypatch):
+ assets = {n: f"https://example.com/{n}" for n in names}
+ monkeypatch.setattr(ILP, "github_release_assets", lambda repo, tag: assets)
+ return ILP.resolve_upstream_asset_choice(host, TAG).name
+
+
+ "names, direct, single",
+ [
+ # Current upstream: only the versioned ROCm bundle.
+ ([ROCM, VULKAN, CPU], [ROCM, VULKAN, CPU], ROCM),
+ ([ROCM, ROCM_OLD, VULKAN, CPU], [ROCM, ROCM_OLD, VULKAN, CPU], ROCM),
+ # No HIP bundle at all: Vulkan before CPU.
+ ([VULKAN, CPU], [VULKAN, CPU], VULKAN),
+ # Older tags with the legacy name keep it first.
+ ([HIP_RADEON, VULKAN, CPU], [HIP_RADEON, VULKAN, CPU], HIP_RADEON),
+ ([HIP_RADEON, CPU], [HIP_RADEON, CPU], HIP_RADEON),
+ ([CPU], [CPU], CPU),
+ ],
+)
+def test_windows_rocm_host_accepts_win_rocm_assets(names, direct, single, monkeypatch):
+ host = _host(has_rocm = True)
+ assert _direct_names(names, host) == direct
+ assert _single_name(names, host, monkeypatch) == single
+
+
+def test_windows_rocm_bundle_keeps_the_hip_install_kind(monkeypatch):
+ host = _host(has_rocm = True)
+ plan = ILP.direct_upstream_release_plan(
+ _release([ROCM, VULKAN, CPU]), host, ILP.UPSTREAM_REPO, "latest"
+ )
+ assert [a.install_kind for a in plan.attempts] == [
+ "windows-hip",
+ "windows-vulkan",
+ "windows-cpu",
+ ]
+
+
+def test_windows_rocm_with_physical_nvidia_never_takes_vulkan(monkeypatch):
+ # Vulkan ignores CUDA_VISIBLE_DEVICES; a hidden NVIDIA card must not be enumerated.
+ host = _host(has_rocm = True, has_physical_nvidia = True, visible_cuda_devices = "-1")
+ assert _direct_names([VULKAN, CPU], host) == [CPU]
+ assert _single_name([VULKAN, CPU], host, monkeypatch) == CPU
+
+
+ "trace",
+ [
+ dict(nvidia_smi = "C:/Windows/System32/nvidia-smi.exe"),
+ dict(compute_caps = ["89"]),
+ dict(physical_compute_caps = ["89"]),
+ ],
+)
+def test_windows_rocm_following_torch_keeps_nvidia_out_of_vulkan(trace, monkeypatch):
+ # Following ROCm torch clears has_physical_nvidia, but Vulkan would still enumerate the card.
+ host = _host(has_rocm = True, **trace)
+ assert _direct_names([ROCM, VULKAN, CPU], host) == [ROCM, CPU]
+ assert _single_name([VULKAN, CPU], host, monkeypatch) == CPU
+
+
+ "var", ["HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"]
+)
+def test_windows_rocm_hip_mask_never_falls_back_to_vulkan(var, monkeypatch):
+ # Vulkan ignores HIP masks, so it could enumerate an AMD card the user hid.
+ monkeypatch.setenv(var, "1")
+ host = _host(has_rocm = True)
+ assert _direct_names([ROCM, VULKAN, CPU], host) == [ROCM, CPU]
+ assert _single_name([VULKAN, CPU], host, monkeypatch) == CPU
+
+
+ "var", ["HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES", None]
+)
[email protected]("other", ["has_intel_gpu", "has_amd_gpu_without_rocm"])
+def test_windows_rocm_beside_another_gpu_keeps_the_vulkan_guard(var, other, monkeypatch):
+ # An Intel iGPU or a non-ROCm AMD card must not bypass the ROCm mask / NVIDIA guard.
+ if var:
+ monkeypatch.setenv(var, "1")
+ host = _host(has_rocm = True, **{other: True})
+ else:
+ host = _host(has_rocm = True, nvidia_smi = "C:/nvidia-smi.exe", **{other: True})
+ assert _direct_names([VULKAN, CPU], host) == [CPU]
+ assert _single_name([VULKAN, CPU], host, monkeypatch) == CPU
+
+
+def test_windows_rocm_beside_an_intel_igpu_still_takes_vulkan_unmasked(monkeypatch):
+ host = _host(has_rocm = True, has_intel_gpu = True)
+ assert _single_name([VULKAN, CPU], host, monkeypatch) == VULKAN
+
+
+ "flags, expected",
+ [
+ (dict(has_amd_gpu_without_rocm = True), [VULKAN, CPU]),
+ (dict(has_intel_gpu = True), [VULKAN, CPU]),
+ (dict(), [CPU]),
+ ],
+)
+def test_windows_non_rocm_hosts_unchanged(flags, expected, monkeypatch):
+ host = _host(**flags)
+ names = [ROCM, HIP_RADEON, VULKAN, CPU]
+ assert _direct_names(names, host) == expected
+ assert _single_name(names, host, monkeypatch) == expected[0]
+
+
+MAC_ARM = f"llama-{TAG}-bin-macos-arm64.tar.gz"
+MAC_X64 = f"llama-{TAG}-bin-macos-x64.tar.gz"
+
+
+def _fake_mac(
+ monkeypatch,
+ machine,
+ sysctl_out,
+ sysctl_rc = 0,
+ sysctl_raises = None,
+):
+ calls = []
+ real_run = subprocess.run
+
+ def fake_run(cmd, *args, **kwargs):
+ if list(cmd)[:1] == ["sysctl"]:
+ calls.append(list(cmd))
+ if sysctl_raises is not None:
+ raise sysctl_raises
+ return subprocess.CompletedProcess(cmd, sysctl_rc, sysctl_out, "")
+ return real_run(cmd, *args, **kwargs)
+
+ monkeypatch.setattr(ILP.platform, "system", lambda: "Darwin")
+ monkeypatch.setattr(ILP.platform, "machine", lambda: machine)
+ monkeypatch.setattr(ILP.platform, "mac_ver", lambda: ("14.5", ("", "", ""), machine))
+ monkeypatch.setattr(ILP.shutil, "which", lambda *_a, **_k: None)
+ monkeypatch.setattr(ILP.subprocess, "run", fake_run)
+ return calls
+
+
+ "machine, sysctl_out, rc, raises, expected_machine, expected_asset",
+ [
+ ("x86_64", "1\n", 0, None, "arm64", MAC_ARM), # Rosetta
+ ("x86_64", "0\n", 0, None, "x86_64", MAC_X64), # Intel Mac
+ ("x86_64", "", 1, None, "x86_64", MAC_X64), # key absent (older Intel macOS)
+ ("x86_64", "", 0, FileNotFoundError("sysctl"), "x86_64", MAC_X64),
+ ("x86_64", "", 0, subprocess.TimeoutExpired("sysctl", 5), "x86_64", MAC_X64),
+ ("arm64", "1\n", 0, None, "arm64", MAC_ARM), # native arm64
+ ],
+)
+def test_macos_host_machine(
+ machine, sysctl_out, rc, raises, expected_machine, expected_asset, monkeypatch
+):
+ _fake_mac(monkeypatch, machine, sysctl_out, rc, raises)
+ host = ILP.detect_host()
+ assert host.machine == expected_machine
+ assert host.is_arm64 == (expected_machine == "arm64")
+ assert host.is_x86_64 == (expected_machine == "x86_64")
+ plan = ILP.direct_upstream_release_plan(
+ _release([MAC_ARM, MAC_X64]), host, ILP.UPSTREAM_REPO, "latest"
+ )
+ assert [a.name for a in plan.attempts] == [expected_asset]
+ cheap = ILP.platform_only_host()
+ assert (cheap.machine, cheap.is_arm64, cheap.is_x86_64) == (
+ host.machine,
+ host.is_arm64,
+ host.is_x86_64,
+ )
+
+
+def test_sysctl_is_not_run_off_macos(monkeypatch):
+ calls = _fake_mac(monkeypatch, "x86_64", "1\n")
+ monkeypatch.setattr(ILP.platform, "system", lambda: "Linux")
+ assert ILP.platform_only_host().machine == "x86_64"
+ assert calls == []
+
+
+def _fake_tegra(
+ monkeypatch,
+ *,
+ release_file = False,
+ compatible = None,
+):
+ real_exists = os.path.exists
+ real_open = builtins.open
+
+ def fake_exists(path):
+ if path == "/etc/nv_tegra_release":
+ return release_file
+ return real_exists(path)
+
+ def fake_open(path, *args, **kwargs):
+ if path == "/proc/device-tree/compatible":
+ if compatible is None:
+ raise FileNotFoundError(path)
+ return io.BytesIO(compatible)
+ return real_open(path, *args, **kwargs)
+
+ monkeypatch.setattr(ILP.os.path, "exists", fake_exists)
+ monkeypatch.setattr(builtins, "open", fake_open)
+ tegra = getattr(ILP, "_is_tegra", None)
+ if tegra is not None:
+ tegra.cache_clear()
+ monkeypatch.setattr(ILP, "_TEGRA_FALLBACK_LOGGED", [])
+
+
[email protected](autouse = True)
+def _clear_tegra_cache():
+ yield
+ tegra = getattr(ILP, "_is_tegra", None)
+ if tegra is not None:
+ tegra.cache_clear()
+
+
+def _arm64_cuda_release():
+ def art(name, line, sms, lo, hi):
+ return ILP.PublishedLlamaArtifact(
+ asset_name = name,
+ install_kind = "linux-arm64-cuda",
+ runtime_line = line,
+ coverage_class = "portable",
+ supported_sms = sms,
+ min_sm = lo,
+ max_sm = hi,
+ bundle_profile = f"{line}-portable",
+ rank = 100,
+ )
+
+ artifacts = [
+ art("app-linux-arm64-cuda12-portable.tar.gz", "cuda12", ["87", "90"], 87, 90),
+ art(
+ "app-linux-arm64-cuda13-portable.tar.gz", "cuda13", ["87", "90", "110", "121"], 87, 121
+ ),
+ ]
+ return ILP.PublishedReleaseBundle(
+ repo = "unslothai/llama.cpp",
+ release_tag = "v1.0",
+ upstream_tag = TAG,
+ source_repo = None,
+ source_repo_url = None,
+ source_ref_kind = None,
+ requested_source_ref = None,
+ resolved_source_ref = None,
+ source_commit = None,
+ source_commit_short = None,
+ assets = {a.asset_name: f"https://example.com/{a.asset_name}" for a in artifacts},
+ manifest_asset_name = "llama-prebuilt-manifest.json",
+ artifacts = artifacts,
+ selection_log = [],
+ )
+
+
+def _linux_attempts(monkeypatch, host, lines):
+ monkeypatch.setattr(
+ ILP, "detected_linux_runtime_lines", lambda: (list(lines), {l: ["/stub"] for l in lines})
+ )
+ monkeypatch.setattr(
+ ILP,
+ "detect_torch_cuda_runtime_preference",
+ lambda host, **_: ILP.CudaRuntimePreference(runtime_line = None, selection_log = []),
+ )
+ return [a.name for a in ILP._linux_published_attempts(host, _arm64_cuda_release())]
+
+
+def _arm_host(driver, caps):
+ return _host(
+ system = "Linux",
+ machine = "aarch64",
+ nvidia_smi = "/usr/bin/nvidia-smi",
+ driver_cuda_version = driver,
+ compute_caps = caps,
+ physical_compute_caps = caps,
+ has_physical_nvidia = True,
+ has_usable_nvidia = True,
+ )
+
+
+ "tegra_fs",
+ [dict(release_file = True), dict(compatible = b"nvidia,p3768-0000\x00nvidia,tegra234\x00")],
+)
+def test_jetson_cuda12_source_builds(tegra_fs, monkeypatch):
+ _fake_tegra(monkeypatch, **tegra_fs)
+ host = _arm_host((12, 6), ["87"])
+ assert _linux_attempts(monkeypatch, host, ["cuda12"]) == []
+ with pytest.raises(ILP.PrebuiltFallback):
+ ILP.resolve_linux_cuda_choice(host, _arm64_cuda_release())
+
+
+ "tegra_fs, driver, caps, lines, expected",
+ [
+ # GH200 / DGX Spark: server ARM64, not Tegra.
+ (dict(), (12, 8), ["90"], ["cuda12"], ["app-linux-arm64-cuda12-portable.tar.gz"]),
+ (dict(), (13, 0), ["121"], ["cuda13"], ["app-linux-arm64-cuda13-portable.tar.gz"]),
+ (
+ dict(compatible = b"nvidia,grace\x00"),
+ (13, 0),
+ ["90"],
+ ["cuda13"],
+ ["app-linux-arm64-cuda13-portable.tar.gz"],
+ ),
+ # Thor, JetPack 7: Tegra but CUDA 13 (SBSA).
+ (
+ dict(release_file = True),
+ (13, 0),
+ ["110"],
+ ["cuda13"],
+ ["app-linux-arm64-cuda13-portable.tar.gz"],
+ ),
+ # Tegra with an unknown driver version fails open.
+ (dict(release_file = True), None, ["87"], ["cuda12"], None),
+ ],
+)
+def test_non_jetson_or_cuda13_arm64_unchanged(tegra_fs, driver, caps, lines, expected, monkeypatch):
+ _fake_tegra(monkeypatch)
+ names_base = _linux_attempts(monkeypatch, _arm_host(driver, caps), lines)
+ _fake_tegra(monkeypatch, **tegra_fs)
+ names = _linux_attempts(monkeypatch, _arm_host(driver, caps), lines)
+ assert names == names_base
+ if expected is not None:
+ assert names[:1] == expected
+
+
+def _jetson_cuda12(monkeypatch):
+ _fake_tegra(monkeypatch, release_file = True)
+ return _arm_host((12, 6), ["87"])
+
+
+CUDA12_MARKER = {"backend": "cuda", "runtime_line": "cuda12", "supported_sms": ["87", "90"]}
+
+
+def test_jetson_rejects_an_existing_cuda_prebuilt(tmp_path, monkeypatch):
+ # Installed before the Jetson check: an update must not keep or reuse it.
+ monkeypatch.setattr(ILP, "compatible_linux_runtime_lines", lambda host: ["cuda12"])
+ monkeypatch.setattr(ILP, "_existing_install_runs", lambda *_a: True)
+ (tmp_path / "UNSLOTH_PREBUILT_INFO.json").write_text(json.dumps(CUDA12_MARKER))
+ _fake_tegra(monkeypatch)
+ server = _arm_host((12, 6), ["87"])
+ assert ILP._kept_install_covers_host(CUDA12_MARKER, server)
+ assert ILP.reusable_existing_install(tmp_path, server)
+ host = _jetson_cuda12(monkeypatch)
+ assert not ILP._kept_install_covers_host(CUDA12_MARKER, host)
+ assert not ILP.reusable_existing_install(tmp_path, host)
+ assert ILP._kept_install_covers_host({"backend": "cpu"}, host)
+
+
+def test_jetson_stored_cuda_request_with_lazy_plans_falls_back(monkeypatch):
+ # The fork's plans are lazy: their Jetson rejection must not surface after the try that converts it.
+ def lazy_plans(*_a, **_k):
+ def plans():
+ raise ILP.PrebuiltFallback("no compatible Linux prebuilt asset was found")
+ yield # pragma: no cover
+
+ return "latest", plans()
+
+ monkeypatch.setattr(ILP, "resolve_simple_install_release_plans", lazy_plans)
+ monkeypatch.setattr(ILP, "_rocm_torch_preferred", lambda: False)
+ host = _jetson_cuda12(monkeypatch)
+ with pytest.raises(ILP.BackendUnavailable):
+ selection = ILP.select_backend_install(
+ backend = "cuda",
+ llama_tag = "latest",
+ published_repo = "unslothai/llama.cpp",
+ published_release_tag = "",
+ host = host,
+ )
+ list(getattr(selection, "release_plans", []) or [])
+
+
+def test_jetson_stored_cuda_request_falls_back_to_detection(monkeypatch):
+ def no_plans(*_a, **_k):
+ raise ILP.PrebuiltFallback("no compatible Linux prebuilt asset was found")
+
+ monkeypatch.setattr(ILP, "resolve_simple_install_release_plans", no_plans)
+ monkeypatch.setattr(ILP, "_rocm_torch_preferred", lambda: False)
+ host = _jetson_cuda12(monkeypatch)
+ with pytest.raises(ILP.BackendUnavailable):
+ ILP.select_backend_install(
+ backend = "cuda",
+ llama_tag = "latest",
+ published_repo = "unslothai/llama.cpp",
+ published_release_tag = "",
+ host = host,
+ )