Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: default to the Vulkan llama.cpp bundle when a CPU-bound host has a Vulkan GPU (#13599)

Daniel Han6a31011f2efe
diff --git a/install.ps1 b/install.ps1
index ba6101cc7ff..f79b94eebf3 100644
--- a/install.ps1
+++ b/install.ps1
@@ -9855,12 +9855,9 @@ main()
# NotImplementedError at import (unsloth/device_type.py). The Vulkan setter is
# single-quoted so PowerShell prints $env:... rather than expanding it; a pasted
# VAR=value resolves as a command name here and sets nothing.
- # Both claims below are conditional: an explicit index pin reaches the ROCm install
- # path further down even for an arch Unsloth has no wheels for, and studio/setup.ps1
- # THROWS on the Vulkan variable on Windows ARM64, where no bundle is published.
+ # An explicit index pin reaches the ROCm path below even for an arch without Unsloth wheels.
$unsupPinned = (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_TORCH_INDEX_URL)) -or `
(-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_TORCH_INDEX_FAMILY))
- $unsupArm64 = (Get-HostMachineArch) -eq "arm64"
step "gpu" "AMD GPU detected ($ROCmUnsupportedGfxArch) -- no ROCm PyTorch wheels Unsloth installs" "Yellow"
substep "Detected: $ROCmGpuLabel" "Yellow"
if ($unsupPinned) {
@@ -9871,14 +9868,10 @@ main()
substep "CPU-only: Unsloth training and GPU inference are unavailable. Installing the" "Yellow"
substep "HIP SDK or setting UNSLOTH_ROCM_GFX_ARCH will not change that for it." "Yellow"
}
- if ($unsupArm64) {
- substep "GGUF chat would need Vulkan on this GPU, and no Windows ARM64 Vulkan bundle is published: build llama.cpp from source, or run this on x64." "Yellow"
- } else {
- substep "GGUF chat can still use this GPU through Vulkan: set" "Yellow"
- substep '$env:UNSLOTH_LLAMA_CPP_BACKEND = "vulkan" and re-run this installer. It' "Yellow"
- substep "selects the llama.cpp bundle at install time, so setting it afterwards has" "Yellow"
- substep "no effect until you install or update again." "Yellow"
- }
+ substep "GGUF chat can still use this GPU through Vulkan: set" "Yellow"
+ substep '$env:UNSLOTH_LLAMA_CPP_BACKEND = "vulkan" and re-run this installer. It' "Yellow"
+ substep "selects the llama.cpp bundle at install time, so setting it afterwards has" "Yellow"
+ substep "no effect until you install or update again." "Yellow"
} elseif ($ROCmGpuLabel) {
step "gpu" "AMD GPU detected -- arch unknown" "Yellow"
substep "Detected: $ROCmGpuLabel" "Yellow"
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index fd900f791e0..5ae43195ada 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -86,6 +86,8 @@ os.environ.setdefault("UNSLOTH_DIFFUSION_ATTENTION_INSTALL", "0")
os.environ.setdefault("UNSLOTH_STUDIO_DISABLE_DEVICE_PROBE", "1")
# A test that hides nvidia-smi to fake a CPU host must not find the real GPUs through NVML.
os.environ.setdefault("UNSLOTH_NVIDIA_LIBRARY_PROBE", "0")
+# A fake CPU host must not take the Vulkan route through this machine's real GPUs.
+os.environ.setdefault("UNSLOTH_VULKAN_DEVICE_PROBE", "0")
# settled_snapshot_device_memory spaces its retried VRAM reads a real second apart so a
# transient tenant on a live card has time to clear. Under test the snapshots are stubs
# whose answers do not change with time, so the wait buys nothing and the max() over the
diff --git a/studio/backend/tests/test_install_resolve_prebuilt.py b/studio/backend/tests/test_install_resolve_prebuilt.py
index 1f9e69160fe..9ea81cd1cdc 100644
--- a/studio/backend/tests/test_install_resolve_prebuilt.py
+++ b/studio/backend/tests/test_install_resolve_prebuilt.py
@@ -21,6 +21,7 @@ import importlib
import json
import ntpath
import os
+import subprocess
import sys
from pathlib import Path
from types import ModuleType, SimpleNamespace
@@ -2658,20 +2659,440 @@ def _windows_arm64_host(**overrides):
({}, "vulkan"),
],
)
-def test_vulkan_opt_in_keeps_windows_arm64_host_for_strict_rejection(monkeypatch, env, flag):
- # Windows arm64 has no compatible Vulkan bundle. Keep the real host shape so
- # strict filtering rejects the CPU plan instead of routing through x64.
+def test_vulkan_opt_in_routes_windows_arm64_to_vulkan(monkeypatch, env, flag):
+ # Windows arm64 Vulkan bundles exist (#11674): an explicit request is honoured.
for name, value in env.items():
monkeypatch.setenv(name, value)
- host = _windows_arm64_host()
routed, repo, tag, persist = ilp._route_to_vulkan_prebuilt(
- host, FORK, "pin", force_cpu = False, llama_backend = flag
+ _windows_arm64_host(), FORK, "pin", force_cpu = False, llama_backend = flag
)
- assert routed is host
+ assert routed.vulkan_routed is True
+ assert routed.is_arm64 is True
+ assert routed.vulkan_device_required is False
assert (repo, tag) == (FORK, "pin")
+ assert persist == "vulkan"
+
+
+_ADRENO = ("Qualcomm(R) Adreno(TM) X1-85 GPU",)
+
+
+def _no_vulkan_probe():
+ raise AssertionError("this host must not probe for a Vulkan GPU")
+
+
[email protected]("has_intel_gpu", [False, True])
[email protected]("flag", [None, "auto"])
+def test_windows_arm64_auto_routes_to_vulkan_when_a_gpu_enumerates(
+ monkeypatch, flag, has_intel_gpu
+):
+ monkeypatch.delenv("UNSLOTH_LLAMA_CPP_BACKEND", raising = False)
+ monkeypatch.delenv("UNSLOTH_FORCE_VULKAN", raising = False)
+ monkeypatch.setattr(ilp, "_vulkan_gpu_devices", lambda: _ADRENO)
+ routed, repo, tag, persist = ilp._route_to_vulkan_prebuilt(
+ _windows_arm64_host(has_intel_gpu = has_intel_gpu),
+ FORK,
+ "pin",
+ force_cpu = False,
+ llama_backend = flag,
+ )
+ assert routed.vulkan_routed is True
+ # Detected, not chosen: updates re-detect and the Vulkan CPU crash recovery stays armed.
+ assert (repo, tag, persist) == (FORK, "pin", None)
+
+
[email protected]("has_intel_gpu", [False, True])
+def test_windows_arm64_without_a_vulkan_gpu_keeps_its_cpu_bundle(monkeypatch, has_intel_gpu):
+ # A stale Intel adapter entry in the registry names no GPU here: only the probe counts.
+ monkeypatch.delenv("UNSLOTH_LLAMA_CPP_BACKEND", raising = False)
+ monkeypatch.delenv("UNSLOTH_FORCE_VULKAN", raising = False)
+ monkeypatch.setattr(ilp, "_vulkan_gpu_devices", lambda: ())
+ host = _windows_arm64_host(has_intel_gpu = has_intel_gpu)
+ routed, repo, tag, persist = ilp._route_to_vulkan_prebuilt(host, FORK, "pin", force_cpu = False)
+ assert routed is host
+ assert (repo, tag, persist) == (FORK, "pin", None)
+
+
+def _cpu_bound_host(system, arch, **overrides):
+ is_arm64 = arch == "arm64"
+ defaults = dict(
+ system = system,
+ machine = "ARM64" if is_arm64 else "x86_64",
+ is_windows = system == "Windows",
+ is_linux = system == "Linux",
+ is_macos = system == "Darwin",
+ is_x86_64 = not is_arm64,
+ is_arm64 = is_arm64,
+ nvidia_smi = None,
+ driver_cuda_version = None,
+ compute_caps = [],
+ visible_cuda_devices = None,
+ has_physical_nvidia = False,
+ has_usable_nvidia = False,
+ has_rocm = False,
+ has_intel_gpu = False,
+ )
+ defaults.update(overrides)
+ return ilp.HostInfo(**defaults)
+
+
+ "system, arch, overrides",
+ [
+ ("Windows", "x64", {}),
+ ("Linux", "x64", {}),
+ ("Linux", "arm64", {}),
+ # Linux arm64 never took the AMD widening x86_64 has; a GPU that enumerates now counts.
+ ("Linux", "arm64", {"has_amd_gpu_without_rocm": True}),
+ ],
+)
+def test_cpu_bound_host_with_a_vulkan_gpu_routes_to_vulkan(monkeypatch, system, arch, overrides):
+ monkeypatch.delenv("UNSLOTH_LLAMA_CPP_BACKEND", raising = False)
+ monkeypatch.delenv("UNSLOTH_FORCE_VULKAN", raising = False)
+ monkeypatch.delenv("GGML_VK_VISIBLE_DEVICES", raising = False)
+ monkeypatch.setattr(ilp, "_vulkan_gpu_devices", lambda: ("Mali-G720",))
+ host = _cpu_bound_host(system, arch, **overrides)
+ routed, _repo, _tag, persist = ilp._route_to_vulkan_prebuilt(host, FORK, "pin", force_cpu = False)
+ assert routed.vulkan_routed is True and routed.has_intel_gpu is True
+ assert routed.vulkan_device_required is True
assert persist is None
+ "host_args, kwargs, env",
+ [
+ (("Darwin", "arm64"), {}, {}),
+ (("Linux", "x64", {"has_physical_nvidia": True}), {}, {}),
+ (("Linux", "x64", {"has_rocm": True}), {}, {}),
+ (("Windows", "arm64", {"has_physical_nvidia": True, "has_usable_nvidia": True}), {}, {}),
+ # x86_64 AMD without ROCm already takes Vulkan in the selectors.
+ (("Windows", "x64", {"has_amd_gpu_without_rocm": True}), {}, {}),
+ (("Linux", "x64"), {"force_cpu": True}, {}),
+ (("Linux", "x64"), {}, {"UNSLOTH_LLAMA_CPP_BACKEND": "cpu"}),
+ (("Windows", "arm64"), {}, {"UNSLOTH_LLAMA_CPP_BACKEND": "cuda"}),
+ (("Linux", "arm64"), {}, {"GGML_VK_VISIBLE_DEVICES": ""}),
+ ],
+ ids = [
+ "macos",
+ "nvidia",
+ "rocm",
+ "arm64-nvidia",
+ "x64-amd-no-rocm",
+ "force-cpu",
+ "backend-cpu",
+ "backend-cuda",
+ "vk-mask",
+ ],
+)
+def test_hosts_not_headed_for_cpu_never_probe_vulkan(monkeypatch, host_args, kwargs, env):
+ monkeypatch.delenv("UNSLOTH_LLAMA_CPP_BACKEND", raising = False)
+ monkeypatch.delenv("UNSLOTH_FORCE_VULKAN", raising = False)
+ monkeypatch.delenv("GGML_VK_VISIBLE_DEVICES", raising = False)
+ for name, value in env.items():
+ monkeypatch.setenv(name, value)
+ monkeypatch.setattr(ilp, "_vulkan_gpu_devices", _no_vulkan_probe)
+ host = _cpu_bound_host(*host_args[:2], **(host_args[2] if len(host_args) > 2 else {}))
+ routed, _repo, _tag, persist = ilp._route_to_vulkan_prebuilt(
+ host, FORK, "pin", force_cpu = kwargs.get("force_cpu", False)
+ )
+ assert routed.vulkan_routed is False
+ assert persist is None
+
+
+def _probe_result(
+ monkeypatch,
+ devices,
+ returncode = 0,
+):
+ out = devices if isinstance(devices, str) else json.dumps(devices)
+ monkeypatch.setattr(
+ ilp,
+ "run_capture",
+ lambda *a, **k: subprocess.CompletedProcess(a[0], returncode, out + "\n", ""),
+ )
+ for name in ("CUDA_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES"):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setenv("UNSLOTH_VULKAN_DEVICE_PROBE", "1")
+ return ilp._vulkan_gpu_devices.__wrapped__
+
+
+def _vk_device(
+ name,
+ vendor = 0x5143,
+ type_ = 1,
+ api = (1 << 22) | (3 << 12),
+ driver = 8,
+ fp16 = True,
+):
+ return {
+ "vendor": vendor,
+ "type": type_,
+ "api": api,
+ "driver": driver,
+ "fp16": fp16,
+ "name": name,
+ }
+
+
+def test_vulkan_probe_keeps_only_gpus_llama_cpp_takes(monkeypatch):
+ probe = _probe_result(
+ monkeypatch,
+ [
+ _vk_device("Adreno"),
+ _vk_device("Radeon", vendor = 0x1002, type_ = 2, driver = 3),
+ _vk_device("llvmpipe", vendor = 0x10005, type_ = 4, driver = 13),
+ _vk_device("Dozen", driver = 23),
+ _vk_device("Vulkan 1.1 part", api = (1 << 22) | (1 << 12)),
+ _vk_device("No 16-bit storage", fp16 = False),
+ _vk_device("virtio", type_ = 3),
+ ],
+ )
+ assert probe() == ("Adreno", "Radeon")
+
+
+def test_vulkan_probe_hides_vendors_the_caller_masked(monkeypatch):
+ devices = [
+ _vk_device("RTX", vendor = 0x10DE, type_ = 2, driver = 4),
+ _vk_device("Radeon", vendor = 0x1002),
+ ]
+ probe = _probe_result(monkeypatch, devices)
+ monkeypatch.setattr(ilp.sys, "platform", "linux")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
+ assert probe() == ("Radeon",)
+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "0")
+ assert probe() == ()
+
+
[email protected]("stdout, returncode", [("[]", 0), ("not json", 0), ("", 0), ("[]", 1)])
+def test_vulkan_probe_failures_mean_no_gpu(monkeypatch, stdout, returncode):
+ probe = _probe_result(monkeypatch, stdout, returncode)
+ assert probe() == ()
+
+
+def test_vulkan_probe_off_switch_skips_the_child(monkeypatch):
+ probe = _probe_result(monkeypatch, [_vk_device("Adreno")])
+ monkeypatch.setattr(ilp, "run_capture", lambda *a, **k: pytest.fail("probe ran"))
+ monkeypatch.setenv("UNSLOTH_VULKAN_DEVICE_PROBE", "0")
+ assert probe() == ()
+
+
+def test_vulkan_probe_script_with_no_driver_lists_nothing(tmp_path):
+ # The real child, with the loader forced onto a driver file that does not exist.
+ env = {**os.environ, "VK_DRIVER_FILES": str(tmp_path / "x.json")}
+ result = subprocess.run(
+ [sys.executable, "-I", "-c", ilp._VULKAN_DEVICE_PROBE],
+ capture_output = True,
+ text = True,
+ timeout = 60,
+ env = env,
+ )
+ # No loader at all (macOS, a bare container) exits non-zero, which also reads as no GPU.
+ if result.returncode == 0:
+ assert json.loads(result.stdout) == []
+
+
+_FORK_TAG = "b11541-mix-2500203"
+_ARM64_VULKAN = f"app-{_FORK_TAG}-windows-arm64-vulkan.zip"
+_X64_VULKAN = f"app-{_FORK_TAG}-windows-x64-vulkan.zip"
+_ARM64_CPU = f"app-{_FORK_TAG}-windows-arm64-cpu.zip"
+
+
+def _windows_arm64_release(*names):
+ # Manifest shape the fork publishes (install_kind, bundle_profile, rank, sha index kind).
+ kinds = {
+ _ARM64_VULKAN: ("windows-arm64-vulkan", "windows-vulkan-arm64", 60),
+ _X64_VULKAN: ("windows-vulkan", "windows-vulkan-x64", 60),
+ _ARM64_CPU: ("windows-arm64", "windows-cpu-arm64", 1000),
+ }
+ artifacts = [
+ ilp.PublishedLlamaArtifact(
+ asset_name = name,
+ install_kind = kinds[name][0],
+ runtime_line = None,
+ coverage_class = None,
+ supported_sms = [],
+ min_sm = None,
+ max_sm = None,
+ bundle_profile = kinds[name][1],
+ rank = kinds[name][2],
+ )
+ for name in names
+ ]
+ bundle = ilp.PublishedReleaseBundle(
+ repo = FORK,
+ release_tag = _FORK_TAG,
+ upstream_tag = "b11541",
+ assets = {name: f"https://example/{name}" for name in names},
+ artifacts = artifacts,
+ )
+ checksums = ilp.ApprovedReleaseChecksums(
+ repo = FORK,
+ release_tag = bundle.release_tag,
+ upstream_tag = bundle.upstream_tag,
+ artifacts = {
+ artifact.asset_name: ilp.ApprovedArtifactHash(
+ asset_name = artifact.asset_name,
+ sha256 = "a" * 64,
+ repo = FORK,
+ kind = f"{artifact.install_kind}-app",
+ )
+ for artifact in artifacts
+ },
+ )
+ return bundle, checksums
+
+
+def _forced_windows_arm64_host():
+ routed, _repo, _tag, _persist = ilp._route_to_vulkan_prebuilt(
+ _windows_arm64_host(), FORK, "", force_cpu = False, llama_backend = "vulkan"
+ )
+ return routed
+
+
+def test_auto_routed_windows_arm64_picks_arm64_vulkan_then_cpu(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_LLAMA_CPP_BACKEND", raising = False)
+ monkeypatch.delenv("UNSLOTH_FORCE_VULKAN", raising = False)
+ monkeypatch.setattr(ilp, "_vulkan_gpu_devices", lambda: _ADRENO)
+ routed, _repo, _tag, _persist = ilp._route_to_vulkan_prebuilt(
+ _windows_arm64_host(), FORK, "", force_cpu = False
+ )
+ bundle, checksums = _windows_arm64_release(_X64_VULKAN, _ARM64_VULKAN, _ARM64_CPU)
+ attempts = ilp.resolve_release_asset_choice(routed, bundle.upstream_tag, bundle, checksums)
+ assert [a.name for a in attempts] == [_ARM64_VULKAN, _ARM64_CPU]
+
+
+def test_auto_routed_linux_arm64_picks_arm64_vulkan_then_cpu(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_LLAMA_CPP_BACKEND", raising = False)
+ monkeypatch.delenv("UNSLOTH_FORCE_VULKAN", raising = False)
+ monkeypatch.delenv("GGML_VK_VISIBLE_DEVICES", raising = False)
+ monkeypatch.setattr(ilp, "_vulkan_gpu_devices", lambda: ("Mali-G720",))
+ routed, _repo, _tag, _persist = ilp._route_to_vulkan_prebuilt(
+ _cpu_bound_host("Linux", "arm64"), FORK, "", force_cpu = False
+ )
+ names = {
+ f"app-{_FORK_TAG}-linux-x64-vulkan.tar.gz": ("linux-vulkan", "linux-vulkan-x64", 60),
+ f"app-{_FORK_TAG}-linux-arm64-vulkan.tar.gz": ("linux-vulkan", "linux-vulkan-arm64", 60),
+ f"app-{_FORK_TAG}-linux-arm64-cpu.tar.gz": ("linux-arm64", "linux-cpu-arm64", 1000),
+ }
+ bundle = ilp.PublishedReleaseBundle(
+ repo = FORK,
+ release_tag = _FORK_TAG,
+ upstream_tag = "b11541",
+ assets = {name: f"https://example/{name}" for name in names},
+ artifacts = [
+ ilp.PublishedLlamaArtifact(
+ asset_name = name,
+ install_kind = kind,
+ runtime_line = None,
+ coverage_class = None,
+ supported_sms = [],
+ min_sm = None,
+ max_sm = None,
+ bundle_profile = profile,
+ rank = rank,
+ )
+ for name, (kind, profile, rank) in names.items()
+ ],
+ )
+ attempts = ilp._linux_published_attempts(routed, bundle)
+ assert [a.name for a in attempts] == [
+ f"app-{_FORK_TAG}-linux-arm64-vulkan.tar.gz",
+ f"app-{_FORK_TAG}-linux-arm64-cpu.tar.gz",
+ ]
+
+
+def test_forced_vulkan_windows_arm64_picks_the_published_arm64_vulkan_bundle():
+ bundle, checksums = _windows_arm64_release(
+ _X64_VULKAN,
+ _ARM64_VULKAN,
+ _ARM64_CPU,
+ )
+ attempts = ilp.resolve_release_asset_choice(
+ _forced_windows_arm64_host(), bundle.upstream_tag, bundle, checksums
+ )
+ assert [(a.name, a.install_kind) for a in attempts] == [
+ (_ARM64_VULKAN, "windows-arm64-vulkan"),
+ (_ARM64_CPU, "windows-arm64"),
+ ]
+ # The strict Vulkan filter a named request applies keeps the arm64 Vulkan bundle.
+ assert [a.install_kind for a in ilp._backend_only_attempts(attempts, "vulkan")] == [
+ "windows-arm64-vulkan"
+ ]
+
+
+def test_forced_vulkan_windows_arm64_without_the_bundle_falls_back_to_cpu(capsys):
+ # Current fork releases ship no arm64 Vulkan archive; the x64 one must never be taken.
+ bundle, checksums = _windows_arm64_release(_X64_VULKAN, _ARM64_CPU)
+ attempts = ilp.resolve_release_asset_choice(
+ _forced_windows_arm64_host(), bundle.upstream_tag, bundle, checksums
+ )
+ assert [(a.name, a.install_kind) for a in attempts] == [(_ARM64_CPU, "windows-arm64")]
+ captured = capsys.readouterr()
+ assert "no published Windows ARM64 Vulkan bundle" in captured.out + captured.err
+
+
[email protected]("has_intel_gpu", [False, True])
+def test_unforced_windows_arm64_keeps_the_published_cpu_bundle(has_intel_gpu):
+ bundle, checksums = _windows_arm64_release(_ARM64_VULKAN, _ARM64_CPU)
+ attempts = ilp.resolve_release_asset_choice(
+ _windows_arm64_host(has_intel_gpu = has_intel_gpu), bundle.upstream_tag, bundle, checksums
+ )
+ assert [(a.name, a.install_kind) for a in attempts] == [(_ARM64_CPU, "windows-arm64")]
+
+
+_WIN_ARM64_UPSTREAM_ASSETS = [
+ "llama-b11400-bin-win-vulkan-x64.zip",
+ "llama-b11400-bin-win-vulkan-arm64.zip",
+ "llama-b11400-bin-win-cpu-arm64.zip",
+]
+
+
+def test_direct_upstream_forced_vulkan_windows_arm64_prefers_vulkan():
+ rel = _upstream_release("b11400", _WIN_ARM64_UPSTREAM_ASSETS)
+ plan = ilp.direct_upstream_release_plan(rel, _forced_windows_arm64_host(), UPSTREAM, "latest")
+ assert [(a.name, a.install_kind) for a in plan.attempts] == [
+ ("llama-b11400-bin-win-vulkan-arm64.zip", "windows-arm64-vulkan"),
+ ("llama-b11400-bin-win-cpu-arm64.zip", "windows-arm64"),
+ ]
+
+
+def test_direct_upstream_unforced_windows_arm64_stays_cpu():
+ rel = _upstream_release("b11400", _WIN_ARM64_UPSTREAM_ASSETS)
+ plan = ilp.direct_upstream_release_plan(
+ rel, _windows_arm64_host(has_intel_gpu = True), UPSTREAM, "latest"
+ )
+ assert [a.install_kind for a in plan.attempts] == ["windows-arm64"]
+
+
+def test_upstream_resolver_forced_vulkan_windows_arm64(monkeypatch):
+ assets = {name: f"https://example/{name}" for name in _WIN_ARM64_UPSTREAM_ASSETS}
+ monkeypatch.setattr(ilp, "github_release_assets", lambda repo, tag: dict(assets))
+ choice = ilp.resolve_upstream_asset_choice(_forced_windows_arm64_host(), "b11400")
+ assert (choice.name, choice.install_kind) == (
+ "llama-b11400-bin-win-vulkan-arm64.zip",
+ "windows-arm64-vulkan",
+ )
+ del assets["llama-b11400-bin-win-vulkan-arm64.zip"]
+ choice = ilp.resolve_upstream_asset_choice(_forced_windows_arm64_host(), "b11400")
+ assert choice.install_kind == "windows-arm64"
+ choice = ilp.resolve_upstream_asset_choice(_windows_arm64_host(has_intel_gpu = True), "b11400")
+ assert choice.install_kind == "windows-arm64"
+
+
+def test_windows_arm64_vulkan_kind_is_a_vulkan_bundle():
+ from utils.prebuilt.llama_backend import backend_for_install_kind, marker_backend
+
+ assert backend_for_install_kind("windows-arm64-vulkan") == "vulkan"
+ assert "windows-arm64-vulkan" in ilp.VULKAN_INSTALL_KINDS
+ assert "windows-arm64-vulkan" in ilp.install_kinds_for_backend("vulkan")
+ assert marker_backend({"install_kind": "windows-arm64-vulkan"}) == "vulkan"
+ choice = _choice("windows-arm64-vulkan", "llama-b11400-bin-win-vulkan-arm64.zip")
+ assert ilp.persisted_llama_backend("vulkan", choice) == "vulkan"
+ assert "*.dll" in ilp.runtime_patterns_for_choice(choice)
+ assert ["ggml-vulkan.dll"] in ilp.runtime_payload_health_groups(
+ "windows-arm64-vulkan", source_label = "upstream", tag = "b11400"
+ )
+
+
def test_vulkan_opt_in_still_routes_on_windows_x64(monkeypatch):
# Negative control for the arm64 guard: x64 keeps its Vulkan routing.
monkeypatch.setenv("UNSLOTH_LLAMA_CPP_BACKEND", "vulkan")
diff --git a/studio/backend/tests/test_setup_llama_cpp_backend.py b/studio/backend/tests/test_setup_llama_cpp_backend.py
index 019aa8e17ed..a8652e8c377 100644
--- a/studio/backend/tests/test_setup_llama_cpp_backend.py
+++ b/studio/backend/tests/test_setup_llama_cpp_backend.py
@@ -14,6 +14,7 @@ import os
import re
import shutil
import subprocess
+import sys
from pathlib import Path
import pytest
@@ -332,7 +333,7 @@ def test_llama_backend_source_choice_in_setup_ps1(backend, force_vulkan, expecte
assert out.stdout.strip() == f"RESULT:{expected_backend}:{expected_explicit}"
-def _run_ps1(value: str | None) -> str:
+def _run_ps1(value: str | None, extra_env: dict[str, str] | None = None) -> str:
normalize = _ps1_search(
r"\$llamaBackend = \$sourceLlamaBackend.*?Ignoring UNSLOTH_LLAMA_CPP_BACKEND.*?\n\s*\}",
re.DOTALL,
@@ -346,6 +347,7 @@ def _run_ps1(value: str | None) -> str:
}
if value is not None:
env["UNSLOTH_LLAMA_CPP_BACKEND"] = value
+ env.update(extra_env or {})
harness = (
# The spliced snippet warns through setup.ps1's output sink, which the real
# script defines above every call site. Stub it to Write-Host so the warning
@@ -393,15 +395,15 @@ def test_ps1_backend_gpu_opt_out_is_accepted(value):
assert out.strip() == "ARGS:"
-def test_ps1_forced_vulkan_fails_closed_on_windows_arm64():
- ps1 = _SETUP_PS1.read_text(encoding = "utf-8")
- branch = ps1.index(
- 'if ($llamaBackend -eq "vulkan" -or $explicitLlamaSourceBackend -eq "vulkan")'
+@_SKIP_NO_PWSH
[email protected](sys.platform == "darwin", reason = "the branch reads $IsMacOS, read-only in pwsh")
+def test_ps1_forced_vulkan_reaches_the_installer_on_windows_arm64():
+ # The installer resolves the Windows ARM64 Vulkan bundle (#11674), so setup must not refuse first.
+ out = _run_ps1(
+ "vulkan",
+ {"OS": "Windows_NT", "PROCESSOR_ARCHITECTURE": "ARM64", "PROCESSOR_ARCHITEW6432": ""},
)
- guarded = ps1[branch : ps1.index("Vulkan selected for GGUF inference", branch)]
- assert "elseif ($windowsArm64)" in guarded
- assert "no Windows ARM64 Vulkan bundle is published" in guarded
- assert "UNSLOTH_FORCE_VULKAN" in guarded
+ assert "Vulkan selected for GGUF inference" in out
@_SKIP_NO_PWSH
diff --git a/studio/backend/utils/prebuilt/llama_backend.py b/studio/backend/utils/prebuilt/llama_backend.py
index 3ce655f8b2b..1e195971ab5 100644
--- a/studio/backend/utils/prebuilt/llama_backend.py
+++ b/studio/backend/utils/prebuilt/llama_backend.py
@@ -20,6 +20,7 @@ INSTALL_KIND_BACKENDS: dict[str, str] = {
"windows-rocm": "rocm",
"linux-vulkan": "vulkan",
"windows-vulkan": "vulkan",
+ "windows-arm64-vulkan": "vulkan",
"linux-cpu": "cpu",
"linux-arm64": "cpu",
"windows-cpu": "cpu",
diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py
index 91b3ecda5e6..a86a9042a09 100644
--- a/studio/install_llama_prebuilt.py
+++ b/studio/install_llama_prebuilt.py
@@ -10,6 +10,7 @@ import argparse
import atexit
import errno
import fnmatch
+import functools
import glob
import hashlib
import importlib.util
@@ -166,8 +167,8 @@ UPSTREAM_WINDOWS_HIP_GFX_TARGETS = frozenset(
)
# install_kinds that really are a Vulkan bundle. A Vulkan request can still end on a CPU
-# bundle (no Vulkan archive on Windows arm64; x64 falls through when it is missing or fails
-# validation), so check against this to keep the marker honest (#7357).
+# bundle (it falls through when the Vulkan archive is missing or fails validation), so check
+# against this to keep the marker honest (#7357).
# Derived from the shared vocabulary rather than spelled out again: a new Vulkan bundle
# family added to INSTALL_KIND_BACKENDS must not be missed here.
VULKAN_INSTALL_KINDS = frozenset(
@@ -429,6 +430,10 @@ class HostInfo:
# (major, minor) from platform.mac_ver(); None off macOS or if unparseable.
# Skips a macos prebuilt whose minimum-OS exceeds this host.
macos_version: tuple[int, int] | None = None
+ # Set only by _vulkan_only_host. The Windows arm64 branches read it, not has_intel_gpu.
+ vulkan_routed: bool = False
+ # Set only by the probe route: the installed Vulkan bundle must itself list a Vulkan GPU.
+ vulkan_device_required: bool = False
@dataclass
@@ -1385,6 +1390,21 @@ def direct_upstream_release_plan(
)
)
attempts[:] = _drop_blackwell_incapable_windows_cuda(host, attempts)
+ # Upstream ships win-vulkan-arm64 since b11396; CPU bundle still last.
+ if host.vulkan_routed:
+ vulkan_asset = f"llama-{release_tag}-bin-win-vulkan-arm64.zip"
+ vulkan_url = assets.get(vulkan_asset)
+ if vulkan_url:
+ attempts.append(
+ AssetChoice(
+ repo = repo,
+ tag = release_tag,
+ name = vulkan_asset,
+ url = vulkan_url,
+ source_label = "upstream",
+ install_kind = "windows-arm64-vulkan",
+ )
+ )
# Upstream ggml-org/llama.cpp ships llama-bNNNN-bin-win-cpu-arm64.zip
# (visible in the b9334 release manifest). Without this branch the
# selector returned 0 attempts and the installer fell back to a
@@ -4279,6 +4299,23 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice
if attempts:
return attempts[0]
log("no compatible Windows ARM64 CUDA asset was found -- falling back to CPU")
+ if host.vulkan_routed:
+ vulkan_name = f"llama-{llama_tag}-bin-win-vulkan-arm64.zip"
+ if vulkan_name in upstream_assets:
+ log(
+ f"Vulkan requested on Windows ARM64 -- using upstream Vulkan prebuilt {vulkan_name}"
+ )
+ return AssetChoice(
+ repo = UPSTREAM_REPO,
+ tag = llama_tag,
+ name = vulkan_name,
+ url = upstream_assets[vulkan_name],
+ source_label = "upstream",
+ install_kind = "windows-arm64-vulkan",
+ )
+ log(
+ "Vulkan requested on Windows ARM64 but no Vulkan prebuilt found -- falling back to CPU"
+ )
upstream_name = f"llama-{llama_tag}-bin-win-cpu-arm64.zip"
if upstream_name not in upstream_assets:
raise PrebuiltFallback("upstream Windows ARM64 CPU asset was not found")
@@ -4428,6 +4465,25 @@ def resolve_release_asset_choice(
else:
published_choice = published_asset_choice_for_kind(release, "windows-cpu")
elif host.is_windows and host.is_arm64:
+ if host.vulkan_routed:
+ vulkan_choice = published_asset_choice_for_kind(
+ release, "windows-arm64-vulkan", host = host
+ )
+ if vulkan_choice is None:
+ log(
+ f"no published Windows ARM64 Vulkan bundle in {release.repo}@"
+ f"{release.release_tag}; falling through to the ARM64 CPU bundle."
+ )
+ choices = [
+ choice
+ for choice in (
+ vulkan_choice,
+ published_asset_choice_for_kind(release, "windows-arm64"),
+ )
+ if choice is not None
+ ]
+ if choices:
+ return apply_approved_hashes(choices, checksums)
# A host the fork's windows-arm64-cuda bundle does not cover takes its ARM64 CPU bundle.
# The opt-out gates the WHOLE branch.
if host.has_usable_nvidia and _upstream_arm64_cuda_allowed():
@@ -4868,6 +4924,7 @@ def runtime_patterns_for_install_kind(
"windows-arm64-cuda",
"windows-hip",
"windows-vulkan",
+ "windows-arm64-vulkan",
"windows-rocm",
"windows-arm64",
}:
@@ -6724,6 +6781,7 @@ def validate_server(
"windows-arm64-cuda",
"windows-hip",
"windows-vulkan",
+ "windows-arm64-vulkan",
"windows-rocm",
"macos-arm64",
}
@@ -8312,7 +8370,7 @@ def runtime_payload_health_groups(
# left the group satisfied by three libraries that cannot stand in for it.
# Measured on app-b10798-mix-659e406-windows-x64-rocm-gfx1150.zip.
return _windows_shared_groups(source_label, tag) + [["ggml-hip*.dll"]]
- if install_kind == "windows-vulkan":
+ if install_kind in {"windows-vulkan", "windows-arm64-vulkan"}:
groups = _windows_shared_groups(source_label, tag) + [["ggml-vulkan.dll"]]
if source_label == "published":
groups.append(["llama-diffusion-gemma-visual-server.exe"])
@@ -9897,6 +9955,29 @@ def existing_install_matches_plan(
)
+_LISTED_VULKAN_DEVICE_RE = re.compile(r"^\s*Vulkan\d+:", re.MULTILINE)
+
+
+def bundle_lists_vulkan_device(server_path: Path, install_dir: Path, host: HostInfo) -> bool:
+ """Whether llama.cpp itself, from the installed bundle, enumerates a Vulkan GPU."""
+ try:
+ result = subprocess.run(
+ [str(server_path), "--list-devices"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ # Device init compiles every shader pipeline, over a minute on a loaded host.
+ timeout = 180,
+ env = binary_env(server_path, install_dir, host),
+ **windows_hidden_subprocess_kwargs(),
+ )
+ except Exception as exc:
+ log(f"llama-server --list-devices failed: {exc}")
+ return False
+ return result.returncode == 0 and bool(_LISTED_VULKAN_DEVICE_RE.search(result.stdout or ""))
+
+
def validate_prebuilt_choice(
choice: AssetChoice,
host: HostInfo,
@@ -9957,6 +10038,15 @@ def validate_prebuilt_choice(
(server_path, quantize_path), install_dir, host
)
ensure_repo_shape(install_dir)
+ if (
+ host.vulkan_device_required
+ and choice.install_kind in VULKAN_INSTALL_KINDS
+ and not bundle_lists_vulkan_device(server_path, install_dir, host)
+ ):
+ # The bundle runs on CPU without a device, so validation alone would keep it.
+ raise PrebuiltFallback(
+ f"{choice.name} lists no Vulkan GPU on this host; using the CPU bundle instead"
+ )
write_prebuilt_metadata(
install_dir,
host = host,
@@ -10257,6 +10347,114 @@ def _vulkan_visible_device_mask_set() -> bool:
return os.environ.get("GGML_VK_VISIBLE_DEVICES") is not None
+# The checks ggml_vk_instance_init applies before it takes a device: a Vulkan 1.2 loader,
+# then a discrete or integrated GPU with 16-bit storage buffers (ggml_vk_device_is_supported).
+_VULKAN_DEVICE_PROBE = r"""
+import ctypes as c, json, os, sys
+if sys.platform == "win32":
+ root = os.environ.get("SystemRoot") or r"C:\Windows"
+ lib = c.WinDLL(os.path.join(root, "System32", "vulkan-1.dll"))
+else:
+ lib = c.CDLL("libvulkan.so.1")
+P, U32 = c.c_void_p, c.c_uint32
+def fn(name, *args):
+ f = getattr(lib, name)
+ f.argtypes, f.restype = list(args), c.c_int32
+ return f
+class App(c.Structure):
+ _fields_ = [("sType", c.c_int32), ("pNext", P), ("name", c.c_char_p), ("ver", U32),
+ ("engine", c.c_char_p), ("engineVer", U32), ("api", U32)]
+class Create(c.Structure):
+ _fields_ = [("sType", c.c_int32), ("pNext", P), ("flags", U32), ("app", P),
+ ("nLayers", U32), ("layers", P), ("nExts", U32), ("exts", P)]
+class Driver(c.Structure):
+ _fields_ = [("sType", c.c_int32), ("pNext", P), ("id", c.c_int32),
+ ("slack", c.c_char * 1024)]
+class Props(c.Structure):
+ _fields_ = [("sType", c.c_int32), ("pNext", P), ("api", U32), ("driverVersion", U32),
+ ("vendor", U32), ("device", U32), ("type", c.c_int32),
+ ("name", c.c_char * 256), ("slack", c.c_uint64 * 256)]
+class Vk11(c.Structure):
+ _fields_ = [("sType", c.c_int32), ("pNext", P), ("bits", U32 * 32)]
+class Features(c.Structure):
+ _fields_ = [("sType", c.c_int32), ("pNext", P), ("bits", U32 * 128)]
+V12 = (1 << 22) | (2 << 12)
+version = U32(1 << 22)
+if hasattr(lib, "vkEnumerateInstanceVersion"):
+ fn("vkEnumerateInstanceVersion", c.POINTER(U32))(c.byref(version))
+devices = []
+if version.value >= V12:
+ app = App(0, None, b"unsloth", 1, None, 0, version.value)
+ instance = P()
+ if fn("vkCreateInstance", P, P, P)(c.byref(Create(1, None, 0, c.addressof(app))), None, c.byref(instance)) == 0:
+ enum = fn("vkEnumeratePhysicalDevices", P, c.POINTER(U32), P)
+ count = U32(0)
+ enum(instance, c.byref(count), None)
+ handles = (P * count.value)()
+ enum(instance, c.byref(count), handles)
+ get_props = fn("vkGetPhysicalDeviceProperties2", P, P)
+ get_features = fn("vkGetPhysicalDeviceFeatures2", P, P)
+ for handle in handles[: count.value]:
+ driver = Driver(1000196000)
+ props = Props(1000059001, c.addressof(driver))
+ get_props(handle, c.byref(props))
+ fp16 = False
+ if props.api >= V12:
+ vk11 = Vk11(49)
+ get_features(handle, c.byref(Features(1000059000, c.addressof(vk11))))
+ fp16 = bool(vk11.bits[0])
+ devices.append({"vendor": props.vendor, "type": props.type, "api": props.api,
+ "driver": driver.id, "fp16": fp16,
+ "name": props.name.decode("utf-8", "replace")})
+ fn("vkDestroyInstance", P, P)(instance, None)
+print(json.dumps(devices))
+"""
+# VK_DRIVER_ID_GOOGLE_SWIFTSHADER, _MESA_LLVMPIPE (software), _MESA_DOZEN (Vulkan over D3D12).
+_VULKAN_SKIPPED_DRIVER_IDS = frozenset({10, 13, 23})
+
+
[email protected]_cache(maxsize = 1)
+def _vulkan_gpu_devices() -> tuple[str, ...]:
+ """Names of the GPUs the Vulkan bundle would run on; empty when none or the probe fails.
+
+ A child process, since loading a broken driver can take its caller down with it.
+ UNSLOTH_VULKAN_DEVICE_PROBE=0 turns it off.
+ """
+ if os.environ.get("UNSLOTH_VULKAN_DEVICE_PROBE", "1") == "0":
+ return ()
+ try:
+ result = run_capture([sys.executable, "-I", "-c", _VULKAN_DEVICE_PROBE], timeout = 30)
+ devices = json.loads(result.stdout.strip().splitlines()[-1])
+ except Exception:
+ return ()
+ if result.returncode != 0 or not isinstance(devices, list):
+ return ()
+ # Vulkan ignores the CUDA and HIP masks, so a vendor the caller hid stays hidden.
+ hidden_vendors = set()
+ if os.environ.get("CUDA_VISIBLE_DEVICES") is not None:
+ hidden_vendors.add(0x10DE)
+ if (
+ _hip_visible_device_mask_set()
+ if sys.platform == "win32"
+ else os.environ.get("ROCR_VISIBLE_DEVICES") is not None
+ ):
+ hidden_vendors.add(0x1002)
+ names = []
+ for device in devices:
+ try:
+ if (
+ device["type"] in (1, 2)
+ and device["api"] >= (1 << 22) | (2 << 12)
+ and device["fp16"]
+ and device["driver"] not in _VULKAN_SKIPPED_DRIVER_IDS
+ and device["vendor"] not in hidden_vendors
+ ):
+ names.append(str(device["name"]))
+ except (KeyError, TypeError):
+ continue
+ return tuple(names)
+
+
def _windows_hip_gfx_targets(published_repo: str | None) -> frozenset[str]:
"""gfx targets the Windows HIP bundle of ``published_repo`` is actually built for.
@@ -10769,15 +10967,6 @@ def _should_auto_vulkan_for_amd_linux(host: HostInfo) -> bool:
return _amd_vulkan_icd_present()
-def _has_no_vulkan_prebuilt(host: HostInfo) -> bool:
- """Platforms that ship no Vulkan prebuilt at all, so routing there is pointless.
-
- Upstream builds win-vulkan for x64 only; Windows arm64 gets CPU plus opencl-adreno, so
- rewriting it to Vulkan-only would just swap the published bundle for the upstream CPU
- one. macOS is handled separately (Metal)."""
- return host.is_windows and host.is_arm64
-
-
def _vulkan_only_host(host: HostInfo) -> HostInfo:
"""Rewrite ``host`` so the asset selectors take their Vulkan branch.
@@ -10794,6 +10983,7 @@ def _vulkan_only_host(host: HostInfo) -> HostInfo:
rocm_gfx_target = None,
rocm_gfx_targets = [],
has_intel_gpu = True,
+ vulkan_routed = True,
)
@@ -10835,7 +11025,7 @@ def _route_to_vulkan_prebuilt(
"""Point a Vulkan-capable host at the selected repository's Vulkan prebuilt.
The default Unsloth release manifest includes Vulkan app bundles, including the
- DiffusionGemma visual server. Three triggers route here, all suppressed when a CPU
+ DiffusionGemma visual server. These triggers route here, all suppressed when a CPU
flag (--cpu-fallback or --force-cpu, folded into force_cpu) wins:
* ``UNSLOTH_LLAMA_CPP_BACKEND=vulkan`` / ``UNSLOTH_FORCE_VULKAN`` /
``--llama-backend vulkan`` forces Vulkan over the detected CUDA/ROCm backend;
@@ -10844,7 +11034,9 @@ def _route_to_vulkan_prebuilt(
* an AMD host whose every GPU is an integrated part Vulkan serves better prefers
Vulkan over the HIP bundle it could otherwise run;
* an auto-detected Intel GPU with NO physical NVIDIA/ROCm, the purpose of the
- has_intel_gpu probe.
+ has_intel_gpu probe;
+ * any other non-macOS host that would get the CPU bundle, when the Vulkan loader
+ enumerates a GPU llama.cpp accepts (Snapdragon X, Linux arm64 GPUs, nouveau).
Applied by BOTH the install path and the --resolve-prebuilt probe so the "is a prebuilt
available" answer matches what actually gets installed.
@@ -10866,10 +11058,26 @@ def _route_to_vulkan_prebuilt(
auto_igpu = detecting and amd_igpu_host
# No PHYSICAL NVIDIA, not merely no usable one: Vulkan ignores CUDA_VISIBLE_DEVICES, so
# auto-routing a host that hides its NVIDIA card would let it grab the reserved GPU.
+ # Not Windows arm64: a stale Intel adapter entry there names no GPU; the probe below decides.
auto_intel = (
- detecting and host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm
+ detecting
+ and host.has_intel_gpu
+ and not host.has_physical_nvidia
+ and not host.has_rocm
+ and not (host.is_windows and host.is_arm64)
+ )
+ # Probe last so only hosts otherwise headed for the CPU bundle pay for it; CPU stays the fallback.
+ auto_device = (
+ detecting
+ and not (forced or force_cpu or auto_intel or auto_no_hip or auto_igpu)
+ and not host.is_macos
+ and not host.has_physical_nvidia
+ and not host.has_rocm
+ and not (host.is_x86_64 and host.has_amd_gpu_without_rocm)
+ and not _vulkan_visible_device_mask_set()
+ and bool(_vulkan_gpu_devices())
)
- if force_cpu or not (forced or auto_intel or auto_no_hip or auto_igpu):
+ if force_cpu or not (forced or auto_intel or auto_no_hip or auto_igpu or auto_device):
return host, published_repo, published_release_tag, None
if host.is_macos:
if forced:
@@ -10878,13 +11086,6 @@ def _route_to_vulkan_prebuilt(
"(Metal is used; there is no Vulkan prebuilt)"
)
return host, published_repo, published_release_tag, None
- if _has_no_vulkan_prebuilt(host):
- if forced:
- log(
- "Vulkan llama.cpp backend requested on Windows arm64, but no "
- "compatible Vulkan bundle is published"
- )
- return host, published_repo, published_release_tag, None
if auto_no_hip:
active = _active_rocm_gfx_target(host) or "unknown"
if host.is_linux:
@@ -10920,9 +11121,18 @@ def _route_to_vulkan_prebuilt(
# same. Persist it as automatic so pre-#8050 installs (and the --llama-backend
# vulkan every update re-asserts) stay eligible for the CPU crash recovery.
persist_backend = "auto" if (amd_no_hip_host or amd_igpu_host) else "vulkan"
- else:
+ elif auto_intel:
log("Intel GPU detected; installing the Vulkan llama.cpp prebuilt")
persist_backend = None
+ else:
+ names = ", ".join(dict.fromkeys(_vulkan_gpu_devices()))
+ log(
+ f"Vulkan GPU found ({names}); installing the Vulkan llama.cpp prebuilt, "
+ "with the CPU bundle as fallback"
+ )
+ host = dataclasses_replace(_vulkan_only_host(host), vulkan_device_required = True)
+ # Detected, like the Intel route: updates re-detect and the CPU crash recovery holds.
+ persist_backend = None
# A fork release without a linux-vulkan-arm64 bundle source-builds; no upstream prebuilt.
return host, published_repo, published_release_tag, persist_backend
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index edc73ebe155..d0b4e6f4932 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -2337,21 +2337,13 @@ def _detect_windows_gfx_arch() -> str | None:
# Torch ends here, llama.cpp does not: Vulkan drives these cards
# (#8458 ran an RX 580 through it). PowerShell syntax because this
# branch is Windows-only: a pasted VAR=value parses there as a
- # command name and sets nothing. Not on ARM64: setup.ps1 THROWS on
- # that variable there, so this would abort the next update.
- if _is_windows_arm64():
- _safe_print(
- " [INFO] GGUF chat would need Vulkan on this GPU, and no "
- "Windows ARM64 Vulkan bundle is published: build llama.cpp "
- "from source, or run this on x64."
- )
- else:
- _safe_print(
- " [INFO] GGUF chat can still run on this GPU through Vulkan: set "
- '$env:UNSLOTH_LLAMA_CPP_BACKEND = "vulkan" and re-run the installer. '
- "It selects the llama.cpp bundle at install time, so setting it "
- "afterwards has no effect until you install or update again."
- )
+ # command name and sets nothing.
+ _safe_print(
+ " [INFO] GGUF chat can still run on this GPU through Vulkan: set "
+ '$env:UNSLOTH_LLAMA_CPP_BACKEND = "vulkan" and re-run the installer. '
+ "It selects the llama.cpp bundle at install time, so setting it "
+ "afterwards has no effect until you install or update again."
+ )
else:
_safe_print(
f" [WARN] could not map '{_names[_sel]}' to a gfx arch, so torch "
diff --git a/studio/install_whisper_prebuilt.py b/studio/install_whisper_prebuilt.py
index 6a455e3af6a..4ff605de311 100644
--- a/studio/install_whisper_prebuilt.py
+++ b/studio/install_whisper_prebuilt.py
@@ -540,10 +540,8 @@ def slim_pairing_for_artifact(
return None
required_sonames = [str(name) for name in sonames]
if host.is_windows and _ships_windows_gpu_ggml_module(llama_bin_dir):
- # The shared Windows manifest lists libomp only because the cpu bundle's ggml links against it, so
- # requiring it for a GPU bundle only mis-rejects; link_ggml_runtime still wires it. Dropping the
- # aarch64 name too is safe only while llama publishes no Windows arm64 GPU bundle, whose
- # clang-built ggml really does need LLVM OpenMP: re-check this gate before adding one.
+ # libomp is only needed by the cpu bundle (arm64 Vulkan builds GGML_OPENMP=OFF); re-check if an
+ # arm64 GPU bundle ever builds with OpenMP.
required_sonames = [
name
for name in required_sonames
diff --git a/studio/setup.ps1 b/studio/setup.ps1
index 86afbd513a8..b19f840efea 100644
--- a/studio/setup.ps1
+++ b/studio/setup.ps1
@@ -5148,9 +5148,7 @@ if ($HasNvidiaSmi) {
# NotImplementedError at import (unsloth/device_type.py). The Vulkan setter is
# single-quoted so PowerShell prints $env:... rather than expanding it; a pasted
# VAR=value resolves as a command name here and sets nothing.
- # Both claims are conditional: an explicit index pin is honoured for any arch, and
- # this same script THROWS on the Vulkan variable on Windows ARM64, where no bundle
- # is published, so the usual advice would abort the next update instead of helping.
+ # Conditional: an explicit index pin is honoured for any arch.
$unsupPinned = (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_TORCH_INDEX_URL)) -or `
(-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_TORCH_INDEX_FAMILY))
if ($unsupPinned) {
@@ -5161,14 +5159,10 @@ if ($HasNvidiaSmi) {
substep "CPU-only: Unsloth training and GPU inference are unavailable. Installing the" "Yellow"
substep "HIP SDK or setting UNSLOTH_ROCM_GFX_ARCH will not change that for it." "Yellow"
}
- if ((Get-HostMachineArch) -eq "arm64") {
- substep "GGUF chat would need Vulkan on this GPU, and no Windows ARM64 Vulkan bundle is published: build llama.cpp from source, or run this on x64." "Yellow"
- } else {
- substep "GGUF chat can still use this GPU through Vulkan: set" "Yellow"
- substep '$env:UNSLOTH_LLAMA_CPP_BACKEND = "vulkan" and re-run the installer. It selects' "Yellow"
- substep "the llama.cpp bundle at install time, so setting it afterwards has no effect" "Yellow"
- substep "until you install or update again." "Yellow"
- }
+ substep "GGUF chat can still use this GPU through Vulkan: set" "Yellow"
+ substep '$env:UNSLOTH_LLAMA_CPP_BACKEND = "vulkan" and re-run the installer. It selects' "Yellow"
+ substep "the llama.cpp bundle at install time, so setting it afterwards has no effect" "Yellow"
+ substep "until you install or update again." "Yellow"
Write-StudioLine ""
} elseif ($ROCmGpuLabel) {
Write-StudioLine ""
@@ -10254,6 +10248,8 @@ if ($LocalLlamaCppLinked) {
$expectedKinds = if ($HasROCm -or $script:ROCmGfxArch) { $_rocmKinds } elseif ($_nvidiaEvidence) { $_nvidiaKinds } else { @("windows-cpu", "windows-arm64", "windows-vulkan") }
# A pre-CUDA 11 library still proves physical NVIDIA presence, even when AMD/Intel selects torch.
if ($script:NvidiaDriverLibraryOnly) { $expectedKinds += $_nvidiaKinds }
+ # ARM64 Vulkan (a Vulkan GPU found, or UNSLOTH_LLAMA_CPP_BACKEND=vulkan) can land on any ARM64 host.
+ if (Test-WinArm64Venv) { $expectedKinds += "windows-arm64-vulkan" }
if ($existingKind -and ($existingKind -notin $expectedKinds)) {
substep "Removing mismatched llama.cpp install (found '$existingKind', need one of: $($expectedKinds -join ', '))..."
Remove-Item -Recurse -Force -LiteralPath $LlamaCppDir -ErrorAction SilentlyContinue
@@ -10289,18 +10285,9 @@ if ($LocalLlamaCppLinked) {
# marker, so forwarding a second copy from here could only ever disagree
# with it. The override does not change the torch backend.
$llamaBackend = $sourceLlamaBackend
- $windowsArm64 = (
- $env:OS -eq "Windows_NT" -and
- (
- "$($env:PROCESSOR_ARCHITECTURE)".ToUpperInvariant() -eq "ARM64" -or
- "$($env:PROCESSOR_ARCHITEW6432)".ToUpperInvariant() -eq "ARM64"
- )
- )
if ($llamaBackend -eq "vulkan" -or $explicitLlamaSourceBackend -eq "vulkan") {
if ($IsMacOS) {
Write-StudioLine "[WARN] Vulkan has no effect on macOS; the universal build uses Metal" -ForegroundColor Yellow
- } elseif ($windowsArm64) {
- throw "Vulkan was requested, but no Windows ARM64 Vulkan bundle is published. Unset UNSLOTH_LLAMA_CPP_BACKEND / UNSLOTH_FORCE_VULKAN or compile llama.cpp from source."
} else {
Write-StudioLine " llama.cpp Vulkan selected for GGUF inference; the PyTorch training backend is unchanged" -ForegroundColor Cyan
}
diff --git a/tests/conftest.py b/tests/conftest.py
index 9f9d1e8c3e0..8b1826af955 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -54,6 +54,8 @@ import pytest
# A test that hides nvidia-smi to fake a CPU host must not find the real GPUs through NVML.
os.environ.setdefault("UNSLOTH_NVIDIA_LIBRARY_PROBE", "0")
+# A fake CPU host must not take the Vulkan route through this machine's real GPUs.
+os.environ.setdefault("UNSLOTH_VULKAN_DEVICE_PROBE", "0")
@pytest.fixture(autouse = True)
diff --git a/tests/studio/install/test_install_llama_prebuilt_logic.py b/tests/studio/install/test_install_llama_prebuilt_logic.py
index 1cee3977fe7..c22cce4fd2f 100644
--- a/tests/studio/install/test_install_llama_prebuilt_logic.py
+++ b/tests/studio/install/test_install_llama_prebuilt_logic.py
@@ -4179,6 +4179,124 @@ def _run_validate_prebuilt_choice(
return calls
+def _run_vulkan_device_gate(
+ monkeypatch,
+ tmp_path,
+ *,
+ required,
+ listed,
+ install_kind = "linux-vulkan",
+):
+ """validate_prebuilt_choice on a stubbed install; `listed` is what the bundle's --list-devices finds."""
+ src = INSTALL_LLAMA_PREBUILT
+ server_path = tmp_path / "install" / "build" / "bin" / "llama-server"
+ listings = []
+
+ def fake_listing(*_a, **_k):
+ listings.append(True)
+ return listed
+
+ monkeypatch.setattr(
+ src, "preferred_source_archive", lambda *a, **k: ("repo", "ref", None, False)
+ )
+ monkeypatch.setattr(src, "hydrate_source_tree", lambda *a, **k: None)
+ monkeypatch.setattr(src, "install_from_archives", lambda *a, **k: (server_path, server_path))
+ monkeypatch.setattr(src, "runtime_payload_is_healthy", lambda *a, **k: True)
+ monkeypatch.setattr(src, "preflight_linux_installed_binaries", lambda *a, **k: None)
+ monkeypatch.setattr(src, "preflight_macos_installed_binaries", lambda *a, **k: None)
+ monkeypatch.setattr(src, "ensure_repo_shape", lambda *a, **k: None)
+ monkeypatch.setattr(src, "write_prebuilt_metadata", lambda *a, **k: None)
+ monkeypatch.setattr(src, "bundle_lists_vulkan_device", fake_listing)
+ monkeypatch.setattr(src, "_RUN_STAGED_PREBUILT_VALIDATION", False)
+ bundle_name = f"app-b9998-{install_kind}.tar.gz"
+ source_archive = tmp_path / "source.tar.gz"
+ bundle_archive = tmp_path / "bundle.tar.gz"
+ source_archive.write_bytes(b"source")
+ bundle_archive.write_bytes(b"bundle")
+ choice = asset_choice(
+ name = bundle_name,
+ url = "file://bundle",
+ repo = "local",
+ tag = "b9998",
+ source_label = "local",
+ is_ready_bundle = True,
+ install_kind = install_kind,
+ expected_sha256 = "ab" * 32,
+ )
+ src.validate_prebuilt_choice(
+ choice,
+ linux_host(has_intel_gpu = True, vulkan_routed = True, vulkan_device_required = required),
+ tmp_path / "install",
+ tmp_path / "work",
+ tmp_path / "stories260K.gguf",
+ requested_tag = "b9998",
+ llama_tag = "b9998",
+ release_tag = "b9998",
+ approved_checksums = approved_checksums_for(
+ "b9998",
+ source_archive = source_archive,
+ bundle_archive = bundle_archive,
+ bundle_name = bundle_name,
+ ),
+ prebuilt_fallback_used = False,
+ quantized_path = tmp_path / "stories260K-q4.gguf",
+ )
+ return listings
+
+
+def test_probe_routed_vulkan_bundle_that_lists_no_gpu_falls_back(tmp_path, monkeypatch):
+ with pytest.raises(PrebuiltFallback, match = "lists no Vulkan GPU"):
+ _run_vulkan_device_gate(monkeypatch, tmp_path, required = True, listed = False)
+
+
+def test_probe_routed_vulkan_bundle_that_lists_a_gpu_is_kept(tmp_path, monkeypatch):
+ assert _run_vulkan_device_gate(monkeypatch, tmp_path, required = True, listed = True) == [True]
+
+
+ "required, install_kind",
+ [(False, "linux-vulkan"), (True, "linux-arm64")],
+ ids = ["intel-or-forced-route", "cpu-bundle"],
+)
+def test_vulkan_device_gate_only_checks_probe_routed_vulkan_bundles(
+ tmp_path, monkeypatch, required, install_kind
+):
+ assert (
+ _run_vulkan_device_gate(
+ monkeypatch, tmp_path, required = required, listed = False, install_kind = install_kind
+ )
+ == []
+ )
+
+
[email protected](sys.platform == "win32", reason = "fake server is a POSIX shell script")
+ "output, code, expected",
+ [
+ ("Available devices:\n Vulkan0: Adreno X1-85 (16000 MiB, 15000 MiB free)\n", 0, True),
+ ("Available devices:\n (none)\n", 0, False),
+ ("Available devices:\n RPC0: host:50052 (0 MiB, 0 MiB free)\n", 0, False),
+ ("Available devices:\n Vulkan0: Adreno\n", 1, False),
+ ],
+ ids = ["vulkan-device", "none", "other-device", "nonzero-exit"],
+)
+def test_bundle_lists_vulkan_device_reads_list_devices(tmp_path, output, code, expected):
+ server = tmp_path / "llama-server"
+ server.write_text(f"#!/bin/sh\nprintf '{output}'\nexit {code}\n", encoding = "utf-8")
+ server.chmod(0o755)
+ assert (
+ INSTALL_LLAMA_PREBUILT.bundle_lists_vulkan_device(server, tmp_path, linux_host())
+ is expected
+ )
+
+
+def test_bundle_lists_vulkan_device_missing_binary_is_no_gpu(tmp_path):
+ missing = tmp_path / "llama-server"
+ assert (
+ INSTALL_LLAMA_PREBUILT.bundle_lists_vulkan_device(missing, tmp_path, linux_host()) is False
+ )
+
+
def test_validate_prebuilt_choice_approved_validation_skipped_when_flag_off(tmp_path, monkeypatch):
calls = _run_validate_prebuilt_choice(monkeypatch, tmp_path, expected_sha256 = "ab" * 32)
assert calls == {"quantize": 0, "server": 0}
diff --git a/tests/studio/install/test_llama_kind_gate_matches_selector.py b/tests/studio/install/test_llama_kind_gate_matches_selector.py
index 54893a61d8f..29b4f6f9672 100644
--- a/tests/studio/install/test_llama_kind_gate_matches_selector.py
+++ b/tests/studio/install/test_llama_kind_gate_matches_selector.py
@@ -173,3 +173,32 @@ def test_no_arm64_row_expects_a_kind_only_published_for_x64():
f"nvidia={nvidia} rocm={rocm} optout={opt_out}: the gate expects only x64 kinds "
f"on an ARM64 venv: {sorted(expected)}"
)
+
+
+@requires_pwsh
+ ("nvidia", "rocm", "opt_out"),
+ [(False, False, False), (True, False, False), (True, False, True), (False, True, False)],
+)
+def test_the_gate_keeps_an_opt_in_arm64_vulkan_install(nvidia, rocm, opt_out):
+ """UNSLOTH_LLAMA_CPP_BACKEND=vulkan on Windows ARM64 installs windows-arm64-vulkan; a gate
+ without it deletes that tree on every update. An x64 venv never expects it."""
+ name = f"llama-{TAG}-bin-win-vulkan-arm64.zip"
+ release = {
+ **RELEASE,
+ "assets": RELEASE["assets"]
+ + [
+ {
+ "name": name,
+ "browser_download_url": f"https://example.invalid/{name}",
+ "digest": f"sha256:{_fixture_digest(name)}",
+ }
+ ],
+ }
+ routed = ip._vulkan_only_host(_host(arm64 = True, nvidia = nvidia, rocm = rocm))
+ plan = ip.direct_upstream_release_plan(release, routed, "ggml-org/llama.cpp", "latest")
+ assert plan.attempts[0].install_kind == "windows-arm64-vulkan"
+ expected = _expected_kinds(arm64_venv = True, nvidia = nvidia, rocm = rocm, opt_out = opt_out)
+ assert "windows-arm64-vulkan" in expected
+ x64 = _expected_kinds(arm64_venv = False, nvidia = nvidia, rocm = rocm, opt_out = opt_out)
+ assert "windows-arm64-vulkan" not in x64
diff --git a/tests/studio/install/test_rdna1_unsupported_message_8529.py b/tests/studio/install/test_rdna1_unsupported_message_8529.py
index 5b1935807f5..37aa56b0774 100644
--- a/tests/studio/install/test_rdna1_unsupported_message_8529.py
+++ b/tests/studio/install/test_rdna1_unsupported_message_8529.py
@@ -220,60 +220,16 @@ class TestExplicitIndexPinIsHonoured:
)
-class TestWindowsArm64GetsNoVulkanAdvice:
- """studio/setup.ps1 THROWS when the Vulkan variable is set on Windows ARM64 (no
- bundle is published there), so telling an ARM64 user to set it and re-run aborts
- the update instead of enabling GGUF acceleration."""
-
- def test_the_throw_this_depends_on_is_still_there(self):
- src = _normalised(_SETUP_PS1)
- assert (
- "no Windows ARM64 Vulkan bundle is published" in src
- ), "the ARM64 guard this test is built on was renamed; re-read setup.ps1"
-
- # setup.ps1 runs the Python stack before its own throw, so its setter is the last advice seen.
- @pytest.mark.parametrize(
- "path,guard",
- [
- (_INSTALL_PS1, "Get-HostMachineArch"),
- (_SETUP_PS1, "Get-HostMachineArch"),
- (_STACK_PY, "_is_windows_arm64"),
- ],
- ids = lambda p: getattr(p, "name", p),
- )
- def test_every_vulkan_offer_is_behind_an_arch_check(self, path, guard):
- lines = _normalised(path).splitlines()
- offers = [
- i
- for i, line in enumerate(lines)
- if _SETTER[path.name] in line and not line.lstrip().startswith("#")
- ]
- assert offers, f"{path.name}: no Vulkan offer found"
- for i in offers:
- # Check the resolver itself: a hardcoded $unsupArm64 = $false mutant survived a name check.
- back = "\n".join(lines[max(i - 20, 0) : i])
- assert guard in back, (
- f"{path.name}:{i + 1}: offers the Vulkan variable without checking for "
- f"Windows ARM64, where setting it throws:\n{back}"
- )
-
-
class TestPythonStackWindowsArm64:
- """The Windows WMI path prints the same Vulkan advice as install.ps1, and the same
- ARM64 throw applies to it: setup.ps1 rejects the variable there."""
+ """The Windows WMI path prints the same Vulkan advice as install.ps1. ARM64 gets it
+ too: releases ship a Windows ARM64 Vulkan bundle, and setup.ps1 accepts the variable."""
- def test_arm64_gets_a_source_build_note_instead_of_the_setter(self):
- _arch, out = _wmi_detect(["AMD Radeon RX 580"], arm64 = True)
+ @pytest.mark.parametrize("arm64", [False, True], ids = ["x64", "arm64"])
+ def test_every_arch_gets_the_setter(self, arm64):
+ _arch, out = _wmi_detect(["AMD Radeon RX 580"], arm64 = arm64)
assert "gfx803" in out, "the card is still named"
- assert (
- "UNSLOTH_LLAMA_CPP_BACKEND" not in out
- ), f"ARM64 is still told to set the variable setup.ps1 throws on:\n{out}"
- assert "ARM64" in out and "source" in out
-
- def test_x64_still_gets_the_setter(self):
- """Positive control: the guard must not silence the advice everywhere."""
- _arch, out = _wmi_detect(["AMD Radeon RX 580"])
assert "UNSLOTH_LLAMA_CPP_BACKEND" in out
+ assert "no Windows ARM64 Vulkan bundle is published" not in out
@pytest.mark.parametrize(
"registry,env,expected",