Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Installer: stop picking cu126 when a slow NVIDIA driver times out CUDA detection (#11916)
Daniel Han5665b5a99fbf
diff --git a/install.ps1 b/install.ps1index c90a1deab54..e13da1e807f 100644--- a/install.ps1+++ b/install.ps1@@ -8260,14 +8260,13 @@ exit 0}# "source;cudaMajor;cudaMinor;cap,cap" from NVML, else the CUDA driver API; "" when neither- # answers. Versions are major*1000 + minor*10. Read in a runspace of its own under a deadline:- # a wedged driver can block inside the library, and the deadline leaves that runspace behind.+ # answers. Versions are major*1000 + minor*10. One runspace + deadline per reader: a shared one let slow NVML starve CUDA.function Read-NvidiaLibraryRaw {- param([int]$TimeoutMs = 10000)+ param([int]$TimeoutMs = 30000)$type = Get-NvidiaLibraryProbeTypeif (-not $type) { return "" }$reader = {- param($T)+ param($T, $Which)function Read-Nvml {if ($T::nvmlInit_v2() -ne 0) { return "" }try {@@ -8309,26 +8308,30 @@ exit 0return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"}$r = ""- try { $r = Read-Nvml } catch { $r = "" }- if (-not $r) { try { $r = Read-Cuda } catch { $r = "" } }+ try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }return "$r"}- $ps = $null; $handle = $null- try {- $ps = [powershell]::Create()- $null = $ps.AddScript($reader.ToString()).AddArgument($type)- $handle = $ps.BeginInvoke()- if (-not $handle.AsyncWaitHandle.WaitOne($TimeoutMs)) { return "" }- return "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"- } catch { return "" }- finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }+ foreach ($which in @("nvml", "cuda")) {+ $ps = $null; $handle = $null; $r = ""+ try {+ $ps = [powershell]::Create()+ $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)+ $handle = $ps.BeginInvoke()+ if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {+ $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"+ }+ } catch { $r = "" }+ finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }+ if ($r) { return $r }+ }+ return ""}# NVIDIA inventory from the driver's own libraries (NVML, then the CUDA driver API), for a# host whose nvidia-smi is absent, stale or hangs (#9255). Twin of studio/nvidia_probe.py.# Cached. $null, or @{ Source; CudaMajor; CudaMinor; ComputeCaps ("8.9" strings); Count }.function Get-NvidiaLibraryInventory {- param([int]$TimeoutSec = 10)+ param([int]$TimeoutSec = 30)if ($script:NvidiaLibraryInventoryProbed) { return $script:NvidiaLibraryInventory }$script:NvidiaLibraryInventoryProbed = $true$script:NvidiaLibraryInventory = $null@@ -9262,6 +9265,10 @@ exit 0if ($NvidiaSmiExe) {try {$output = Invoke-NvidiaSmiBounded $NvidiaSmiExe+ if ($LASTEXITCODE -eq 124) {+ substep "nvidia-smi did not answer within 10s; retrying with a 45s limit..." "Yellow"+ $output = Invoke-NvidiaSmiBounded $NvidiaSmiExe -TimeoutSec 45+ }if ($output -match 'CUDA(?: UMD)? Version:\s+(\d+)\.(\d+)') {$major = [int]$Matches[1]; $minor = [int]$Matches[2]}@@ -9276,6 +9283,9 @@ exit 0return "$baseUrl/cpu"} else {substep "could not determine CUDA version from nvidia-smi, defaulting to cu126" "Yellow"+ $pinHint = if ($env:UNSLOTH_PYTORCH_MIRROR) { "UNSLOTH_TORCH_INDEX_FAMILY=" } else { "UNSLOTH_TORCH_INDEX_URL=$baseUrl/" }+ substep "cu126 has no kernels for Blackwell (sm_100 / sm_120). To choose the wheel yourself, re-run with" "Yellow"+ substep " ${pinHint}cu128 (or cu130 on a driver that supports CUDA 13)" "Yellow"return "$baseUrl/cu126"}}diff --git a/install.sh b/install.shindex fab0e9bf5ca..ee3ca5df0e5 100755--- a/install.sh+++ b/install.sh@@ -2644,8 +2644,10 @@ _wsl_amd_gpu_name() {# ── Bounded command runner ──_run_bounded() {+ _rb_secs=10+ if [ "${1:-}" = "--secs" ]; then _rb_secs=$2; shift 2; fiif command -v timeout >/dev/null 2>&1; then- timeout 10 "$@"+ timeout "$_rb_secs" "$@"else"$@"fi@@ -2676,7 +2678,11 @@ _nvidia_library_inventory() {else return 1fi_NVIDIA_LIBRARY_INVENTORY_STATE="none"- _NVIDIA_LIBRARY_INVENTORY_VALUE=$(_run_bounded "$_nli_py" -I - 2>/dev/null <<'PY'+ _NVIDIA_LIBRARY_INVENTORY_VALUE=""+ # One deadline per reader: a shared bound let slow NVML starve the CUDA driver API reader.+ for _nli_reader in nvml cuda; do+ case "$_nli_reader" in nvml) _nli_secs=30 ;; *) _nli_secs=20 ;; esac+ _NVIDIA_LIBRARY_INVENTORY_VALUE=$(_run_bounded --secs "$_nli_secs" "$_nli_py" -I - "$_nli_reader" 2>/dev/null <<'PY'import ctypes, os, sysdef load(*names):@@ -2733,8 +2739,10 @@ def cuda():caps.append(f"{major.value}.{minor.value}")return version.value, caps+# argv[1] runs one reader ("nvml" / "cuda") so each gets its own deadline; none runs both.+only = {"nvml": (nvml,), "cuda": (cuda,)}.get(sys.argv[1] if len(sys.argv) > 1 else "")found = None-for reader in (nvml, cuda): # a reader that raises (a missing symbol) yields to the next+for reader in only or (nvml, cuda): # a reader that raises (a missing symbol) yields to the nexttry:found = reader()except Exception:@@ -2746,11 +2754,52 @@ if not found or not found[1]:version, caps = foundprint(f"{version // 1000}.{version % 1000 // 10} {','.join(caps)}")PY-) && [ -n "$_NVIDIA_LIBRARY_INVENTORY_VALUE" ] || return 1+) && [ -n "$_NVIDIA_LIBRARY_INVENTORY_VALUE" ] && break+ _NVIDIA_LIBRARY_INVENTORY_VALUE=""+ done+ [ -n "$_NVIDIA_LIBRARY_INVENTORY_VALUE" ] || return 1_NVIDIA_LIBRARY_INVENTORY_STATE="found"printf '%s\n' "$_NVIDIA_LIBRARY_INVENTORY_VALUE"}+# Driver CUDA version without cuInit (cuDriverGetVersion, else /proc as in nvidia_probe.py _DRIVER_MAJOR_CUDA); picks a family only.+_nvidia_driver_cuda_version() {+ [ "${UNSLOTH_NVIDIA_LIBRARY_PROBE:-1}" != "0" ] || return 1+ if command -v python3 >/dev/null 2>&1; then _ndv_py=python3+ elif [ -n "${VENV_DIR:-}" ] && [ -x "$VENV_DIR/bin/python" ]; then _ndv_py="$VENV_DIR/bin/python"+ else _ndv_py=""+ fi+ if [ -n "$_ndv_py" ]; then+ _ndv_ver=$(_run_bounded "$_ndv_py" -I -c '+import ctypes, sys+for name in ("libcuda.so.1", "libcuda.so"):+ try:+ lib = ctypes.CDLL(name)+ break+ except OSError:+ lib = None+v = ctypes.c_int()+if lib is None or lib.cuDriverGetVersion(ctypes.byref(v)) != 0 or v.value < 1000:+ sys.exit(1)+print(f"{v.value // 1000}.{v.value % 1000 // 10}")+' 2>/dev/null </dev/null | awk 'NR == 1 { print $1 }') || _ndv_ver=""+ case "$_ndv_ver" in+ [0-9]*.[0-9]*) printf '%s\n' "$_ndv_ver"; return 0 ;;+ esac+ fi+ # "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 590.48.01 Release Build ..."+ _ndv_drv=$(awk 'NR == 1 { for (i = 1; i <= NF; i++) if ($i ~ /^[0-9]+\.[0-9]+(\.[0-9]+)?$/) { split($i, v, "."); print v[1]; exit } }' \+ /proc/driver/nvidia/version 2>/dev/null) || _ndv_drv=""+ case "$_ndv_drv" in ''|*[!0-9]*) return 1 ;; esac+ if [ "$_ndv_drv" -ge 580 ]; then echo "13.0"+ elif [ "$_ndv_drv" -ge 570 ]; then echo "12.8"+ elif [ "$_ndv_drv" -ge 560 ]; then echo "12.6"+ elif [ "$_ndv_drv" -ge 525 ]; then echo "12.0"+ elif [ "$_ndv_drv" -ge 450 ]; then echo "11.0"+ else return 1+ fi+}+# ── NVIDIA usable-GPU helper ──# nvidia-smi -L primary, /proc/driver/nvidia/gpus/ fallback, driver library last; a hidden GPU is NOT usable._has_usable_nvidia_gpu() {@@ -5235,19 +5284,37 @@ get_torch_index_url() {echo "$_base/cpu"; returnfi# CUDA version from nvidia-smi: accept "CUDA Version:" and the newer "CUDA UMD Version:".- _cuda_ver=$(export LC_ALL=C; _run_bounded "$_smi" 2>/dev/null \+ _smi_rc=0+ _smi_out=$(export LC_ALL=C; _run_bounded "$_smi" 2>/dev/null) || _smi_rc=$?+ if [ "$_smi_rc" = "124" ]; then+ echo "[INFO] nvidia-smi did not answer within 10s; retrying with a 45s limit..." >&2+ _smi_rc=0+ _smi_out=$(export LC_ALL=C; _run_bounded --secs 45 "$_smi" 2>/dev/null) || _smi_rc=$?+ # Still hung: do not spend another bound asking it for compute capabilities below.+ [ "$_smi_rc" = "124" ] && _smi=""+ fi+ _cuda_ver=$(printf '%s\n' "$_smi_out" \| sed -n \-e 's/.*CUDA UMD Version:[[:space:]]*\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' \-e 's/.*CUDA Version:[[:space:]]*\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' \| head -1)_inventory_caps=""+ _cuda_from_driver=""+ # A mirror base can carry credentials, so name only the leaf.+ if [ -n "${UNSLOTH_PYTORCH_MIRROR:-}" ]; then _pin_hint="UNSLOTH_TORCH_INDEX_FAMILY="+ else _pin_hint="UNSLOTH_TORCH_INDEX_URL=$_base/"+ fiif [ -z "$_cuda_ver" ]; then# nvidia-smi absent, stale or hung: the driver library knows both; cu126 is the last resort.if _inventory=$(_nvidia_library_inventory) && [ -n "$_inventory" ]; then_cuda_ver=${_inventory%% *}_inventory_caps=$(printf '%s' "${_inventory#* }" | tr ',' '\n')+ elif _cuda_ver=$(_nvidia_driver_cuda_version) && [ -n "$_cuda_ver" ]; then+ _cuda_from_driver=1elseecho "[WARN] Could not determine CUDA version from nvidia-smi, defaulting to cu126" >&2+ echo "[WARN] cu126 has no kernels for Blackwell (sm_100 / sm_120). To choose the wheel yourself, re-run with" >&2+ echo "[WARN] ${_pin_hint}cu128 (or cu130 on a driver that supports CUDA 13)" >&2echo "$_base/cu126"; returnfifi@@ -5259,7 +5326,14 @@ get_torch_index_url() {elif [ "$_major" -ge 12 ]; then _cuda_tag=cu124elif [ "$_major" -ge 11 ]; then _cuda_tag=cu118else echo "$_base/cpu"; return; fi- echo "$_base/$(_cap_cuda_family_for_pre_turing "$_cuda_tag" "$_smi" "$_inventory_caps")"+ _cuda_tag=$(_cap_cuda_family_for_pre_turing "$_cuda_tag" "$_smi" "$_inventory_caps")+ if [ -n "$_cuda_from_driver" ]; then+ echo "[WARN] nvidia-smi and the NVIDIA driver libraries did not answer in time; the driver supports CUDA $_cuda_ver." >&2+ echo "[WARN] Selecting the $_cuda_tag PyTorch wheels from the driver version alone. If that is wrong for this GPU, re-run with" >&2+ echo "[WARN] ${_pin_hint}cu126 (Maxwell to Hopper, sm_50-90)" >&2+ echo "[WARN] ${_pin_hint}cu128 (Turing and newer, including Blackwell)" >&2+ fi+ echo "$_base/$_cuda_tag"}# ── Torch flavor helpers (to repair a stale CPU / wrong-CUDA wheel) ──@@ -7371,6 +7445,112 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; thenfifi+# A wrong CUDA family fails only at first kernel launch; never cuInit here (minutes on a congested driver).+if [ "$SKIP_TORCH" = false ] && ! _cvd_hides_nvidia; then+ case "${_expected_torch_tag:-}" in+ cu[0-9]*)+ _arch_check=$(_run_bounded --secs 120 "$_VENV_PY" -c '+import ctypes, sys++def load(*names):+ for name in names:+ try:+ return ctypes.CDLL(name)+ except OSError:+ pass++def nvml_caps():+ lib = load("libnvidia-ml.so.1", "libnvidia-ml.so")+ if lib is None or lib.nvmlInit_v2() != 0:+ return None+ try:+ count, caps = ctypes.c_uint(), set()+ if lib.nvmlDeviceGetCount_v2(ctypes.byref(count)) != 0 or not count.value:+ return None+ for i in range(count.value):+ dev, major, minor = ctypes.c_void_p(), ctypes.c_int(), ctypes.c_int()+ if lib.nvmlDeviceGetHandleByIndex_v2(i, ctypes.byref(dev)) != 0 or \+ lib.nvmlDeviceGetCudaComputeCapability(dev, ctypes.byref(major), ctypes.byref(minor)) != 0:+ return None+ caps.add((major.value, minor.value))+ return sorted(caps)+ finally:+ lib.nvmlShutdown()++try:+ import torch+ if not torch.version.cuda or getattr(torch.version, "hip", None):+ sys.exit(0)+ try:+ archs = torch._C._cuda_getArchFlags().split()+ except Exception:+ archs = torch.cuda.get_arch_list()+ try:+ caps = nvml_caps()+ except Exception:+ caps = None+ if caps is None:+ if not torch.cuda.is_available():+ sys.exit(0)+ caps = sorted({torch.cuda.get_device_capability(i) for i in range(torch.cuda.device_count())})+except Exception:+ sys.exit(0)++def runs(arch, cap):+ kind, _, rest = arch.partition("_")+ n = len(rest) - len(rest.lstrip("0123456789"))+ digits, suffix = rest[:n], rest[n:]+ if n < 2:+ return False+ built = (int(digits[:-1]), int(digits[-1]))+ if suffix == "a": # arch-specific cubin / PTX: that exact GPU only+ return built == cap+ if kind == "sm": # a cubin runs on its own major at the same or a newer minor+ return built[0] == cap[0] and built[1] <= cap[1]+ return kind == "compute" and built <= cap # PTX is JIT-compiled forward++missing = [c for c in caps if not any(runs(a, c) for a in archs)]+if not archs or not caps or not missing:+ sys.exit(0)+driver = ctypes.c_int()+cuda = load("libcuda.so.1", "libcuda.so") # cuDriverGetVersion needs no cuInit+if cuda is None or cuda.cuDriverGetVersion(ctypes.byref(driver)) != 0:+ driver.value = 0+family = "cu126" if min(missing) < (7, 5) else ("cu130" if driver.value >= 13000 else "cu128")+status = "none" if len(missing) == len(caps) else "some"+# No wheel to point at (pre-Maxwell, or the family already installed): warn, never fail the install.+if min(missing) < (5, 0) or family == "cu" + torch.version.cuda.replace(".", ""):+ status = "nofix"+fmt = lambda cs: ",".join(f"{a}.{b}" for a, b in cs)+print("UNSLOTH_ARCH_CHECK=%s|%s|%s|%s|%s" % (status, fmt(missing), torch.__version__, " ".join(archs), family))+' 2>/dev/null | sed -n 's/^UNSLOTH_ARCH_CHECK=//p' | tail -n 1 || true)+ if [ -n "$_arch_check" ]; then+ IFS='|' read -r _ac_status _ac_caps _ac_torch _ac_archs _ac_family <<EOF_ARCH+$_arch_check+EOF_ARCH+ if [ -n "${UNSLOTH_PYTORCH_MIRROR:-}" ]; then _ac_pin="UNSLOTH_TORCH_INDEX_FAMILY=$_ac_family"+ else _ac_pin="UNSLOTH_TORCH_INDEX_URL=https://download.pytorch.org/whl/$_ac_family"+ fi+ if [ "$_ac_status" = "none" ] && [ "$_torch_index_pinned" = false ]; then+ tauri_log "ERROR" "PyTorch $_ac_torch has no kernels for this GPU (compute capability $_ac_caps)"+ substep "[ERROR] PyTorch $_ac_torch has no kernels for this GPU (compute capability $_ac_caps)." "$C_ERR"+ substep "[ERROR] It was built for: $_ac_archs" "$C_ERR"+ substep "[ERROR] Training would fail with \"no kernel image is available for execution on the device\"." "$C_ERR"+ substep "[ERROR] Re-run this installer with the matching PyTorch wheels:" "$C_ERR"+ substep "[ERROR] $_ac_pin" "$C_ERR"+ exit 1+ fi+ substep "[WARN] PyTorch $_ac_torch has no kernels for the GPUs with compute capability $_ac_caps." "$C_WARN"+ if [ "$_ac_status" = "nofix" ]; then+ substep "[WARN] It was built for: $_ac_archs. Those GPUs will not be usable for training." "$C_WARN"+ else+ substep "[WARN] It was built for: $_ac_archs. Those GPUs will not be usable; for them, re-run with $_ac_pin" "$C_WARN"+ fi+ fi+ ;;+ esac+fi+# An extras pin lands on a leaf the flavor enforcement above does not recognise, so it skips# the pin, yet whether the build works IS the reason to pin an extra. Ask torch rather than# read the version label: these indexes need not carry a +rocm local tag, whichdiff --git a/studio/setup.ps1 b/studio/setup.ps1index e02316a3382..315143ba843 100644--- a/studio/setup.ps1+++ b/studio/setup.ps1@@ -1301,14 +1301,13 @@ function Get-NvidiaLibraryProbeType {}# "source;cudaMajor;cudaMinor;cap,cap" from NVML, else the CUDA driver API; "" when neither-# answers. Versions are major*1000 + minor*10. Read in a runspace of its own under a deadline:-# a wedged driver can block inside the library, and the deadline leaves that runspace behind.+# answers. Versions are major*1000 + minor*10. One runspace + deadline per reader: a shared one let slow NVML starve CUDA.function Read-NvidiaLibraryRaw {- param([int]$TimeoutMs = 10000)+ param([int]$TimeoutMs = 30000)$type = Get-NvidiaLibraryProbeTypeif (-not $type) { return "" }$reader = {- param($T)+ param($T, $Which)function Read-Nvml {if ($T::nvmlInit_v2() -ne 0) { return "" }try {@@ -1350,26 +1349,30 @@ function Read-NvidiaLibraryRaw {return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"}$r = ""- try { $r = Read-Nvml } catch { $r = "" }- if (-not $r) { try { $r = Read-Cuda } catch { $r = "" } }+ try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }return "$r"}- $ps = $null; $handle = $null- try {- $ps = [powershell]::Create()- $null = $ps.AddScript($reader.ToString()).AddArgument($type)- $handle = $ps.BeginInvoke()- if (-not $handle.AsyncWaitHandle.WaitOne($TimeoutMs)) { return "" }- return "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"- } catch { return "" }- finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }+ foreach ($which in @("nvml", "cuda")) {+ $ps = $null; $handle = $null; $r = ""+ try {+ $ps = [powershell]::Create()+ $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)+ $handle = $ps.BeginInvoke()+ if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {+ $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"+ }+ } catch { $r = "" }+ finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }+ if ($r) { return $r }+ }+ return ""}# NVIDIA inventory from the driver's own libraries (NVML, then the CUDA driver API), for a# host whose nvidia-smi is absent, stale or hangs (#9255). Twin of studio/nvidia_probe.py.# Cached. $null, or @{ Source; CudaMajor; CudaMinor; ComputeCaps ("8.9" strings); Count }.function Get-NvidiaLibraryInventory {- param([int]$TimeoutSec = 10)+ param([int]$TimeoutSec = 30)if ($script:NvidiaLibraryInventoryProbed) { return $script:NvidiaLibraryInventory }$script:NvidiaLibraryInventoryProbed = $true$script:NvidiaLibraryInventory = $nulldiff --git a/tests/sh/test_get_torch_index_url.sh b/tests/sh/test_get_torch_index_url.shindex de4e234c1c6..8764bfa76be 100755--- a/tests/sh/test_get_torch_index_url.sh+++ b/tests/sh/test_get_torch_index_url.sh@@ -57,6 +57,8 @@ _FAKE_ROCM_DIR=$(mktemp -d)echo ""sed -n '/^_nvidia_library_inventory()/,/^}/p' "$INSTALL_SH"echo ""+ sed -n '/^_nvidia_driver_cuda_version()/,/^}/p' "$INSTALL_SH"+ echo ""sed -n '/^_nvidia_cu126_verdict()/,/^}/p' "$INSTALL_SH"echo ""sed -n '/^_cap_cuda_family_for_pre_turing()/,/^}/p' "$INSTALL_SH"@@ -80,6 +82,7 @@ _FAKE_ROCM_DIR=$(mktemp -d)sed -n '/^get_torch_index_url()/,/^}/p' "$INSTALL_SH"} | sed -e "s|/usr/bin/nvidia-smi|$_FAKE_SMI_DIR/nvidia-smi-absent|g" \-e "s|/opt/rocm|$_FAKE_ROCM_DIR|g" \+ -e "s|/proc/driver/nvidia/version|$_FAKE_SMI_DIR/proc-nvidia-version|g" \> "$_FUNC_FILE"for _fn in _rocm_tag_from_amd_smi _rocm_tag_from_version_file _rocm_tag_from_hipconfig \@@ -323,6 +326,56 @@ if [ -n "$_prime" ] && [ -n "$_assign" ] && [ "$_prime" -lt "$_assign" ]; then _assert_eq "presence check primes the inventory before the index substitution" "ordered" "$_result"assert_eq "the prime is skipped for a pinned index or no torch" 'if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ]; then' "$_guard"+# 8i) Only cuDriverGetVersion answers (mock answers -c, not the inventory): Blackwell avoids cu126.+_dir=$(mktemp -d)+cat > "$_dir/nvidia-smi" <<'MOCK'+#!/bin/sh+case "$1" in+ -L) echo "GPU 0: NVIDIA B200 (UUID: GPU-fake-uuid)" ;;+ *) echo "something completely unexpected" ;;+esac+MOCK+chmod +x "$_dir/nvidia-smi"+printf '#!/bin/sh\ncat >/dev/null\n[ "$2" = "-c" ] && echo "13.1" && exit 0\nexit 1\n' > "$_dir/python3"+chmod +x "$_dir/python3"+_result=$(run_func "$_dir")+assert_eq "no inventory, cuDriverGetVersion says 13.1 -> cu130" "https://download.pytorch.org/whl/cu130" "$_result"+_err=$(PATH="$_dir:$_TOOLS_DIR" bash -c "unset CUDA_VISIBLE_DEVICES; _ARCH=x86_64; . '$_FUNC_FILE'; get_torch_index_url" 2>&1 >/dev/null)+case "$_err" in *"Selecting the cu130 PyTorch wheels"*UNSLOTH_TORCH_INDEX_URL=*) _result=warned ;; *) _result="$_err" ;; esac+assert_eq "driver-only selection names the index and the override" "warned" "$_result"+# 8j) No interpreter answers either: the kernel module version in /proc bounds the CUDA version.+printf '#!/bin/sh\ncat >/dev/null\nexit 1\n' > "$_dir/python3"+for _case in "590.48.01 cu130" "575.51.03 cu128" "565.57.01 cu126" "535.183.01 cu124"; do+ printf 'NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 %s Release Build (dvs-builder@U22) Mon Dec 8 13:05:00 UTC 2025\n' "${_case% *}" > "$_FAKE_SMI_DIR/proc-nvidia-version"+ _result=$(run_func "$_dir")+ assert_eq "/proc driver ${_case% *} -> ${_case#* }" "https://download.pytorch.org/whl/${_case#* }" "$_result"+done+# The probe switched off turns the /proc bound off too, like studio/nvidia_probe.py.+_result=$(PATH="$_dir:$_TOOLS_DIR" bash -c "unset CUDA_VISIBLE_DEVICES; _ARCH=x86_64; UNSLOTH_NVIDIA_LIBRARY_PROBE=0; . '$_FUNC_FILE'; get_torch_index_url" 2>/dev/null)+assert_eq "probe off ignores /proc -> cu126 default" "https://download.pytorch.org/whl/cu126" "$_result"+rm -f "$_FAKE_SMI_DIR/proc-nvidia-version"+# 8k) Nothing answers and there is no /proc version: still cu126, now with the override named.+_err=$(PATH="$_dir:$_TOOLS_DIR" bash -c "unset CUDA_VISIBLE_DEVICES; _ARCH=x86_64; . '$_FUNC_FILE'; get_torch_index_url" 2>&1 >/dev/null)+case "$_err" in *"defaulting to cu126"*"UNSLOTH_TORCH_INDEX_URL=https://download.pytorch.org/whl/cu128"*) _result=warned ;; *) _result="$_err" ;; esac+assert_eq "cu126 default names the override" "warned" "$_result"+rm -rf "$_dir"++# 8l) nvidia-smi timing out once is retried with a longer bound. Fake `timeout`: 124 on the first 10s call.+_dir=$(make_mock_smi "13.0" "10.0")+cat > "$_dir/timeout" <<MOCK+#!/bin/sh+if [ "\$1" = 10 ] && [ "\$#" = 2 ] && [ ! -f "$_dir/timed-out" ]; then : > "$_dir/timed-out"; exit 124; fi+echo "\$1" >> "$_dir/bounds"+shift+exec "\$@"+MOCK+chmod +x "$_dir/timeout"+_result=$(run_func "$_dir")+assert_eq "banner timed out once, retry reads 13.0 -> cu130" "https://download.pytorch.org/whl/cu130" "$_result"+case "$(tr '\n' ' ' < "$_dir/bounds")" in *45*) _result=retried ;; *) _result="bounds: $(cat "$_dir/bounds")" ;; esac+assert_eq "the retry uses the 45s bound" "retried" "$_result"+rm -rf "$_dir"+# 9) ROCm 6.3 (no nvidia-smi) -> rocm6.3_dir=$(make_mock_amd_smi "6.3")_result=$(run_func "$_dir")diff --git a/tests/studio/install/test_install_sh_nvidia_inventory_reader.py b/tests/studio/install/test_install_sh_nvidia_inventory_reader.pyindex 21492e4d7fb..b67e0473d02 100644--- a/tests/studio/install/test_install_sh_nvidia_inventory_reader.py+++ b/tests/studio/install/test_install_sh_nvidia_inventory_reader.py@@ -60,10 +60,17 @@ class _Attr:return 0-def _run(monkeypatch, libraries: dict[str, object]) -> tuple[int, str]:- """Run the reader with ctypes.CDLL answering from `libraries`; (exit code, stdout)."""+def _run(+ monkeypatch,+ libraries: dict[str, object],+ reader: str = "",+) -> tuple[int, str]:+ """Run the reader with ctypes.CDLL answering from `libraries`; (exit code, stdout).+ `reader` is the argument install.sh passes to run one reader under its own deadline."""import ctypes as real_ctypes+ monkeypatch.setattr(sys, "argv", ["-", reader] if reader else ["-"])+def cdll(name):if name in libraries:return libraries[name]@@ -133,3 +140,32 @@ def test_nvml_answers_first_when_it_can(monkeypatch):)code, out = _run(monkeypatch, {"libnvidia-ml.so.1": nvml, "libcuda.so.1": _cuda_lib()})assert (code, out) == (0, "12.8 6.1,6.1")+++def _nvml_lib():+ return _lib(+ nvmlInit_v2 = _Ok(),+ nvmlShutdown = _Ok(),+ nvmlDeviceGetCount_v2 = _Ok(1),+ nvmlSystemGetCudaDriverVersion_v2 = _Ok(12080),+ nvmlDeviceGetHandleByIndex_v2 = _Ok(1),+ nvmlDeviceGetCudaComputeCapability = _Ok(10, 0),+ )+++def test_each_reader_runs_alone_under_its_own_deadline(monkeypatch):+ """install.sh runs NVML and the CUDA driver API as separate bounded processes, so a slow+ NVML no longer uses up the deadline the driver API needed (8x B200: NVML took ~23s)."""+ libs = {"libnvidia-ml.so.1": _nvml_lib(), "libcuda.so.1": _cuda_lib(cap = (10, 0))}+ assert _run(monkeypatch, libs, "nvml") == (0, "12.8 10.0")+ assert _run(monkeypatch, libs, "cuda") == (0, "13.1 10.0")+ assert _run(monkeypatch, {"libcuda.so.1": _cuda_lib()}, "nvml") == (1, "")+ assert _run(monkeypatch, {"libnvidia-ml.so.1": _nvml_lib()}, "cuda") == (1, "")+++def test_install_sh_gives_each_reader_its_own_bound():+ text = INSTALL_SH.read_text(encoding = "utf-8")+ body = text[text.index("_nvidia_library_inventory() {") :]+ body = body[: body.index("\n}\n")]+ assert "for _nli_reader in nvml cuda; do" in body+ assert '_run_bounded --secs "$_nli_secs" "$_nli_py" -I - "$_nli_reader"' in bodydiff --git a/tests/studio/install/test_install_sh_torch_arch_check.py b/tests/studio/install/test_install_sh_torch_arch_check.pynew file mode 100644index 00000000000..96637361b44--- /dev/null+++ b/tests/studio/install/test_install_sh_torch_arch_check.py@@ -0,0 +1,113 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+"""install.sh's post-install check that the CUDA torch has kernels for the GPUs, run against a+stub torch with the driver libraries hidden."""++from __future__ import annotations++import os+import re+import subprocess+import sys+from pathlib import Path++import pytest++INSTALL_SH = Path(__file__).resolve().parents[3] / "install.sh"++CU126 = "sm_50 sm_60 sm_70 sm_75 sm_80 sm_86 sm_90"+CU130 = "sm_75 sm_80 sm_86 sm_90 sm_100 sm_120"++_SITECUSTOMIZE = """+import ctypes+_real = ctypes.CDLL+class _NoDriver(ctypes.CDLL):+ def __init__(self, name, *a, **k):+ if name and ("nvidia-ml" in name or "libcuda" in name):+ raise OSError(name)+ super().__init__(name, *a, **k)+ctypes.CDLL = _NoDriver+"""++_TORCH = """+class version:+ cuda = {cuda!r}+ hip = None+__version__ = {ver!r}+class _C:+ @staticmethod+ def _cuda_getArchFlags():+ return {archs!r}+class cuda:+ @staticmethod+ def is_available():+ return True+ @staticmethod+ def device_count():+ return {n}+ @staticmethod+ def get_device_capability(i):+ return {caps!r}[i]+"""+++def _check_source() -> str:+ text = INSTALL_SH.read_text(encoding = "utf-8")+ m = re.search(r"_run_bounded --secs 120 \"\$_VENV_PY\" -c '\n(.*?)\n' 2>/dev/null", text, re.S)+ assert m, "the post-install arch check was not found"+ return m.group(1)+++def _run(tmp_path, cuda, archs, caps):+ stub = tmp_path / "stub"+ (stub / "torch").mkdir(parents = True)+ (stub / "sitecustomize.py").write_text(_SITECUSTOMIZE)+ (stub / "torch" / "__init__.py").write_text(+ _TORCH.format(+ cuda = cuda, ver = f"2.11.0+cu{cuda.replace('.', '')}", archs = archs, n = len(caps), caps = caps+ )+ )+ out = subprocess.run(+ [sys.executable, "-c", _check_source()],+ env = {**os.environ, "PYTHONPATH": str(stub)},+ capture_output = True,+ text = True,+ timeout = 60,+ )+ assert out.returncode == 0, out.stderr+ line = [l for l in out.stdout.splitlines() if l.startswith("UNSLOTH_ARCH_CHECK=")]+ return line[-1].split("=", 1)[1].split("|") if line else None+++ "cuda, archs, caps, expected",+ [+ ("12.6", CU126, [(10, 0)], ("none", "10.0", "cu128")),+ ("12.6", CU126, [(6, 1), (10, 0)], ("some", "10.0", "cu128")),+ ("13.0", CU130, [(6, 1)], ("none", "6.1", "cu126")),+ # No wheel family serves Kepler, and re-installing the same family fixes nothing.+ ("11.8", CU126, [(3, 7)], ("nofix", "3.7", "cu126")),+ ("12.8", CU130, [(13, 0)], ("nofix", "13.0", "cu128")),+ ],+)+def test_uncovered_gpus(tmp_path, cuda, archs, caps, expected):+ status, missing, _torch, _archs, family = _run(tmp_path, cuda, archs, caps)+ assert (status, missing, family) == expected+++ "archs, caps",+ [+ (CU130, [(10, 3)]), # sm_100 cubin runs on 10.3+ (CU130, [(12, 1)]),+ ("sm_80 sm_90a compute_90", [(9, 0)]),+ ("sm_80 compute_80", [(8, 9)]), # PTX JIT forward+ ],+)+def test_covered_gpus_pass_silently(tmp_path, archs, caps):+ assert _run(tmp_path, "13.0", archs, caps) is None+++def test_nofix_status_only_warns():+ text = INSTALL_SH.read_text(encoding = "utf-8")+ assert '[ "$_ac_status" = "none" ] && [ "$_torch_index_pinned" = false ]' in textdiff --git a/tests/studio/install/test_probe_timeouts.py b/tests/studio/install/test_probe_timeouts.pyindex 4c6acbe87de..859c05c6de5 100644--- a/tests/studio/install/test_probe_timeouts.py+++ b/tests/studio/install/test_probe_timeouts.py@@ -57,7 +57,8 @@ class TestInstallShBoundedProbe:assert ("command -v timeout" in body), "_run_bounded must check for the `timeout` binary before using it"- assert "timeout 10" in body, "_run_bounded must apply a 10s timeout"+ assert "_rb_secs=10" in body, "_run_bounded must apply a 10s timeout by default"+ assert 'timeout "$_rb_secs"' in body, "_run_bounded must pass its bound to timeout"# Falls back to unbounded when `timeout` is absent (e.g. macOS), keeping semantics there.assert ("else" in body and '"$@"' in bodydiff --git a/tests/studio/test_nvidia_library_inventory.ps1 b/tests/studio/test_nvidia_library_inventory.ps1index c7b6789a8fb..a3bc4b8a277 100644--- a/tests/studio/test_nvidia_library_inventory.ps1+++ b/tests/studio/test_nvidia_library_inventory.ps1@@ -43,8 +43,17 @@ $script:FakeIntelAdapters = @()function Get-IntelRegistryAdapterNames { return $script:FakeIntelAdapters }# nvidia-smi stand-in: $script:FakeSmiStdout answers every probe, $script:FakeSmiRc is its exit code.+# $script:FakeSmiTimeouts calls time out first (exit 124); every bound asked for is recorded.+$script:FakeSmiTimeouts = 0+$script:FakeSmiBounds = @()function Invoke-NvidiaSmiBounded {param([string]$Exe, [string[]]$SmiArgs = @(), [int]$TimeoutSec = 10, [switch]$StdoutOnly)+ $script:FakeSmiBounds += $TimeoutSec+ if ($script:FakeSmiTimeouts -gt 0) {+ $script:FakeSmiTimeouts--+ $global:LASTEXITCODE = 124+ return ""+ }$global:LASTEXITCODE = $script:FakeSmiRcreturn $script:FakeSmiStdout}@@ -279,6 +288,14 @@ Check "the cap applies to inventory capabilities" ((Get-TorchIndexUrl) -eq "http$script:FakeInventory = $null$script:Substeps = @()Check "an unreadable banner with no inventory keeps the cu126 default" ((Get-TorchIndexUrl) -eq "https://download.pytorch.org/whl/cu126")+Check "the cu126 default names the override" (@($script:Substeps | Where-Object { $_ -match 'UNSLOTH_TORCH_INDEX_URL=https://download.pytorch.org/whl/cu128' }).Count -eq 1)+# A congested driver: nvidia-smi times out once, and the longer retry reads the banner.+$script:FakeSmiStdout = "| CUDA Version: 13.1 |"+$script:FakeSmiTimeouts = 1+$script:FakeSmiBounds = @()+Check "a timed-out banner is retried and read" ((Get-TorchIndexUrl) -eq "https://download.pytorch.org/whl/cu130")+Check "the retry has the longer bound" (($script:FakeSmiBounds[0..1] -join ",") -eq "10,45")+$script:FakeSmiTimeouts = 0$script:FakeSmiStdout = "| CUDA Version: 12.6 |"Check "a readable banner is still authoritative" ((Get-TorchIndexUrl) -eq "https://download.pytorch.org/whl/cu126")