Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Installer: stop picking cu126 when a slow NVIDIA driver times out CUDA detection (#11916)

Daniel Han5665b5a99fbf
diff --git a/install.ps1 b/install.ps1
index c90a1deab54..e13da1e807f 100644
--- a/install.ps1
+++ b/install.ps1
@@ -8260,14 +8260,13 @@ exit 0
}
# "source;cudaMajor;cudaMinor;cap,cap" from NVML, else the CUDA driver API; "" when neither
- # answers. Versions are major*1000 + minor*10. Read in a runspace of its own under a deadline:
- # a wedged driver can block inside the library, and the deadline leaves that runspace behind.
+ # answers. Versions are major*1000 + minor*10. One runspace + deadline per reader: a shared one let slow NVML starve CUDA.
function Read-NvidiaLibraryRaw {
- param([int]$TimeoutMs = 10000)
+ param([int]$TimeoutMs = 30000)
$type = Get-NvidiaLibraryProbeType
if (-not $type) { return "" }
$reader = {
- param($T)
+ param($T, $Which)
function Read-Nvml {
if ($T::nvmlInit_v2() -ne 0) { return "" }
try {
@@ -8309,26 +8308,30 @@ exit 0
return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
}
$r = ""
- try { $r = Read-Nvml } catch { $r = "" }
- if (-not $r) { try { $r = Read-Cuda } catch { $r = "" } }
+ try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }
return "$r"
}
- $ps = $null; $handle = $null
- try {
- $ps = [powershell]::Create()
- $null = $ps.AddScript($reader.ToString()).AddArgument($type)
- $handle = $ps.BeginInvoke()
- if (-not $handle.AsyncWaitHandle.WaitOne($TimeoutMs)) { return "" }
- return "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"
- } catch { return "" }
- finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }
+ foreach ($which in @("nvml", "cuda")) {
+ $ps = $null; $handle = $null; $r = ""
+ try {
+ $ps = [powershell]::Create()
+ $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)
+ $handle = $ps.BeginInvoke()
+ if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {
+ $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"
+ }
+ } catch { $r = "" }
+ finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }
+ if ($r) { return $r }
+ }
+ return ""
}
# NVIDIA inventory from the driver's own libraries (NVML, then the CUDA driver API), for a
# host whose nvidia-smi is absent, stale or hangs (#9255). Twin of studio/nvidia_probe.py.
# Cached. $null, or @{ Source; CudaMajor; CudaMinor; ComputeCaps ("8.9" strings); Count }.
function Get-NvidiaLibraryInventory {
- param([int]$TimeoutSec = 10)
+ param([int]$TimeoutSec = 30)
if ($script:NvidiaLibraryInventoryProbed) { return $script:NvidiaLibraryInventory }
$script:NvidiaLibraryInventoryProbed = $true
$script:NvidiaLibraryInventory = $null
@@ -9262,6 +9265,10 @@ exit 0
if ($NvidiaSmiExe) {
try {
$output = Invoke-NvidiaSmiBounded $NvidiaSmiExe
+ if ($LASTEXITCODE -eq 124) {
+ substep "nvidia-smi did not answer within 10s; retrying with a 45s limit..." "Yellow"
+ $output = Invoke-NvidiaSmiBounded $NvidiaSmiExe -TimeoutSec 45
+ }
if ($output -match 'CUDA(?: UMD)? Version:\s+(\d+)\.(\d+)') {
$major = [int]$Matches[1]; $minor = [int]$Matches[2]
}
@@ -9276,6 +9283,9 @@ exit 0
return "$baseUrl/cpu"
} else {
substep "could not determine CUDA version from nvidia-smi, defaulting to cu126" "Yellow"
+ $pinHint = if ($env:UNSLOTH_PYTORCH_MIRROR) { "UNSLOTH_TORCH_INDEX_FAMILY=" } else { "UNSLOTH_TORCH_INDEX_URL=$baseUrl/" }
+ substep "cu126 has no kernels for Blackwell (sm_100 / sm_120). To choose the wheel yourself, re-run with" "Yellow"
+ substep " ${pinHint}cu128 (or cu130 on a driver that supports CUDA 13)" "Yellow"
return "$baseUrl/cu126"
}
}
diff --git a/install.sh b/install.sh
index fab0e9bf5ca..ee3ca5df0e5 100755
--- a/install.sh
+++ b/install.sh
@@ -2644,8 +2644,10 @@ _wsl_amd_gpu_name() {
# ── Bounded command runner ──
_run_bounded() {
+ _rb_secs=10
+ if [ "${1:-}" = "--secs" ]; then _rb_secs=$2; shift 2; fi
if command -v timeout >/dev/null 2>&1; then
- timeout 10 "$@"
+ timeout "$_rb_secs" "$@"
else
"$@"
fi
@@ -2676,7 +2678,11 @@ _nvidia_library_inventory() {
else return 1
fi
_NVIDIA_LIBRARY_INVENTORY_STATE="none"
- _NVIDIA_LIBRARY_INVENTORY_VALUE=$(_run_bounded "$_nli_py" -I - 2>/dev/null <<'PY'
+ _NVIDIA_LIBRARY_INVENTORY_VALUE=""
+ # One deadline per reader: a shared bound let slow NVML starve the CUDA driver API reader.
+ for _nli_reader in nvml cuda; do
+ case "$_nli_reader" in nvml) _nli_secs=30 ;; *) _nli_secs=20 ;; esac
+ _NVIDIA_LIBRARY_INVENTORY_VALUE=$(_run_bounded --secs "$_nli_secs" "$_nli_py" -I - "$_nli_reader" 2>/dev/null <<'PY'
import ctypes, os, sys
def load(*names):
@@ -2733,8 +2739,10 @@ def cuda():
caps.append(f"{major.value}.{minor.value}")
return version.value, caps
+# argv[1] runs one reader ("nvml" / "cuda") so each gets its own deadline; none runs both.
+only = {"nvml": (nvml,), "cuda": (cuda,)}.get(sys.argv[1] if len(sys.argv) > 1 else "")
found = None
-for reader in (nvml, cuda): # a reader that raises (a missing symbol) yields to the next
+for reader in only or (nvml, cuda): # a reader that raises (a missing symbol) yields to the next
try:
found = reader()
except Exception:
@@ -2746,11 +2754,52 @@ if not found or not found[1]:
version, caps = found
print(f"{version // 1000}.{version % 1000 // 10} {','.join(caps)}")
PY
-) && [ -n "$_NVIDIA_LIBRARY_INVENTORY_VALUE" ] || return 1
+) && [ -n "$_NVIDIA_LIBRARY_INVENTORY_VALUE" ] && break
+ _NVIDIA_LIBRARY_INVENTORY_VALUE=""
+ done
+ [ -n "$_NVIDIA_LIBRARY_INVENTORY_VALUE" ] || return 1
_NVIDIA_LIBRARY_INVENTORY_STATE="found"
printf '%s\n' "$_NVIDIA_LIBRARY_INVENTORY_VALUE"
}
+# Driver CUDA version without cuInit (cuDriverGetVersion, else /proc as in nvidia_probe.py _DRIVER_MAJOR_CUDA); picks a family only.
+_nvidia_driver_cuda_version() {
+ [ "${UNSLOTH_NVIDIA_LIBRARY_PROBE:-1}" != "0" ] || return 1
+ if command -v python3 >/dev/null 2>&1; then _ndv_py=python3
+ elif [ -n "${VENV_DIR:-}" ] && [ -x "$VENV_DIR/bin/python" ]; then _ndv_py="$VENV_DIR/bin/python"
+ else _ndv_py=""
+ fi
+ if [ -n "$_ndv_py" ]; then
+ _ndv_ver=$(_run_bounded "$_ndv_py" -I -c '
+import ctypes, sys
+for name in ("libcuda.so.1", "libcuda.so"):
+ try:
+ lib = ctypes.CDLL(name)
+ break
+ except OSError:
+ lib = None
+v = ctypes.c_int()
+if lib is None or lib.cuDriverGetVersion(ctypes.byref(v)) != 0 or v.value < 1000:
+ sys.exit(1)
+print(f"{v.value // 1000}.{v.value % 1000 // 10}")
+' 2>/dev/null </dev/null | awk 'NR == 1 { print $1 }') || _ndv_ver=""
+ case "$_ndv_ver" in
+ [0-9]*.[0-9]*) printf '%s\n' "$_ndv_ver"; return 0 ;;
+ esac
+ fi
+ # "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 590.48.01 Release Build ..."
+ _ndv_drv=$(awk 'NR == 1 { for (i = 1; i <= NF; i++) if ($i ~ /^[0-9]+\.[0-9]+(\.[0-9]+)?$/) { split($i, v, "."); print v[1]; exit } }' \
+ /proc/driver/nvidia/version 2>/dev/null) || _ndv_drv=""
+ case "$_ndv_drv" in ''|*[!0-9]*) return 1 ;; esac
+ if [ "$_ndv_drv" -ge 580 ]; then echo "13.0"
+ elif [ "$_ndv_drv" -ge 570 ]; then echo "12.8"
+ elif [ "$_ndv_drv" -ge 560 ]; then echo "12.6"
+ elif [ "$_ndv_drv" -ge 525 ]; then echo "12.0"
+ elif [ "$_ndv_drv" -ge 450 ]; then echo "11.0"
+ else return 1
+ fi
+}
+
# ── NVIDIA usable-GPU helper ──
# nvidia-smi -L primary, /proc/driver/nvidia/gpus/ fallback, driver library last; a hidden GPU is NOT usable.
_has_usable_nvidia_gpu() {
@@ -5235,19 +5284,37 @@ get_torch_index_url() {
echo "$_base/cpu"; return
fi
# CUDA version from nvidia-smi: accept "CUDA Version:" and the newer "CUDA UMD Version:".
- _cuda_ver=$(export LC_ALL=C; _run_bounded "$_smi" 2>/dev/null \
+ _smi_rc=0
+ _smi_out=$(export LC_ALL=C; _run_bounded "$_smi" 2>/dev/null) || _smi_rc=$?
+ if [ "$_smi_rc" = "124" ]; then
+ echo "[INFO] nvidia-smi did not answer within 10s; retrying with a 45s limit..." >&2
+ _smi_rc=0
+ _smi_out=$(export LC_ALL=C; _run_bounded --secs 45 "$_smi" 2>/dev/null) || _smi_rc=$?
+ # Still hung: do not spend another bound asking it for compute capabilities below.
+ [ "$_smi_rc" = "124" ] && _smi=""
+ fi
+ _cuda_ver=$(printf '%s\n' "$_smi_out" \
| sed -n \
-e 's/.*CUDA UMD Version:[[:space:]]*\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' \
-e 's/.*CUDA Version:[[:space:]]*\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' \
| head -1)
_inventory_caps=""
+ _cuda_from_driver=""
+ # A mirror base can carry credentials, so name only the leaf.
+ if [ -n "${UNSLOTH_PYTORCH_MIRROR:-}" ]; then _pin_hint="UNSLOTH_TORCH_INDEX_FAMILY="
+ else _pin_hint="UNSLOTH_TORCH_INDEX_URL=$_base/"
+ fi
if [ -z "$_cuda_ver" ]; then
# nvidia-smi absent, stale or hung: the driver library knows both; cu126 is the last resort.
if _inventory=$(_nvidia_library_inventory) && [ -n "$_inventory" ]; then
_cuda_ver=${_inventory%% *}
_inventory_caps=$(printf '%s' "${_inventory#* }" | tr ',' '\n')
+ elif _cuda_ver=$(_nvidia_driver_cuda_version) && [ -n "$_cuda_ver" ]; then
+ _cuda_from_driver=1
else
echo "[WARN] Could not determine CUDA version from nvidia-smi, defaulting to cu126" >&2
+ echo "[WARN] cu126 has no kernels for Blackwell (sm_100 / sm_120). To choose the wheel yourself, re-run with" >&2
+ echo "[WARN] ${_pin_hint}cu128 (or cu130 on a driver that supports CUDA 13)" >&2
echo "$_base/cu126"; return
fi
fi
@@ -5259,7 +5326,14 @@ get_torch_index_url() {
elif [ "$_major" -ge 12 ]; then _cuda_tag=cu124
elif [ "$_major" -ge 11 ]; then _cuda_tag=cu118
else echo "$_base/cpu"; return; fi
- echo "$_base/$(_cap_cuda_family_for_pre_turing "$_cuda_tag" "$_smi" "$_inventory_caps")"
+ _cuda_tag=$(_cap_cuda_family_for_pre_turing "$_cuda_tag" "$_smi" "$_inventory_caps")
+ if [ -n "$_cuda_from_driver" ]; then
+ echo "[WARN] nvidia-smi and the NVIDIA driver libraries did not answer in time; the driver supports CUDA $_cuda_ver." >&2
+ echo "[WARN] Selecting the $_cuda_tag PyTorch wheels from the driver version alone. If that is wrong for this GPU, re-run with" >&2
+ echo "[WARN] ${_pin_hint}cu126 (Maxwell to Hopper, sm_50-90)" >&2
+ echo "[WARN] ${_pin_hint}cu128 (Turing and newer, including Blackwell)" >&2
+ fi
+ echo "$_base/$_cuda_tag"
}
# ── Torch flavor helpers (to repair a stale CPU / wrong-CUDA wheel) ──
@@ -7371,6 +7445,112 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then
fi
fi
+# A wrong CUDA family fails only at first kernel launch; never cuInit here (minutes on a congested driver).
+if [ "$SKIP_TORCH" = false ] && ! _cvd_hides_nvidia; then
+ case "${_expected_torch_tag:-}" in
+ cu[0-9]*)
+ _arch_check=$(_run_bounded --secs 120 "$_VENV_PY" -c '
+import ctypes, sys
+
+def load(*names):
+ for name in names:
+ try:
+ return ctypes.CDLL(name)
+ except OSError:
+ pass
+
+def nvml_caps():
+ lib = load("libnvidia-ml.so.1", "libnvidia-ml.so")
+ if lib is None or lib.nvmlInit_v2() != 0:
+ return None
+ try:
+ count, caps = ctypes.c_uint(), set()
+ if lib.nvmlDeviceGetCount_v2(ctypes.byref(count)) != 0 or not count.value:
+ return None
+ for i in range(count.value):
+ dev, major, minor = ctypes.c_void_p(), ctypes.c_int(), ctypes.c_int()
+ if lib.nvmlDeviceGetHandleByIndex_v2(i, ctypes.byref(dev)) != 0 or \
+ lib.nvmlDeviceGetCudaComputeCapability(dev, ctypes.byref(major), ctypes.byref(minor)) != 0:
+ return None
+ caps.add((major.value, minor.value))
+ return sorted(caps)
+ finally:
+ lib.nvmlShutdown()
+
+try:
+ import torch
+ if not torch.version.cuda or getattr(torch.version, "hip", None):
+ sys.exit(0)
+ try:
+ archs = torch._C._cuda_getArchFlags().split()
+ except Exception:
+ archs = torch.cuda.get_arch_list()
+ try:
+ caps = nvml_caps()
+ except Exception:
+ caps = None
+ if caps is None:
+ if not torch.cuda.is_available():
+ sys.exit(0)
+ caps = sorted({torch.cuda.get_device_capability(i) for i in range(torch.cuda.device_count())})
+except Exception:
+ sys.exit(0)
+
+def runs(arch, cap):
+ kind, _, rest = arch.partition("_")
+ n = len(rest) - len(rest.lstrip("0123456789"))
+ digits, suffix = rest[:n], rest[n:]
+ if n < 2:
+ return False
+ built = (int(digits[:-1]), int(digits[-1]))
+ if suffix == "a": # arch-specific cubin / PTX: that exact GPU only
+ return built == cap
+ if kind == "sm": # a cubin runs on its own major at the same or a newer minor
+ return built[0] == cap[0] and built[1] <= cap[1]
+ return kind == "compute" and built <= cap # PTX is JIT-compiled forward
+
+missing = [c for c in caps if not any(runs(a, c) for a in archs)]
+if not archs or not caps or not missing:
+ sys.exit(0)
+driver = ctypes.c_int()
+cuda = load("libcuda.so.1", "libcuda.so") # cuDriverGetVersion needs no cuInit
+if cuda is None or cuda.cuDriverGetVersion(ctypes.byref(driver)) != 0:
+ driver.value = 0
+family = "cu126" if min(missing) < (7, 5) else ("cu130" if driver.value >= 13000 else "cu128")
+status = "none" if len(missing) == len(caps) else "some"
+# No wheel to point at (pre-Maxwell, or the family already installed): warn, never fail the install.
+if min(missing) < (5, 0) or family == "cu" + torch.version.cuda.replace(".", ""):
+ status = "nofix"
+fmt = lambda cs: ",".join(f"{a}.{b}" for a, b in cs)
+print("UNSLOTH_ARCH_CHECK=%s|%s|%s|%s|%s" % (status, fmt(missing), torch.__version__, " ".join(archs), family))
+' 2>/dev/null | sed -n 's/^UNSLOTH_ARCH_CHECK=//p' | tail -n 1 || true)
+ if [ -n "$_arch_check" ]; then
+ IFS='|' read -r _ac_status _ac_caps _ac_torch _ac_archs _ac_family <<EOF_ARCH
+$_arch_check
+EOF_ARCH
+ if [ -n "${UNSLOTH_PYTORCH_MIRROR:-}" ]; then _ac_pin="UNSLOTH_TORCH_INDEX_FAMILY=$_ac_family"
+ else _ac_pin="UNSLOTH_TORCH_INDEX_URL=https://download.pytorch.org/whl/$_ac_family"
+ fi
+ if [ "$_ac_status" = "none" ] && [ "$_torch_index_pinned" = false ]; then
+ tauri_log "ERROR" "PyTorch $_ac_torch has no kernels for this GPU (compute capability $_ac_caps)"
+ substep "[ERROR] PyTorch $_ac_torch has no kernels for this GPU (compute capability $_ac_caps)." "$C_ERR"
+ substep "[ERROR] It was built for: $_ac_archs" "$C_ERR"
+ substep "[ERROR] Training would fail with \"no kernel image is available for execution on the device\"." "$C_ERR"
+ substep "[ERROR] Re-run this installer with the matching PyTorch wheels:" "$C_ERR"
+ substep "[ERROR] $_ac_pin" "$C_ERR"
+ exit 1
+ fi
+ substep "[WARN] PyTorch $_ac_torch has no kernels for the GPUs with compute capability $_ac_caps." "$C_WARN"
+ if [ "$_ac_status" = "nofix" ]; then
+ substep "[WARN] It was built for: $_ac_archs. Those GPUs will not be usable for training." "$C_WARN"
+ else
+ substep "[WARN] It was built for: $_ac_archs. Those GPUs will not be usable; for them, re-run with $_ac_pin" "$C_WARN"
+ fi
+ fi
+ ;;
+ esac
+fi
+
# An extras pin lands on a leaf the flavor enforcement above does not recognise, so it skips
# the pin, yet whether the build works IS the reason to pin an extra. Ask torch rather than
# read the version label: these indexes need not carry a +rocm local tag, which
diff --git a/studio/setup.ps1 b/studio/setup.ps1
index e02316a3382..315143ba843 100644
--- a/studio/setup.ps1
+++ b/studio/setup.ps1
@@ -1301,14 +1301,13 @@ function Get-NvidiaLibraryProbeType {
}
# "source;cudaMajor;cudaMinor;cap,cap" from NVML, else the CUDA driver API; "" when neither
-# answers. Versions are major*1000 + minor*10. Read in a runspace of its own under a deadline:
-# a wedged driver can block inside the library, and the deadline leaves that runspace behind.
+# answers. Versions are major*1000 + minor*10. One runspace + deadline per reader: a shared one let slow NVML starve CUDA.
function Read-NvidiaLibraryRaw {
- param([int]$TimeoutMs = 10000)
+ param([int]$TimeoutMs = 30000)
$type = Get-NvidiaLibraryProbeType
if (-not $type) { return "" }
$reader = {
- param($T)
+ param($T, $Which)
function Read-Nvml {
if ($T::nvmlInit_v2() -ne 0) { return "" }
try {
@@ -1350,26 +1349,30 @@ function Read-NvidiaLibraryRaw {
return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
}
$r = ""
- try { $r = Read-Nvml } catch { $r = "" }
- if (-not $r) { try { $r = Read-Cuda } catch { $r = "" } }
+ try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }
return "$r"
}
- $ps = $null; $handle = $null
- try {
- $ps = [powershell]::Create()
- $null = $ps.AddScript($reader.ToString()).AddArgument($type)
- $handle = $ps.BeginInvoke()
- if (-not $handle.AsyncWaitHandle.WaitOne($TimeoutMs)) { return "" }
- return "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"
- } catch { return "" }
- finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }
+ foreach ($which in @("nvml", "cuda")) {
+ $ps = $null; $handle = $null; $r = ""
+ try {
+ $ps = [powershell]::Create()
+ $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)
+ $handle = $ps.BeginInvoke()
+ if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {
+ $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"
+ }
+ } catch { $r = "" }
+ finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }
+ if ($r) { return $r }
+ }
+ return ""
}
# NVIDIA inventory from the driver's own libraries (NVML, then the CUDA driver API), for a
# host whose nvidia-smi is absent, stale or hangs (#9255). Twin of studio/nvidia_probe.py.
# Cached. $null, or @{ Source; CudaMajor; CudaMinor; ComputeCaps ("8.9" strings); Count }.
function Get-NvidiaLibraryInventory {
- param([int]$TimeoutSec = 10)
+ param([int]$TimeoutSec = 30)
if ($script:NvidiaLibraryInventoryProbed) { return $script:NvidiaLibraryInventory }
$script:NvidiaLibraryInventoryProbed = $true
$script:NvidiaLibraryInventory = $null
diff --git a/tests/sh/test_get_torch_index_url.sh b/tests/sh/test_get_torch_index_url.sh
index de4e234c1c6..8764bfa76be 100755
--- a/tests/sh/test_get_torch_index_url.sh
+++ b/tests/sh/test_get_torch_index_url.sh
@@ -57,6 +57,8 @@ _FAKE_ROCM_DIR=$(mktemp -d)
echo ""
sed -n '/^_nvidia_library_inventory()/,/^}/p' "$INSTALL_SH"
echo ""
+ sed -n '/^_nvidia_driver_cuda_version()/,/^}/p' "$INSTALL_SH"
+ echo ""
sed -n '/^_nvidia_cu126_verdict()/,/^}/p' "$INSTALL_SH"
echo ""
sed -n '/^_cap_cuda_family_for_pre_turing()/,/^}/p' "$INSTALL_SH"
@@ -80,6 +82,7 @@ _FAKE_ROCM_DIR=$(mktemp -d)
sed -n '/^get_torch_index_url()/,/^}/p' "$INSTALL_SH"
} | sed -e "s|/usr/bin/nvidia-smi|$_FAKE_SMI_DIR/nvidia-smi-absent|g" \
-e "s|/opt/rocm|$_FAKE_ROCM_DIR|g" \
+ -e "s|/proc/driver/nvidia/version|$_FAKE_SMI_DIR/proc-nvidia-version|g" \
> "$_FUNC_FILE"
for _fn in _rocm_tag_from_amd_smi _rocm_tag_from_version_file _rocm_tag_from_hipconfig \
@@ -323,6 +326,56 @@ if [ -n "$_prime" ] && [ -n "$_assign" ] && [ "$_prime" -lt "$_assign" ]; then _
assert_eq "presence check primes the inventory before the index substitution" "ordered" "$_result"
assert_eq "the prime is skipped for a pinned index or no torch" 'if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ]; then' "$_guard"
+# 8i) Only cuDriverGetVersion answers (mock answers -c, not the inventory): Blackwell avoids cu126.
+_dir=$(mktemp -d)
+cat > "$_dir/nvidia-smi" <<'MOCK'
+#!/bin/sh
+case "$1" in
+ -L) echo "GPU 0: NVIDIA B200 (UUID: GPU-fake-uuid)" ;;
+ *) echo "something completely unexpected" ;;
+esac
+MOCK
+chmod +x "$_dir/nvidia-smi"
+printf '#!/bin/sh\ncat >/dev/null\n[ "$2" = "-c" ] && echo "13.1" && exit 0\nexit 1\n' > "$_dir/python3"
+chmod +x "$_dir/python3"
+_result=$(run_func "$_dir")
+assert_eq "no inventory, cuDriverGetVersion says 13.1 -> cu130" "https://download.pytorch.org/whl/cu130" "$_result"
+_err=$(PATH="$_dir:$_TOOLS_DIR" bash -c "unset CUDA_VISIBLE_DEVICES; _ARCH=x86_64; . '$_FUNC_FILE'; get_torch_index_url" 2>&1 >/dev/null)
+case "$_err" in *"Selecting the cu130 PyTorch wheels"*UNSLOTH_TORCH_INDEX_URL=*) _result=warned ;; *) _result="$_err" ;; esac
+assert_eq "driver-only selection names the index and the override" "warned" "$_result"
+# 8j) No interpreter answers either: the kernel module version in /proc bounds the CUDA version.
+printf '#!/bin/sh\ncat >/dev/null\nexit 1\n' > "$_dir/python3"
+for _case in "590.48.01 cu130" "575.51.03 cu128" "565.57.01 cu126" "535.183.01 cu124"; do
+ printf 'NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 %s Release Build (dvs-builder@U22) Mon Dec 8 13:05:00 UTC 2025\n' "${_case% *}" > "$_FAKE_SMI_DIR/proc-nvidia-version"
+ _result=$(run_func "$_dir")
+ assert_eq "/proc driver ${_case% *} -> ${_case#* }" "https://download.pytorch.org/whl/${_case#* }" "$_result"
+done
+# The probe switched off turns the /proc bound off too, like studio/nvidia_probe.py.
+_result=$(PATH="$_dir:$_TOOLS_DIR" bash -c "unset CUDA_VISIBLE_DEVICES; _ARCH=x86_64; UNSLOTH_NVIDIA_LIBRARY_PROBE=0; . '$_FUNC_FILE'; get_torch_index_url" 2>/dev/null)
+assert_eq "probe off ignores /proc -> cu126 default" "https://download.pytorch.org/whl/cu126" "$_result"
+rm -f "$_FAKE_SMI_DIR/proc-nvidia-version"
+# 8k) Nothing answers and there is no /proc version: still cu126, now with the override named.
+_err=$(PATH="$_dir:$_TOOLS_DIR" bash -c "unset CUDA_VISIBLE_DEVICES; _ARCH=x86_64; . '$_FUNC_FILE'; get_torch_index_url" 2>&1 >/dev/null)
+case "$_err" in *"defaulting to cu126"*"UNSLOTH_TORCH_INDEX_URL=https://download.pytorch.org/whl/cu128"*) _result=warned ;; *) _result="$_err" ;; esac
+assert_eq "cu126 default names the override" "warned" "$_result"
+rm -rf "$_dir"
+
+# 8l) nvidia-smi timing out once is retried with a longer bound. Fake `timeout`: 124 on the first 10s call.
+_dir=$(make_mock_smi "13.0" "10.0")
+cat > "$_dir/timeout" <<MOCK
+#!/bin/sh
+if [ "\$1" = 10 ] && [ "\$#" = 2 ] && [ ! -f "$_dir/timed-out" ]; then : > "$_dir/timed-out"; exit 124; fi
+echo "\$1" >> "$_dir/bounds"
+shift
+exec "\$@"
+MOCK
+chmod +x "$_dir/timeout"
+_result=$(run_func "$_dir")
+assert_eq "banner timed out once, retry reads 13.0 -> cu130" "https://download.pytorch.org/whl/cu130" "$_result"
+case "$(tr '\n' ' ' < "$_dir/bounds")" in *45*) _result=retried ;; *) _result="bounds: $(cat "$_dir/bounds")" ;; esac
+assert_eq "the retry uses the 45s bound" "retried" "$_result"
+rm -rf "$_dir"
+
# 9) ROCm 6.3 (no nvidia-smi) -> rocm6.3
_dir=$(make_mock_amd_smi "6.3")
_result=$(run_func "$_dir")
diff --git a/tests/studio/install/test_install_sh_nvidia_inventory_reader.py b/tests/studio/install/test_install_sh_nvidia_inventory_reader.py
index 21492e4d7fb..b67e0473d02 100644
--- a/tests/studio/install/test_install_sh_nvidia_inventory_reader.py
+++ b/tests/studio/install/test_install_sh_nvidia_inventory_reader.py
@@ -60,10 +60,17 @@ class _Attr:
return 0
-def _run(monkeypatch, libraries: dict[str, object]) -> tuple[int, str]:
- """Run the reader with ctypes.CDLL answering from `libraries`; (exit code, stdout)."""
+def _run(
+ monkeypatch,
+ libraries: dict[str, object],
+ reader: str = "",
+) -> tuple[int, str]:
+ """Run the reader with ctypes.CDLL answering from `libraries`; (exit code, stdout).
+ `reader` is the argument install.sh passes to run one reader under its own deadline."""
import ctypes as real_ctypes
+ monkeypatch.setattr(sys, "argv", ["-", reader] if reader else ["-"])
+
def cdll(name):
if name in libraries:
return libraries[name]
@@ -133,3 +140,32 @@ def test_nvml_answers_first_when_it_can(monkeypatch):
)
code, out = _run(monkeypatch, {"libnvidia-ml.so.1": nvml, "libcuda.so.1": _cuda_lib()})
assert (code, out) == (0, "12.8 6.1,6.1")
+
+
+def _nvml_lib():
+ return _lib(
+ nvmlInit_v2 = _Ok(),
+ nvmlShutdown = _Ok(),
+ nvmlDeviceGetCount_v2 = _Ok(1),
+ nvmlSystemGetCudaDriverVersion_v2 = _Ok(12080),
+ nvmlDeviceGetHandleByIndex_v2 = _Ok(1),
+ nvmlDeviceGetCudaComputeCapability = _Ok(10, 0),
+ )
+
+
+def test_each_reader_runs_alone_under_its_own_deadline(monkeypatch):
+ """install.sh runs NVML and the CUDA driver API as separate bounded processes, so a slow
+ NVML no longer uses up the deadline the driver API needed (8x B200: NVML took ~23s)."""
+ libs = {"libnvidia-ml.so.1": _nvml_lib(), "libcuda.so.1": _cuda_lib(cap = (10, 0))}
+ assert _run(monkeypatch, libs, "nvml") == (0, "12.8 10.0")
+ assert _run(monkeypatch, libs, "cuda") == (0, "13.1 10.0")
+ assert _run(monkeypatch, {"libcuda.so.1": _cuda_lib()}, "nvml") == (1, "")
+ assert _run(monkeypatch, {"libnvidia-ml.so.1": _nvml_lib()}, "cuda") == (1, "")
+
+
+def test_install_sh_gives_each_reader_its_own_bound():
+ text = INSTALL_SH.read_text(encoding = "utf-8")
+ body = text[text.index("_nvidia_library_inventory() {") :]
+ body = body[: body.index("\n}\n")]
+ assert "for _nli_reader in nvml cuda; do" in body
+ assert '_run_bounded --secs "$_nli_secs" "$_nli_py" -I - "$_nli_reader"' in body
diff --git a/tests/studio/install/test_install_sh_torch_arch_check.py b/tests/studio/install/test_install_sh_torch_arch_check.py
new file mode 100644
index 00000000000..96637361b44
--- /dev/null
+++ b/tests/studio/install/test_install_sh_torch_arch_check.py
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+"""install.sh's post-install check that the CUDA torch has kernels for the GPUs, run against a
+stub torch with the driver libraries hidden."""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+INSTALL_SH = Path(__file__).resolve().parents[3] / "install.sh"
+
+CU126 = "sm_50 sm_60 sm_70 sm_75 sm_80 sm_86 sm_90"
+CU130 = "sm_75 sm_80 sm_86 sm_90 sm_100 sm_120"
+
+_SITECUSTOMIZE = """
+import ctypes
+_real = ctypes.CDLL
+class _NoDriver(ctypes.CDLL):
+ def __init__(self, name, *a, **k):
+ if name and ("nvidia-ml" in name or "libcuda" in name):
+ raise OSError(name)
+ super().__init__(name, *a, **k)
+ctypes.CDLL = _NoDriver
+"""
+
+_TORCH = """
+class version:
+ cuda = {cuda!r}
+ hip = None
+__version__ = {ver!r}
+class _C:
+ @staticmethod
+ def _cuda_getArchFlags():
+ return {archs!r}
+class cuda:
+ @staticmethod
+ def is_available():
+ return True
+ @staticmethod
+ def device_count():
+ return {n}
+ @staticmethod
+ def get_device_capability(i):
+ return {caps!r}[i]
+"""
+
+
+def _check_source() -> str:
+ text = INSTALL_SH.read_text(encoding = "utf-8")
+ m = re.search(r"_run_bounded --secs 120 \"\$_VENV_PY\" -c '\n(.*?)\n' 2>/dev/null", text, re.S)
+ assert m, "the post-install arch check was not found"
+ return m.group(1)
+
+
+def _run(tmp_path, cuda, archs, caps):
+ stub = tmp_path / "stub"
+ (stub / "torch").mkdir(parents = True)
+ (stub / "sitecustomize.py").write_text(_SITECUSTOMIZE)
+ (stub / "torch" / "__init__.py").write_text(
+ _TORCH.format(
+ cuda = cuda, ver = f"2.11.0+cu{cuda.replace('.', '')}", archs = archs, n = len(caps), caps = caps
+ )
+ )
+ out = subprocess.run(
+ [sys.executable, "-c", _check_source()],
+ env = {**os.environ, "PYTHONPATH": str(stub)},
+ capture_output = True,
+ text = True,
+ timeout = 60,
+ )
+ assert out.returncode == 0, out.stderr
+ line = [l for l in out.stdout.splitlines() if l.startswith("UNSLOTH_ARCH_CHECK=")]
+ return line[-1].split("=", 1)[1].split("|") if line else None
+
+
+ "cuda, archs, caps, expected",
+ [
+ ("12.6", CU126, [(10, 0)], ("none", "10.0", "cu128")),
+ ("12.6", CU126, [(6, 1), (10, 0)], ("some", "10.0", "cu128")),
+ ("13.0", CU130, [(6, 1)], ("none", "6.1", "cu126")),
+ # No wheel family serves Kepler, and re-installing the same family fixes nothing.
+ ("11.8", CU126, [(3, 7)], ("nofix", "3.7", "cu126")),
+ ("12.8", CU130, [(13, 0)], ("nofix", "13.0", "cu128")),
+ ],
+)
+def test_uncovered_gpus(tmp_path, cuda, archs, caps, expected):
+ status, missing, _torch, _archs, family = _run(tmp_path, cuda, archs, caps)
+ assert (status, missing, family) == expected
+
+
+ "archs, caps",
+ [
+ (CU130, [(10, 3)]), # sm_100 cubin runs on 10.3
+ (CU130, [(12, 1)]),
+ ("sm_80 sm_90a compute_90", [(9, 0)]),
+ ("sm_80 compute_80", [(8, 9)]), # PTX JIT forward
+ ],
+)
+def test_covered_gpus_pass_silently(tmp_path, archs, caps):
+ assert _run(tmp_path, "13.0", archs, caps) is None
+
+
+def test_nofix_status_only_warns():
+ text = INSTALL_SH.read_text(encoding = "utf-8")
+ assert '[ "$_ac_status" = "none" ] && [ "$_torch_index_pinned" = false ]' in text
diff --git a/tests/studio/install/test_probe_timeouts.py b/tests/studio/install/test_probe_timeouts.py
index 4c6acbe87de..859c05c6de5 100644
--- a/tests/studio/install/test_probe_timeouts.py
+++ b/tests/studio/install/test_probe_timeouts.py
@@ -57,7 +57,8 @@ class TestInstallShBoundedProbe:
assert (
"command -v timeout" in body
), "_run_bounded must check for the `timeout` binary before using it"
- assert "timeout 10" in body, "_run_bounded must apply a 10s timeout"
+ assert "_rb_secs=10" in body, "_run_bounded must apply a 10s timeout by default"
+ assert 'timeout "$_rb_secs"' in body, "_run_bounded must pass its bound to timeout"
# Falls back to unbounded when `timeout` is absent (e.g. macOS), keeping semantics there.
assert (
"else" in body and '"$@"' in body
diff --git a/tests/studio/test_nvidia_library_inventory.ps1 b/tests/studio/test_nvidia_library_inventory.ps1
index c7b6789a8fb..a3bc4b8a277 100644
--- a/tests/studio/test_nvidia_library_inventory.ps1
+++ b/tests/studio/test_nvidia_library_inventory.ps1
@@ -43,8 +43,17 @@ $script:FakeIntelAdapters = @()
function Get-IntelRegistryAdapterNames { return $script:FakeIntelAdapters }
# nvidia-smi stand-in: $script:FakeSmiStdout answers every probe, $script:FakeSmiRc is its exit code.
+# $script:FakeSmiTimeouts calls time out first (exit 124); every bound asked for is recorded.
+$script:FakeSmiTimeouts = 0
+$script:FakeSmiBounds = @()
function Invoke-NvidiaSmiBounded {
param([string]$Exe, [string[]]$SmiArgs = @(), [int]$TimeoutSec = 10, [switch]$StdoutOnly)
+ $script:FakeSmiBounds += $TimeoutSec
+ if ($script:FakeSmiTimeouts -gt 0) {
+ $script:FakeSmiTimeouts--
+ $global:LASTEXITCODE = 124
+ return ""
+ }
$global:LASTEXITCODE = $script:FakeSmiRc
return $script:FakeSmiStdout
}
@@ -279,6 +288,14 @@ Check "the cap applies to inventory capabilities" ((Get-TorchIndexUrl) -eq "http
$script:FakeInventory = $null
$script:Substeps = @()
Check "an unreadable banner with no inventory keeps the cu126 default" ((Get-TorchIndexUrl) -eq "https://download.pytorch.org/whl/cu126")
+Check "the cu126 default names the override" (@($script:Substeps | Where-Object { $_ -match 'UNSLOTH_TORCH_INDEX_URL=https://download.pytorch.org/whl/cu128' }).Count -eq 1)
+# A congested driver: nvidia-smi times out once, and the longer retry reads the banner.
+$script:FakeSmiStdout = "| CUDA Version: 13.1 |"
+$script:FakeSmiTimeouts = 1
+$script:FakeSmiBounds = @()
+Check "a timed-out banner is retried and read" ((Get-TorchIndexUrl) -eq "https://download.pytorch.org/whl/cu130")
+Check "the retry has the longer bound" (($script:FakeSmiBounds[0..1] -join ",") -eq "10,45")
+$script:FakeSmiTimeouts = 0
$script:FakeSmiStdout = "| CUDA Version: 12.6 |"
Check "a readable banner is still authoritative" ((Get-TorchIndexUrl) -eq "https://download.pytorch.org/whl/cu126")