Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Recover CUDA compute capabilities without emitting a P/Invoke type (#11173)

Daniel Hanb7d2b33f73a9
diff --git a/.github/workflows/cross-platform-parity-ci.yml b/.github/workflows/cross-platform-parity-ci.yml
index d0be9c3ea21..69db28eaa72 100644
--- a/.github/workflows/cross-platform-parity-ci.yml
+++ b/.github/workflows/cross-platform-parity-ci.yml
@@ -76,6 +76,7 @@ on:
- 'tests/python/test_windows_python_venv_hardening.py'
- 'tests/python/test_windows_setup_output_encoding.py'
- 'tests/python/test_windows_installer_native_type_fallback.py'
+ - 'tests/python/test_installer_embedded_nvidia_probe.py'
- 'tests/python/test_pwsh_runner_encoding.py'
# The deliberate unreadable-directory and traversal-limit cases for the
# no-compiler detector's temp scan live in this job, and only here. The
@@ -105,6 +106,7 @@ on:
- 'tests/studio/test_amd_name_arch_r9700.ps1'
- 'tests/studio/test_nvidia_library_inventory.ps1'
- 'tests/studio/test_nvidia_smi_discovery.ps1'
+ - 'tests/studio/test_nvidia_python_probe_parity.ps1'
- 'tests/studio/test_install_lock_excludes_aliases.ps1'
- 'tests/studio/test_early_python_path_resolver.ps1'
- 'tests/studio/test_early_python_process_image.ps1'
@@ -191,6 +193,7 @@ on:
- 'tests/python/test_windows_python_venv_hardening.py'
- 'tests/python/test_windows_setup_output_encoding.py'
- 'tests/python/test_windows_installer_native_type_fallback.py'
+ - 'tests/python/test_installer_embedded_nvidia_probe.py'
- 'tests/python/test_pwsh_runner_encoding.py'
# The deliberate unreadable-directory and traversal-limit cases for the
# no-compiler detector's temp scan live in this job, and only here. The
@@ -220,6 +223,7 @@ on:
- 'tests/studio/test_amd_name_arch_r9700.ps1'
- 'tests/studio/test_nvidia_library_inventory.ps1'
- 'tests/studio/test_nvidia_smi_discovery.ps1'
+ - 'tests/studio/test_nvidia_python_probe_parity.ps1'
- 'tests/studio/test_install_lock_excludes_aliases.ps1'
- 'tests/studio/test_early_python_path_resolver.ps1'
- 'tests/studio/test_early_python_process_image.ps1'
@@ -327,6 +331,7 @@ jobs:
tests/python/test_windows_installer_native_type_fallback.py
tests/python/test_pwsh_runner_encoding.py
tests/python/test_compiler_watch_temp_scan.py
+ tests/python/test_installer_embedded_nvidia_probe.py
tests/test_installer_skip_autostart.py
tests/test_installer_system32_guard.py
tests/test_installer_interactive_prompts.py
@@ -390,6 +395,17 @@ jobs:
if: runner.os == 'Windows'
shell: powershell
run: powershell -NoProfile -File tests\studio\test_nvidia_smi_discovery.ps1
+ # The inventory's Python rung, which recovers the CUDA version and the per-device compute
+ # capabilities on hosts that cannot emit a P/Invoke type. Pins the embedded probe against
+ # studio/nvidia_probe.py and against the other copy; the 5.1 leg matters because
+ # Constrained Language Mode is one of the policies that make the emitted rung decline.
+ - name: NVIDIA Python probe parity tests
+ shell: pwsh
+ run: pwsh -NoProfile -File tests/studio/test_nvidia_python_probe_parity.ps1
+ - name: NVIDIA Python probe parity tests (Windows PowerShell 5.1)
+ if: runner.os == 'Windows'
+ shell: powershell
+ run: powershell -NoProfile -File tests\studio\test_nvidia_python_probe_parity.ps1
# Spawns a real holder process and takes the lock through a junction (Windows) or a
# symbolic link (Linux), so it needs both legs: the alias mechanisms differ per OS and
# .NET enforces FileShare.None through a different mechanism on each.
diff --git a/install.ps1 b/install.ps1
index 8b01b97ffb1..241599b2077 100644
--- a/install.ps1
+++ b/install.ps1
@@ -8793,6 +8793,27 @@ exit 0
return ($LASTEXITCODE -eq 0 -and $out -match '(?m)^GPU\s+\d+:')
}
+ # Interpreter for the shared inventory's Python rung, deliberately NOT part of the shared
+ # region: the two files find Python in different places. Here it is the early read-only
+ # ladder from the process-image rung, which never installs anything.
+ function Get-NvidiaProbePythonExe {
+ if ("$($env:UNSLOTH_EARLY_PYTHON_PROBE)".Trim() -eq "0") { return "" }
+ # The interpreter this run installed comes FIRST, and Get-StudioEarlyPython is the
+ # fallback rather than the source.
+ #
+ # That ladder memoises, including a miss: on a fresh host with no Python of its own it
+ # is first called by the install-lock path, finds nothing, and caches $null for the rest
+ # of the run. The inventory is not read until much later, by which point this install
+ # has created a managed interpreter and then a venv, so asking the cache would decline
+ # on exactly the fresh install where nvidia-smi is also most likely to be missing. The
+ # host would take CPU wheels while holding a working NVIDIA card.
+ foreach ($candidate in @($VenvPython, $ManagedPythonPath)) {
+ if ([string]::IsNullOrWhiteSpace($candidate)) { continue }
+ if (Test-Path -LiteralPath $candidate -PathType Leaf) { return $candidate }
+ }
+ try { return "$(Get-StudioEarlyPython)" } catch { return "" }
+ }
+
# ── BEGIN SHARED WITH studio/setup.ps1 (Get-NvidiaLibraryInventory) ──
# Full paths only: a bare name searches PATH, where ZLUDA's nvcuda.dll and nvml.dll pass for NVIDIA.
function Get-NvidiaNvmlLibraryPath {
@@ -8845,72 +8866,311 @@ exit 0
return ($name -as [type])
}
+ # The same inventory with nothing emitted: CPython's ctypes makes the identical NVML and CUDA
+ # driver calls, and the interop leaves the scanned surface rather than moving within it.
+ # A second source BENEATH the emitted one, never ahead of it: "" whenever no interpreter is
+ # available or the probe itself says nothing.
+ # Get-NvidiaProbePythonExe is deliberately per-file. The installer has its early read-only
+ # interpreter ladder; setup.ps1 has the venv a previous run already built.
+ function Read-NvidiaLibraryRawViaPython {
+ param([int]$TimeoutMs = 10000, [switch]$SkipNvml)
+ # Set only when the child is killed at the deadline, so a caller can tell a hung driver from
+ # an empty answer. Reset first: every early return below is an answer, not a timeout.
+ $script:NvidiaPythonProbeTimedOut = $false
+ if ("$($env:UNSLOTH_NVIDIA_PYTHON_PROBE)".Trim() -eq "0") { return "" }
+ $exe = ""
+ try { $exe = "$(Get-NvidiaProbePythonExe)" } catch { return "" }
+ if (-not $exe) { return "" }
+ $windows = ($env:OS -eq "Windows_NT")
+ $nvmlHint = if ($windows) { Get-NvidiaNvmlLibraryPath } else { "libnvidia-ml.so.1" }
+ $cudaHint = if ($windows) { Join-Path (Get-NvidiaSystem32Dir) "nvcuda.dll" } else { "libcuda.so.1" }
+ # Kept byte-identical with studio/nvidia_probe.py's readers by
+ # tests/studio/test_nvidia_python_probe_parity.ps1. Column 0 on purpose: this is Python.
+ $probeSource = @'
+import ctypes, os, sys
+
+
+def _names(kind, hint):
+ if os.name == "nt":
+ # The driver's full path only: a bare name also finds a CUDA stand-in such as ZLUDA
+ # beside the interpreter, and that is not an NVIDIA GPU (#11736).
+ return [hint] if hint and os.path.isabs(hint) else []
+ if kind == "nvml":
+ return ["libnvidia-ml.so.1", "libnvidia-ml.so"]
+ return ["libcuda.so.1", "libcuda.so"]
+
+
+def _load(kind, hint):
+ for name in _names(kind, hint):
+ if not name:
+ continue
+ try:
+ return ctypes.CDLL(name)
+ except Exception:
+ continue
+ return None
+
+
+def _unpack(packed):
+ return "%d;%d" % (packed // 1000, (packed % 1000) // 10)
+
+
+def read_nvml(hint):
+ lib = _load("nvml", hint)
+ if lib is None:
+ return ""
+ try:
+ if lib.nvmlInit_v2() != 0:
+ return ""
+ except Exception:
+ return ""
+ try:
+ count = ctypes.c_uint(0)
+ if lib.nvmlDeviceGetCount_v2(ctypes.byref(count)) != 0 or count.value == 0:
+ return ""
+ packed = ctypes.c_int(0)
+ if lib.nvmlSystemGetCudaDriverVersion_v2(ctypes.byref(packed)) != 0 or packed.value < 1000:
+ return ""
+ # ctypes defaults every return and every pointer argument to a C int, which truncates a
+ # 64-bit nvmlDevice_t handle. Declare both before the first call, not after.
+ handle_of = lib.nvmlDeviceGetHandleByIndex_v2
+ handle_of.restype = ctypes.c_int
+ handle_of.argtypes = [ctypes.c_uint, ctypes.POINTER(ctypes.c_void_p)]
+ cap_of = lib.nvmlDeviceGetCudaComputeCapability
+ cap_of.restype = ctypes.c_int
+ cap_of.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)]
+ caps = []
+ for index in range(count.value):
+ device = ctypes.c_void_p()
+ # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.
+ if handle_of(index, ctypes.byref(device)) != 0:
+ return ""
+ major = ctypes.c_int(0)
+ minor = ctypes.c_int(0)
+ if cap_of(device, ctypes.byref(major), ctypes.byref(minor)) != 0:
+ return ""
+ caps.append("%d.%d" % (major.value, minor.value))
+ return "nvml;%s;%s" % (_unpack(packed.value), ",".join(caps))
+ finally:
+ try:
+ lib.nvmlShutdown()
+ except Exception:
+ pass
+
+
+def read_cuda(hint):
+ lib = _load("cuda", hint)
+ if lib is None:
+ return ""
+ # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one, so a
+ # hidden pre-Turing card still caps the family. cuInit reads the mask once.
+ saved = os.environ.pop("CUDA_VISIBLE_DEVICES", None)
+ try:
+ init = lib.cuInit(0)
+ except Exception:
+ return ""
+ finally:
+ if saved is not None:
+ os.environ["CUDA_VISIBLE_DEVICES"] = saved
+ if init != 0:
+ return ""
+ count = ctypes.c_int(0)
+ if lib.cuDeviceGetCount(ctypes.byref(count)) != 0 or count.value == 0:
+ return ""
+ packed = ctypes.c_int(0)
+ if lib.cuDriverGetVersion(ctypes.byref(packed)) != 0 or packed.value < 1000:
+ return ""
+ caps = []
+ for index in range(count.value):
+ device = ctypes.c_int(0)
+ if lib.cuDeviceGet(ctypes.byref(device), index) != 0:
+ return ""
+ major = ctypes.c_int(0)
+ minor = ctypes.c_int(0)
+ # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.
+ if lib.cuDeviceGetAttribute(ctypes.byref(major), 75, device) != 0:
+ return ""
+ if lib.cuDeviceGetAttribute(ctypes.byref(minor), 76, device) != 0:
+ return ""
+ caps.append("%d.%d" % (major.value, minor.value))
+ return "cuda;%s;%s" % (_unpack(packed.value), ",".join(caps))
+
+
+def main():
+ nvml_hint = os.environ.get("UNSLOTH_NVML_HINT", "")
+ cuda_hint = os.environ.get("UNSLOTH_CUDA_HINT", "")
+ answer = ""
+ # Set only for the CUDA-only retry that follows a child NVML held past its deadline.
+ if os.environ.get("UNSLOTH_NVIDIA_PROBE_SKIP_NVML", "") != "1":
+ try:
+ answer = read_nvml(nvml_hint)
+ except Exception:
+ answer = ""
+ if not answer:
+ try:
+ answer = read_cuda(cuda_hint)
+ except Exception:
+ answer = ""
+ sys.stdout.write(answer)
+
+
+main()
+'@
+ # Cmdlets only. Constrained Language Mode refuses New-Object ProcessStartInfo and
+ # [Process]::Start, and CLM is one of the two policies that make the emitted rung decline,
+ # so this launcher has to work on exactly the hosts that need it most.
+ $tempRoot = if ($env:TEMP) { $env:TEMP } elseif ($env:TMPDIR) { $env:TMPDIR } else { "/tmp" }
+ $stem = Join-Path $tempRoot ("unsloth-nvprobe-" + [guid]::NewGuid().ToString("N"))
+ $scriptFile = "$stem.py"
+ $outFile = "$stem.out"
+ $errFile = "$stem.err"
+ $raw = ""
+ try {
+ Set-Content -LiteralPath $scriptFile -Value $probeSource -Encoding UTF8 -ErrorAction Stop
+ # Whole seconds, rounded up, without [math]::Ceiling: CLM blocks it. PowerShell's / is
+ # floating point and [int] rounds to nearest, so 10000ms must not become 11s.
+ $seconds = ($TimeoutMs - ($TimeoutMs % 1000)) / 1000
+ if (($TimeoutMs % 1000) -ne 0) { $seconds = $seconds + 1 }
+ $seconds = [int]$seconds
+ if ($seconds -lt 1) { $seconds = 1 }
+ # Nothing with a space in it reaches the command line. Windows PowerShell 5.1 appends
+ # each native argument verbatim, so a script under "C:\Users\First Last\AppData\Local\
+ # Temp" or a hint under "C:\Program Files\NVIDIA Corporation\NVSMI" would split on its
+ # spaces and the child would run something else. The script arrives on stdin and the two
+ # library hints in the environment; the only arguments left are -I -S and a bare dash.
+ $savedNvml = $env:UNSLOTH_NVML_HINT
+ $savedCuda = $env:UNSLOTH_CUDA_HINT
+ $savedSkip = $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML
+ $env:UNSLOTH_NVML_HINT = $nvmlHint
+ $env:UNSLOTH_CUDA_HINT = $cudaHint
+ # The switch reaches this child only. An inherited value must not make a first child skip NVML.
+ if ($SkipNvml) { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = "1" }
+ else { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }
+ try {
+ $proc = Start-Process -FilePath $exe -ArgumentList @("-I", "-S", "-") -NoNewWindow -PassThru `
+ -RedirectStandardInput $scriptFile -RedirectStandardOutput $outFile -RedirectStandardError $errFile -ErrorAction Stop
+ } finally {
+ if ($null -eq $savedNvml) { Remove-Item Env:UNSLOTH_NVML_HINT -ErrorAction SilentlyContinue }
+ else { $env:UNSLOTH_NVML_HINT = $savedNvml }
+ if ($null -eq $savedCuda) { Remove-Item Env:UNSLOTH_CUDA_HINT -ErrorAction SilentlyContinue }
+ else { $env:UNSLOTH_CUDA_HINT = $savedCuda }
+ if ($null -eq $savedSkip) { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }
+ else { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = $savedSkip }
+ }
+ if (-not $proc) { return "" }
+ # -InputObject and an error variable, never $proc.Id or $proc.HasExited. Constrained
+ # Language Mode permits property reads only on its allowed type list and
+ # System.Diagnostics.Process is not on it, so reading either one throws on exactly the
+ # hosts this rung exists for. Handing the object to a cmdlet keeps the access inside
+ # compiled code, where the language mode does not reach.
+ $waitError = $null
+ Wait-Process -InputObject $proc -Timeout $seconds -ErrorAction SilentlyContinue -ErrorVariable waitError
+ if ($waitError) {
+ $script:NvidiaPythonProbeTimedOut = $true
+ try { Stop-Process -InputObject $proc -Force -ErrorAction SilentlyContinue } catch { }
+ # Let the killed child release its redirected files before the finally deletes them.
+ Wait-Process -InputObject $proc -Timeout 2 -ErrorAction SilentlyContinue
+ return ""
+ }
+ $raw = "$(Get-Content -LiteralPath $outFile -Raw -ErrorAction SilentlyContinue)"
+ } catch { return "" }
+ finally {
+ foreach ($stale in @($scriptFile, $outFile, $errFile)) {
+ Remove-Item -LiteralPath $stale -Force -ErrorAction SilentlyContinue
+ }
+ }
+ return "$raw".Trim()
+ }
+
# "source;cudaMajor;cudaMinor;cap,cap" from NVML, else the CUDA driver API; "" when neither
# answers. Versions are major*1000 + minor*10. One runspace + deadline per reader: a shared one let slow NVML starve CUDA.
+ # A wedged driver can block inside the library, and each deadline leaves its runspace behind.
+ # The Python rung below spends what the emitted rung left of ONE budget (a per-reader bound for
+ # NVML plus one for CUDA), so the worst-case wall clock is the emitted rung's own. Its first child
+ # reads NVML then the CUDA driver API under at most one per-reader bound; only when that child is
+ # killed at its bound (a hung NVML) and at least 2 s remain does a second child read the CUDA
+ # driver API alone with the rest, so a hung NVML no longer starves a healthy CUDA driver API.
function Read-NvidiaLibraryRaw {
param([int]$TimeoutMs = 30000)
+ $deadline = (Get-Date).AddMilliseconds($TimeoutMs * 2)
+ $native = ""
$type = Get-NvidiaLibraryProbeType
- if (-not $type) { return "" }
- $reader = {
- param($T, $Which)
- function Read-Nvml {
- if ($T::nvmlInit_v2() -ne 0) { return "" }
- try {
- [uint32]$count = 0
- if ($T::nvmlDeviceGetCount_v2([ref]$count) -ne 0 -or $count -eq 0) { return "" }
+ if ($type) {
+ $reader = {
+ param($T, $Which)
+ function Read-Nvml {
+ if ($T::nvmlInit_v2() -ne 0) { return "" }
+ try {
+ [uint32]$count = 0
+ if ($T::nvmlDeviceGetCount_v2([ref]$count) -ne 0 -or $count -eq 0) { return "" }
+ [int]$ver = 0
+ if ($T::nvmlSystemGetCudaDriverVersion_v2([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }
+ $caps = @()
+ for ([uint32]$i = 0; $i -lt $count; $i++) {
+ [IntPtr]$dev = [IntPtr]::Zero; [int]$major = 0; [int]$minor = 0
+ # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.
+ if ($T::nvmlDeviceGetHandleByIndex_v2($i, [ref]$dev) -ne 0) { return "" }
+ if ($T::nvmlDeviceGetCudaComputeCapability($dev, [ref]$major, [ref]$minor) -ne 0) { return "" }
+ $caps += "$major.$minor"
+ }
+ return "nvml;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
+ } finally { $null = $T::nvmlShutdown() }
+ }
+ function Read-Cuda {
+ # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one,
+ # so a hidden pre-Turing card still caps the family. cuInit reads the mask once.
+ $saved = $env:CUDA_VISIBLE_DEVICES
+ Remove-Item Env:CUDA_VISIBLE_DEVICES -ErrorAction SilentlyContinue
+ try { $init = $T::cuInit([uint32]0) } finally { if ($null -ne $saved) { $env:CUDA_VISIBLE_DEVICES = $saved } }
+ if ($init -ne 0) { return "" }
+ [int]$count = 0
+ if ($T::cuDeviceGetCount([ref]$count) -ne 0 -or $count -eq 0) { return "" }
[int]$ver = 0
- if ($T::nvmlSystemGetCudaDriverVersion_v2([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }
+ if ($T::cuDriverGetVersion([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }
$caps = @()
- for ([uint32]$i = 0; $i -lt $count; $i++) {
- [IntPtr]$dev = [IntPtr]::Zero; [int]$major = 0; [int]$minor = 0
- # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.
- if ($T::nvmlDeviceGetHandleByIndex_v2($i, [ref]$dev) -ne 0) { return "" }
- if ($T::nvmlDeviceGetCudaComputeCapability($dev, [ref]$major, [ref]$minor) -ne 0) { return "" }
+ for ($i = 0; $i -lt $count; $i++) {
+ [int]$dev = 0; [int]$major = 0; [int]$minor = 0
+ if ($T::cuDeviceGet([ref]$dev, $i) -ne 0) { return "" }
+ # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.
+ if ($T::cuDeviceGetAttribute([ref]$major, 75, $dev) -ne 0) { return "" }
+ if ($T::cuDeviceGetAttribute([ref]$minor, 76, $dev) -ne 0) { return "" }
$caps += "$major.$minor"
}
- return "nvml;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
- } finally { $null = $T::nvmlShutdown() }
- }
- function Read-Cuda {
- # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one,
- # so a hidden pre-Turing card still caps the family. cuInit reads the mask once.
- $saved = $env:CUDA_VISIBLE_DEVICES
- Remove-Item Env:CUDA_VISIBLE_DEVICES -ErrorAction SilentlyContinue
- try { $init = $T::cuInit([uint32]0) } finally { if ($null -ne $saved) { $env:CUDA_VISIBLE_DEVICES = $saved } }
- if ($init -ne 0) { return "" }
- [int]$count = 0
- if ($T::cuDeviceGetCount([ref]$count) -ne 0 -or $count -eq 0) { return "" }
- [int]$ver = 0
- if ($T::cuDriverGetVersion([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }
- $caps = @()
- for ($i = 0; $i -lt $count; $i++) {
- [int]$dev = 0; [int]$major = 0; [int]$minor = 0
- if ($T::cuDeviceGet([ref]$dev, $i) -ne 0) { return "" }
- # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.
- if ($T::cuDeviceGetAttribute([ref]$major, 75, $dev) -ne 0) { return "" }
- if ($T::cuDeviceGetAttribute([ref]$minor, 76, $dev) -ne 0) { return "" }
- $caps += "$major.$minor"
- }
- return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
- }
- $r = ""
- try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }
- return "$r"
- }
- foreach ($which in @("nvml", "cuda")) {
- $ps = $null; $handle = $null; $r = ""
- try {
- $ps = [powershell]::Create()
- $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)
- $handle = $ps.BeginInvoke()
- if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {
- $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"
+ return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
}
- } catch { $r = "" }
- finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }
- if ($r) { return $r }
- }
- return ""
+ $r = ""
+ try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }
+ return "$r"
+ }
+ foreach ($which in @("nvml", "cuda")) {
+ $ps = $null; $handle = $null; $r = ""
+ try {
+ $ps = [powershell]::Create()
+ $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)
+ $handle = $ps.BeginInvoke()
+ if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {
+ $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"
+ }
+ } catch { $r = "" }
+ finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }
+ if ($r) { $native = $r; break }
+ }
+ }
+ if ($native) { return $native }
+ $remainingMs = [int]($deadline - (Get-Date)).TotalMilliseconds
+ # Not worth a child process we cannot wait out; the emitted rung already spent the budget.
+ if ($remainingMs -lt 3000) { return "" }
+ # Each child gets at most one per-reader bound, compared by hand: CLM refuses [math]. 2 s of
+# every child's share is kept back for reaping it if it hangs, so the pair ends inside the deadline.
+ $childMs = $remainingMs - 2000; if ($childMs -gt $TimeoutMs) { $childMs = $TimeoutMs }
+ $raw = ""
+ try { $raw = Read-NvidiaLibraryRawViaPython -TimeoutMs $childMs } catch { return "" }
+ # Only a first child killed at its bound (a hung NVML) earns a CUDA-only child, with what is left.
+ if (-not $script:NvidiaPythonProbeTimedOut) { return $raw }
+ $remainingMs = [int]($deadline - (Get-Date)).TotalMilliseconds
+ if ($remainingMs -lt 3000) { return "" }
+ $childMs = $remainingMs - 2000; if ($childMs -gt $TimeoutMs) { $childMs = $TimeoutMs }
+ try { return (Read-NvidiaLibraryRawViaPython -TimeoutMs $childMs -SkipNvml) } catch { return "" }
}
# NVIDIA inventory from the driver's own libraries (NVML, then the CUDA driver API), for a
@@ -9029,9 +9289,13 @@ exit 0
}
}
if (-not $HasNvidiaSmi -and (Get-NvidiaLibraryInventory)) {
- # Same promotion as setup.ps1: the gates below read $HasNvidiaSmi as "NVIDIA GPU present".
- $HasNvidiaSmi = $true
- Write-StudioLine " NVIDIA GPU found through the driver library; nvidia-smi is unavailable" -ForegroundColor Gray
+ # A driver too old for any CUDA wheel (below 11) is no GPU this route can serve: the Intel and
+ # AMD routes still get their turn instead of the install falling to CPU.
+ if ((Get-NvidiaLibraryInventory).CudaMajor -ge 11) {
+ # Same promotion as setup.ps1: the gates below read $HasNvidiaSmi as "NVIDIA GPU present".
+ $HasNvidiaSmi = $true
+ Write-StudioLine " NVIDIA GPU found through the driver library; nvidia-smi is unavailable" -ForegroundColor Gray
+ }
}
# nvidia-smi was already resolved above and never asked which card it found, so the
# banner said "NVIDIA GPU detected" on every NVIDIA host alike. compute_cap is the
diff --git a/studio/setup.ps1 b/studio/setup.ps1
index 1703eb6c021..4c2e16dc69c 100644
--- a/studio/setup.ps1
+++ b/studio/setup.ps1
@@ -155,6 +155,7 @@ if ($script:UnslothVerbose) {
$script:LlamaCppDegraded = $false
$script:LlamaKeptGpuPrebuilt = $null
$script:NvidiaSmiRejected = $false
+$script:NvidiaDriverLibraryOnly = $false
$script:NvidiaLibraryInventoryProbed = $false
$script:NvidiaLibraryInventory = $null
# Set by the offline keep, read unconditionally by the sidecar and legacy-migration blocks:
@@ -1184,7 +1185,7 @@ function Get-GpuPrebuiltToKeepOverSourceBuild {
if ("$($env:UNSLOTH_LLAMA_TAG)".Trim() -notin @("", "latest")) { return "" }
$backend = Get-PrebuiltMarkerBackend -Marker (Join-Path $InstallDir "UNSLOTH_PREBUILT_INFO.json")
if (-not $backend) { return "" }
- $nvidia = $HasNvidiaSmi
+ $nvidia = $HasNvidiaSmi -or [bool]$script:NvidiaDriverLibraryOnly
$amd = $HasROCm -or [bool]$script:ROCmGfxArch
$present = switch ($backend) {
"cuda" { $nvidia }
@@ -1485,6 +1486,20 @@ function Get-CudaFamilyCappedForPreTuring {
return $Family
}
+# Interpreter for the shared inventory's Python rung, deliberately NOT part of the shared
+# region: the two files find Python in different places. setup.ps1 runs after the installer,
+# so the venv a previous run built is the interpreter, and $VenvDir is set well before the
+# first Get-NvidiaLibraryInventory call. No interpreter simply means the rung declines.
+function Get-NvidiaProbePythonExe {
+ if ("$($env:UNSLOTH_EARLY_PYTHON_PROBE)".Trim() -eq "0") { return "" }
+ if (-not $VenvDir) { return "" }
+ foreach ($leaf in @("Scripts\python.exe", "bin/python3", "bin/python")) {
+ $candidate = Join-Path $VenvDir $leaf
+ if (Test-Path -LiteralPath $candidate -PathType Leaf) { return $candidate }
+ }
+ return ""
+}
+
# ── BEGIN SHARED WITH install.ps1 (Get-NvidiaLibraryInventory) ──
# Full paths only: a bare name searches PATH, where ZLUDA's nvcuda.dll and nvml.dll pass for NVIDIA.
function Get-NvidiaNvmlLibraryPath {
@@ -1537,72 +1552,311 @@ function Get-NvidiaLibraryProbeType {
return ($name -as [type])
}
+# The same inventory with nothing emitted: CPython's ctypes makes the identical NVML and CUDA
+# driver calls, and the interop leaves the scanned surface rather than moving within it.
+# A second source BENEATH the emitted one, never ahead of it: "" whenever no interpreter is
+# available or the probe itself says nothing.
+# Get-NvidiaProbePythonExe is deliberately per-file. The installer has its early read-only
+# interpreter ladder; setup.ps1 has the venv a previous run already built.
+function Read-NvidiaLibraryRawViaPython {
+ param([int]$TimeoutMs = 10000, [switch]$SkipNvml)
+ # Set only when the child is killed at the deadline, so a caller can tell a hung driver from
+ # an empty answer. Reset first: every early return below is an answer, not a timeout.
+ $script:NvidiaPythonProbeTimedOut = $false
+ if ("$($env:UNSLOTH_NVIDIA_PYTHON_PROBE)".Trim() -eq "0") { return "" }
+ $exe = ""
+ try { $exe = "$(Get-NvidiaProbePythonExe)" } catch { return "" }
+ if (-not $exe) { return "" }
+ $windows = ($env:OS -eq "Windows_NT")
+ $nvmlHint = if ($windows) { Get-NvidiaNvmlLibraryPath } else { "libnvidia-ml.so.1" }
+ $cudaHint = if ($windows) { Join-Path (Get-NvidiaSystem32Dir) "nvcuda.dll" } else { "libcuda.so.1" }
+ # Kept byte-identical with studio/nvidia_probe.py's readers by
+ # tests/studio/test_nvidia_python_probe_parity.ps1. Column 0 on purpose: this is Python.
+ $probeSource = @'
+import ctypes, os, sys
+
+
+def _names(kind, hint):
+ if os.name == "nt":
+ # The driver's full path only: a bare name also finds a CUDA stand-in such as ZLUDA
+ # beside the interpreter, and that is not an NVIDIA GPU (#11736).
+ return [hint] if hint and os.path.isabs(hint) else []
+ if kind == "nvml":
+ return ["libnvidia-ml.so.1", "libnvidia-ml.so"]
+ return ["libcuda.so.1", "libcuda.so"]
+
+
+def _load(kind, hint):
+ for name in _names(kind, hint):
+ if not name:
+ continue
+ try:
+ return ctypes.CDLL(name)
+ except Exception:
+ continue
+ return None
+
+
+def _unpack(packed):
+ return "%d;%d" % (packed // 1000, (packed % 1000) // 10)
+
+
+def read_nvml(hint):
+ lib = _load("nvml", hint)
+ if lib is None:
+ return ""
+ try:
+ if lib.nvmlInit_v2() != 0:
+ return ""
+ except Exception:
+ return ""
+ try:
+ count = ctypes.c_uint(0)
+ if lib.nvmlDeviceGetCount_v2(ctypes.byref(count)) != 0 or count.value == 0:
+ return ""
+ packed = ctypes.c_int(0)
+ if lib.nvmlSystemGetCudaDriverVersion_v2(ctypes.byref(packed)) != 0 or packed.value < 1000:
+ return ""
+ # ctypes defaults every return and every pointer argument to a C int, which truncates a
+ # 64-bit nvmlDevice_t handle. Declare both before the first call, not after.
+ handle_of = lib.nvmlDeviceGetHandleByIndex_v2
+ handle_of.restype = ctypes.c_int
+ handle_of.argtypes = [ctypes.c_uint, ctypes.POINTER(ctypes.c_void_p)]
+ cap_of = lib.nvmlDeviceGetCudaComputeCapability
+ cap_of.restype = ctypes.c_int
+ cap_of.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)]
+ caps = []
+ for index in range(count.value):
+ device = ctypes.c_void_p()
+ # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.
+ if handle_of(index, ctypes.byref(device)) != 0:
+ return ""
+ major = ctypes.c_int(0)
+ minor = ctypes.c_int(0)
+ if cap_of(device, ctypes.byref(major), ctypes.byref(minor)) != 0:
+ return ""
+ caps.append("%d.%d" % (major.value, minor.value))
+ return "nvml;%s;%s" % (_unpack(packed.value), ",".join(caps))
+ finally:
+ try:
+ lib.nvmlShutdown()
+ except Exception:
+ pass
+
+
+def read_cuda(hint):
+ lib = _load("cuda", hint)
+ if lib is None:
+ return ""
+ # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one, so a
+ # hidden pre-Turing card still caps the family. cuInit reads the mask once.
+ saved = os.environ.pop("CUDA_VISIBLE_DEVICES", None)
+ try:
+ init = lib.cuInit(0)
+ except Exception:
+ return ""
+ finally:
+ if saved is not None:
+ os.environ["CUDA_VISIBLE_DEVICES"] = saved
+ if init != 0:
+ return ""
+ count = ctypes.c_int(0)
+ if lib.cuDeviceGetCount(ctypes.byref(count)) != 0 or count.value == 0:
+ return ""
+ packed = ctypes.c_int(0)
+ if lib.cuDriverGetVersion(ctypes.byref(packed)) != 0 or packed.value < 1000:
+ return ""
+ caps = []
+ for index in range(count.value):
+ device = ctypes.c_int(0)
+ if lib.cuDeviceGet(ctypes.byref(device), index) != 0:
+ return ""
+ major = ctypes.c_int(0)
+ minor = ctypes.c_int(0)
+ # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.
+ if lib.cuDeviceGetAttribute(ctypes.byref(major), 75, device) != 0:
+ return ""
+ if lib.cuDeviceGetAttribute(ctypes.byref(minor), 76, device) != 0:
+ return ""
+ caps.append("%d.%d" % (major.value, minor.value))
+ return "cuda;%s;%s" % (_unpack(packed.value), ",".join(caps))
+
+
+def main():
+ nvml_hint = os.environ.get("UNSLOTH_NVML_HINT", "")
+ cuda_hint = os.environ.get("UNSLOTH_CUDA_HINT", "")
+ answer = ""
+ # Set only for the CUDA-only retry that follows a child NVML held past its deadline.
+ if os.environ.get("UNSLOTH_NVIDIA_PROBE_SKIP_NVML", "") != "1":
+ try:
+ answer = read_nvml(nvml_hint)
+ except Exception:
+ answer = ""
+ if not answer:
+ try:
+ answer = read_cuda(cuda_hint)
+ except Exception:
+ answer = ""
+ sys.stdout.write(answer)
+
+
+main()
+'@
+ # Cmdlets only. Constrained Language Mode refuses New-Object ProcessStartInfo and
+ # [Process]::Start, and CLM is one of the two policies that make the emitted rung decline,
+ # so this launcher has to work on exactly the hosts that need it most.
+ $tempRoot = if ($env:TEMP) { $env:TEMP } elseif ($env:TMPDIR) { $env:TMPDIR } else { "/tmp" }
+ $stem = Join-Path $tempRoot ("unsloth-nvprobe-" + [guid]::NewGuid().ToString("N"))
+ $scriptFile = "$stem.py"
+ $outFile = "$stem.out"
+ $errFile = "$stem.err"
+ $raw = ""
+ try {
+ Set-Content -LiteralPath $scriptFile -Value $probeSource -Encoding UTF8 -ErrorAction Stop
+ # Whole seconds, rounded up, without [math]::Ceiling: CLM blocks it. PowerShell's / is
+ # floating point and [int] rounds to nearest, so 10000ms must not become 11s.
+ $seconds = ($TimeoutMs - ($TimeoutMs % 1000)) / 1000
+ if (($TimeoutMs % 1000) -ne 0) { $seconds = $seconds + 1 }
+ $seconds = [int]$seconds
+ if ($seconds -lt 1) { $seconds = 1 }
+ # Nothing with a space in it reaches the command line. Windows PowerShell 5.1 appends
+ # each native argument verbatim, so a script under "C:\Users\First Last\AppData\Local\
+ # Temp" or a hint under "C:\Program Files\NVIDIA Corporation\NVSMI" would split on its
+ # spaces and the child would run something else. The script arrives on stdin and the two
+ # library hints in the environment; the only arguments left are -I -S and a bare dash.
+ $savedNvml = $env:UNSLOTH_NVML_HINT
+ $savedCuda = $env:UNSLOTH_CUDA_HINT
+ $savedSkip = $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML
+ $env:UNSLOTH_NVML_HINT = $nvmlHint
+ $env:UNSLOTH_CUDA_HINT = $cudaHint
+ # The switch reaches this child only. An inherited value must not make a first child skip NVML.
+ if ($SkipNvml) { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = "1" }
+ else { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }
+ try {
+ $proc = Start-Process -FilePath $exe -ArgumentList @("-I", "-S", "-") -NoNewWindow -PassThru `
+ -RedirectStandardInput $scriptFile -RedirectStandardOutput $outFile -RedirectStandardError $errFile -ErrorAction Stop
+ } finally {
+ if ($null -eq $savedNvml) { Remove-Item Env:UNSLOTH_NVML_HINT -ErrorAction SilentlyContinue }
+ else { $env:UNSLOTH_NVML_HINT = $savedNvml }
+ if ($null -eq $savedCuda) { Remove-Item Env:UNSLOTH_CUDA_HINT -ErrorAction SilentlyContinue }
+ else { $env:UNSLOTH_CUDA_HINT = $savedCuda }
+ if ($null -eq $savedSkip) { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }
+ else { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = $savedSkip }
+ }
+ if (-not $proc) { return "" }
+ # -InputObject and an error variable, never $proc.Id or $proc.HasExited. Constrained
+ # Language Mode permits property reads only on its allowed type list and
+ # System.Diagnostics.Process is not on it, so reading either one throws on exactly the
+ # hosts this rung exists for. Handing the object to a cmdlet keeps the access inside
+ # compiled code, where the language mode does not reach.
+ $waitError = $null
+ Wait-Process -InputObject $proc -Timeout $seconds -ErrorAction SilentlyContinue -ErrorVariable waitError
+ if ($waitError) {
+ $script:NvidiaPythonProbeTimedOut = $true
+ try { Stop-Process -InputObject $proc -Force -ErrorAction SilentlyContinue } catch { }
+ # Let the killed child release its redirected files before the finally deletes them.
+ Wait-Process -InputObject $proc -Timeout 2 -ErrorAction SilentlyContinue
+ return ""
+ }
+ $raw = "$(Get-Content -LiteralPath $outFile -Raw -ErrorAction SilentlyContinue)"
+ } catch { return "" }
+ finally {
+ foreach ($stale in @($scriptFile, $outFile, $errFile)) {
+ Remove-Item -LiteralPath $stale -Force -ErrorAction SilentlyContinue
+ }
+ }
+ return "$raw".Trim()
+}
+
# "source;cudaMajor;cudaMinor;cap,cap" from NVML, else the CUDA driver API; "" when neither
# answers. Versions are major*1000 + minor*10. One runspace + deadline per reader: a shared one let slow NVML starve CUDA.
+# A wedged driver can block inside the library, and each deadline leaves its runspace behind.
+# The Python rung below spends what the emitted rung left of ONE budget (a per-reader bound for
+# NVML plus one for CUDA), so the worst-case wall clock is the emitted rung's own. Its first child
+# reads NVML then the CUDA driver API under at most one per-reader bound; only when that child is
+# killed at its bound (a hung NVML) and at least 2 s remain does a second child read the CUDA
+# driver API alone with the rest, so a hung NVML no longer starves a healthy CUDA driver API.
function Read-NvidiaLibraryRaw {
param([int]$TimeoutMs = 30000)
+ $deadline = (Get-Date).AddMilliseconds($TimeoutMs * 2)
+ $native = ""
$type = Get-NvidiaLibraryProbeType
- if (-not $type) { return "" }
- $reader = {
- param($T, $Which)
- function Read-Nvml {
- if ($T::nvmlInit_v2() -ne 0) { return "" }
- try {
- [uint32]$count = 0
- if ($T::nvmlDeviceGetCount_v2([ref]$count) -ne 0 -or $count -eq 0) { return "" }
+ if ($type) {
+ $reader = {
+ param($T, $Which)
+ function Read-Nvml {
+ if ($T::nvmlInit_v2() -ne 0) { return "" }
+ try {
+ [uint32]$count = 0
+ if ($T::nvmlDeviceGetCount_v2([ref]$count) -ne 0 -or $count -eq 0) { return "" }
+ [int]$ver = 0
+ if ($T::nvmlSystemGetCudaDriverVersion_v2([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }
+ $caps = @()
+ for ([uint32]$i = 0; $i -lt $count; $i++) {
+ [IntPtr]$dev = [IntPtr]::Zero; [int]$major = 0; [int]$minor = 0
+ # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.
+ if ($T::nvmlDeviceGetHandleByIndex_v2($i, [ref]$dev) -ne 0) { return "" }
+ if ($T::nvmlDeviceGetCudaComputeCapability($dev, [ref]$major, [ref]$minor) -ne 0) { return "" }
+ $caps += "$major.$minor"
+ }
+ return "nvml;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
+ } finally { $null = $T::nvmlShutdown() }
+ }
+ function Read-Cuda {
+ # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one,
+ # so a hidden pre-Turing card still caps the family. cuInit reads the mask once.
+ $saved = $env:CUDA_VISIBLE_DEVICES
+ Remove-Item Env:CUDA_VISIBLE_DEVICES -ErrorAction SilentlyContinue
+ try { $init = $T::cuInit([uint32]0) } finally { if ($null -ne $saved) { $env:CUDA_VISIBLE_DEVICES = $saved } }
+ if ($init -ne 0) { return "" }
+ [int]$count = 0
+ if ($T::cuDeviceGetCount([ref]$count) -ne 0 -or $count -eq 0) { return "" }
[int]$ver = 0
- if ($T::nvmlSystemGetCudaDriverVersion_v2([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }
+ if ($T::cuDriverGetVersion([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }
$caps = @()
- for ([uint32]$i = 0; $i -lt $count; $i++) {
- [IntPtr]$dev = [IntPtr]::Zero; [int]$major = 0; [int]$minor = 0
- # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.
- if ($T::nvmlDeviceGetHandleByIndex_v2($i, [ref]$dev) -ne 0) { return "" }
- if ($T::nvmlDeviceGetCudaComputeCapability($dev, [ref]$major, [ref]$minor) -ne 0) { return "" }
+ for ($i = 0; $i -lt $count; $i++) {
+ [int]$dev = 0; [int]$major = 0; [int]$minor = 0
+ if ($T::cuDeviceGet([ref]$dev, $i) -ne 0) { return "" }
+ # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.
+ if ($T::cuDeviceGetAttribute([ref]$major, 75, $dev) -ne 0) { return "" }
+ if ($T::cuDeviceGetAttribute([ref]$minor, 76, $dev) -ne 0) { return "" }
$caps += "$major.$minor"
}
- return "nvml;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
- } finally { $null = $T::nvmlShutdown() }
- }
- function Read-Cuda {
- # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one,
- # so a hidden pre-Turing card still caps the family. cuInit reads the mask once.
- $saved = $env:CUDA_VISIBLE_DEVICES
- Remove-Item Env:CUDA_VISIBLE_DEVICES -ErrorAction SilentlyContinue
- try { $init = $T::cuInit([uint32]0) } finally { if ($null -ne $saved) { $env:CUDA_VISIBLE_DEVICES = $saved } }
- if ($init -ne 0) { return "" }
- [int]$count = 0
- if ($T::cuDeviceGetCount([ref]$count) -ne 0 -or $count -eq 0) { return "" }
- [int]$ver = 0
- if ($T::cuDriverGetVersion([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }
- $caps = @()
- for ($i = 0; $i -lt $count; $i++) {
- [int]$dev = 0; [int]$major = 0; [int]$minor = 0
- if ($T::cuDeviceGet([ref]$dev, $i) -ne 0) { return "" }
- # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.
- if ($T::cuDeviceGetAttribute([ref]$major, 75, $dev) -ne 0) { return "" }
- if ($T::cuDeviceGetAttribute([ref]$minor, 76, $dev) -ne 0) { return "" }
- $caps += "$major.$minor"
- }
- return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
- }
- $r = ""
- try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }
- return "$r"
- }
- foreach ($which in @("nvml", "cuda")) {
- $ps = $null; $handle = $null; $r = ""
- try {
- $ps = [powershell]::Create()
- $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)
- $handle = $ps.BeginInvoke()
- if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {
- $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"
+ return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"
}
- } catch { $r = "" }
- finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }
- if ($r) { return $r }
- }
- return ""
+ $r = ""
+ try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }
+ return "$r"
+ }
+ foreach ($which in @("nvml", "cuda")) {
+ $ps = $null; $handle = $null; $r = ""
+ try {
+ $ps = [powershell]::Create()
+ $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)
+ $handle = $ps.BeginInvoke()
+ if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {
+ $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"
+ }
+ } catch { $r = "" }
+ finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }
+ if ($r) { $native = $r; break }
+ }
+ }
+ if ($native) { return $native }
+ $remainingMs = [int]($deadline - (Get-Date)).TotalMilliseconds
+ # Not worth a child process we cannot wait out; the emitted rung already spent the budget.
+ if ($remainingMs -lt 3000) { return "" }
+ # Each child gets at most one per-reader bound, compared by hand: CLM refuses [math]. 2 s of
+# every child's share is kept back for reaping it if it hangs, so the pair ends inside the deadline.
+ $childMs = $remainingMs - 2000; if ($childMs -gt $TimeoutMs) { $childMs = $TimeoutMs }
+ $raw = ""
+ try { $raw = Read-NvidiaLibraryRawViaPython -TimeoutMs $childMs } catch { return "" }
+ # Only a first child killed at its bound (a hung NVML) earns a CUDA-only child, with what is left.
+ if (-not $script:NvidiaPythonProbeTimedOut) { return $raw }
+ $remainingMs = [int]($deadline - (Get-Date)).TotalMilliseconds
+ if ($remainingMs -lt 3000) { return "" }
+ $childMs = $remainingMs - 2000; if ($childMs -gt $TimeoutMs) { $childMs = $TimeoutMs }
+ try { return (Read-NvidiaLibraryRawViaPython -TimeoutMs $childMs -SkipNvml) } catch { return "" }
}
# NVIDIA inventory from the driver's own libraries (NVML, then the CUDA driver API), for a
@@ -3495,11 +3749,18 @@ if (-not $HasNvidiaSmi) {
}
}
if (-not $HasNvidiaSmi -and (Get-NvidiaLibraryInventory)) {
- # The driver lists a GPU nvidia-smi cannot: the gates below read this as "GPU present",
- # and the consumers stop asking nvidia-smi, absent or rejected, for the rest of the run.
- $HasNvidiaSmi = $true
- $script:NvidiaSmiRejected = $true
- Write-StudioLine " NVIDIA GPU found through the driver library; nvidia-smi is unavailable" -ForegroundColor Gray
+ # A driver too old for any CUDA wheel (below 11) is no GPU this route can serve: the Intel and
+ # AMD routes still get their turn instead of the install falling to CPU.
+ if ((Get-NvidiaLibraryInventory).CudaMajor -ge 11) {
+ # The driver lists a GPU nvidia-smi cannot: the gates below read this as "GPU present",
+ # and the consumers stop asking nvidia-smi, absent or rejected, for the rest of the run.
+ $HasNvidiaSmi = $true
+ $script:NvidiaSmiRejected = $true
+ Write-StudioLine " NVIDIA GPU found through the driver library; nvidia-smi is unavailable" -ForegroundColor Gray
+ } else {
+ # Still a physical NVIDIA GPU: an installed llama.cpp prebuilt it runs is kept, as before.
+ $script:NvidiaDriverLibraryOnly = $true
+ }
}
# nvidia-smi was already resolved above and never asked which card it found, so the banner
# said "NVIDIA GPU detected" on every NVIDIA host alike. compute_cap is the counterpart of the
@@ -9270,6 +9531,8 @@ if ($LocalLlamaCppLinked) {
@("windows-rocm", "windows-hip", "windows-arm64", "windows-vulkan")
} else { @("windows-rocm", "windows-hip", "windows-vulkan") }
$expectedKinds = if ($HasROCm -or $script:ROCmGfxArch) { $_rocmKinds } elseif ($_nvidiaEvidence) { $_nvidiaKinds } else { @("windows-cpu", "windows-arm64", "windows-vulkan") }
+ # A pre-CUDA 11 library still proves physical NVIDIA presence, even when AMD/Intel selects torch.
+ if ($script:NvidiaDriverLibraryOnly) { $expectedKinds += $_nvidiaKinds }
if ($existingKind -and ($existingKind -notin $expectedKinds)) {
substep "Removing mismatched llama.cpp install (found '$existingKind', need one of: $($expectedKinds -join ', '))..."
Remove-Item -Recurse -Force -LiteralPath $LlamaCppDir -ErrorAction SilentlyContinue
diff --git a/tests/python/test_installer_embedded_nvidia_probe.py b/tests/python/test_installer_embedded_nvidia_probe.py
new file mode 100644
index 00000000000..0dfb6dfdd95
--- /dev/null
+++ b/tests/python/test_installer_embedded_nvidia_probe.py
@@ -0,0 +1,322 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+"""Behaviour of the ctypes NVIDIA probe embedded in install.ps1 and studio/setup.ps1.
+
+The probe recovers the CUDA version AND the per-device compute capabilities on hosts where the
+installer cannot emit a P/Invoke type (Constrained Language Mode, WDAC, Dynamic Code Security).
+Capabilities are the part WMI cannot supply: they feed ``$CudaArch`` into
+``-DCMAKE_CUDA_ARCHITECTURES`` for the llama.cpp source build (#5854) and the pre-Turing cap.
+
+``tests/studio/test_nvidia_python_probe_parity.ps1`` pins the probe's TEXT against
+``studio/nvidia_probe.py`` and against the other copy. This file pins its BEHAVIOUR, by running
+it against a fake NVML / CUDA library. The two are complementary: a defect edited identically
+into both copies is invisible to a parity check and caught here.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import io
+import ntpath
+import os
+import re
+import sys
+import types
+from contextlib import redirect_stdout
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+POSIX_ONLY = pytest.mark.skipif(
+ os.name == "nt", reason = "the library names below are the POSIX ones"
+)
+LINUX_NVML = "libnvidia-ml.so.1"
+LINUX_CUDA = "libcuda.so.1"
+NVML_OUT = "nvml;12;6;8.9,8.9"
+CUDA_OUT = "cuda;12;6;9.0,9.0"
+
+
+def _embedded(path: Path) -> str:
+ # Anchored on the function: both files carry other here-strings, and the first one is a
+ # different probe.
+ text = path.read_text(encoding = "utf-8")
+ at = text.index("function Read-NvidiaLibraryRawViaPython")
+ match = re.search(r"\$probeSource = @'\n(.*?)\n'@\n", text[at:], re.S)
+ assert match, f"no probe here-string in {path}"
+ return match.group(1)
+
+
+INSTALL_PROBE = _embedded(ROOT / "install.ps1")
+SETUP_PROBE = _embedded(ROOT / "studio" / "setup.ps1")
+
+
+def test_both_copies_are_byte_identical():
+ # Leading whitespace IS Python syntax, so this is a byte comparison, not a stripped one.
+ assert INSTALL_PROBE == SETUP_PROBE
+
+
+def test_the_probe_is_stdlib_only():
+ imports = set(re.findall(r"(?m)^\s*(?:import|from)\s+([A-Za-z_][\w.]*)", INSTALL_PROBE))
+ assert imports <= {"ctypes", "os", "sys"}, imports
+
+
+def test_the_probe_compiles():
+ compile(INSTALL_PROBE, "embedded_probe.py", "exec")
+
+
+class _Fn:
+ # The probe sets .restype / .argtypes on every export.
+ def __init__(self, impl):
+ self._impl, self.restype, self.argtypes = impl, None, None
+
+ def __call__(self, *args):
+ return self._impl(*args)
+
+
+class _Lib:
+ def __init__(self, table):
+ self._table = table
+ self.calls: list[str] = []
+
+ def __getattr__(self, name):
+ if name not in self._table:
+ raise AttributeError(name)
+ return _Fn(lambda *args: (self.calls.append(name), self._table[name](*args))[1])
+
+
+def _run(
+ libs,
+ hints = ("", ""),
+ loaded = None,
+) -> str:
+ """Execute the probe with ctypes.CDLL replaced, and return what it wrote to stdout."""
+
+ def fake_cdll(name):
+ if loaded is not None:
+ loaded.append(name)
+ if libs.get(name) is None:
+ raise OSError(f"cannot load {name}")
+ return libs[name]
+
+ shim = types.ModuleType("ctypes")
+ for attr in ("c_int", "c_uint", "c_void_p", "POINTER"):
+ setattr(shim, attr, getattr(ctypes, attr))
+ shim.CDLL = fake_cdll
+ # Identity byref, so a fake export can write through to the caller's c_int.
+ shim.byref = lambda obj: obj
+ buffer = io.StringIO()
+ with pytest.MonkeyPatch.context() as mp, redirect_stdout(buffer):
+ mp.setitem(sys.modules, "ctypes", shim)
+ mp.setenv("UNSLOTH_NVML_HINT", hints[0])
+ mp.setenv("UNSLOTH_CUDA_HINT", hints[1])
+ exec(compile(INSTALL_PROBE, "embedded_probe.py", "exec"), {})
+ return buffer.getvalue()
+
+
+def _put(value):
+ def write(out):
+ out.value = value
+ return 0
+
+ return write
+
+
+def _nvml(
+ count = 2,
+ packed = 12060,
+ caps = ((8, 9), (8, 9)),
+ bad_handle = -1,
+ bad_cap = -1,
+ init = 0,
+):
+ """A fake libnvidia-ml. bad_handle / bad_cap make that device index fail."""
+
+ def get_handle(index, out):
+ if index == bad_handle:
+ return 999
+ out.value = 0x1000 + index
+ return 0
+
+ def get_cap(handle, major, minor):
+ index = handle.value - 0x1000 if handle.value else 0
+ if index == bad_cap:
+ return 999
+ major.value, minor.value = caps[index]
+ return 0
+
+ return _Lib(
+ {
+ "nvmlInit_v2": lambda: init,
+ "nvmlShutdown": lambda: 0,
+ "nvmlDeviceGetCount_v2": _put(count),
+ "nvmlSystemGetCudaDriverVersion_v2": _put(packed),
+ "nvmlDeviceGetHandleByIndex_v2": get_handle,
+ "nvmlDeviceGetCudaComputeCapability": get_cap,
+ }
+ )
+
+
+def _cuda(
+ count = 2,
+ packed = 12060,
+ caps = ((9, 0), (9, 0)),
+ init = 0,
+ seen = None,
+ attrs = None,
+):
+ def cu_init(flags):
+ if seen is not None:
+ # What the driver would see: the mask at the moment cuInit reads it.
+ seen.append(os.environ.get("CUDA_VISIBLE_DEVICES"))
+ return init
+
+ def get_device(out, index):
+ out.value = index
+ return 0
+
+ def get_attr(out, attr, device):
+ if attrs is not None:
+ attrs.append(attr)
+ out.value = caps[device.value][0 if attr == 75 else 1]
+ return 0
+
+ return _Lib(
+ {
+ "cuInit": cu_init,
+ "cuDriverGetVersion": _put(packed),
+ "cuDeviceGetCount": _put(count),
+ "cuDeviceGet": get_device,
+ "cuDeviceGetAttribute": get_attr,
+ }
+ )
+
+
+@POSIX_ONLY
+ "nvml, cuda, expected",
+ [
+ pytest.param({}, None, NVML_OUT, id = "healthy_two_gpu_nvml"),
+ pytest.param(
+ {"packed": 13010}, None, "nvml;13;1;8.9,8.9", id = "version_is_major_1000_minor_10"
+ ),
+ # A partial list would misreport the lowest capability and so the pre-Turing cap.
+ pytest.param({"bad_handle": 1}, None, "", id = "unreadable_handle_voids_source"),
+ pytest.param({"bad_cap": 1}, None, "", id = "unreadable_capability_voids_source"),
+ pytest.param({"count": 0}, None, "", id = "no_devices"),
+ # Below 1000 the packed form cannot encode a real CUDA version.
+ pytest.param({"packed": 999}, None, "", id = "nonsense_driver_version"),
+ pytest.param({"init": 1}, {}, CUDA_OUT, id = "failed_nvml_init_falls_through"),
+ pytest.param(None, {}, CUDA_OUT, id = "nvml_absent_uses_driver_api"),
+ pytest.param({}, {}, NVML_OUT, id = "nvml_preferred_when_both_answer"),
+ pytest.param(None, None, "", id = "neither_library"),
+ pytest.param(None, {"init": 1}, "", id = "failed_cuinit"),
+ ],
+)
+def test_probe_output(nvml, cuda, expected):
+ libs = {
+ LINUX_NVML: None if nvml is None else _nvml(**nvml),
+ LINUX_CUDA: None if cuda is None else _cuda(**cuda),
+ }
+ assert _run(libs) == expected
+
+
+@POSIX_ONLY
+def test_a_partial_read_still_shuts_nvml_down():
+ lib = _nvml(bad_cap = 1)
+ _run({LINUX_NVML: lib, LINUX_CUDA: None})
+ assert "nvmlShutdown" in lib.calls
+
+
+@POSIX_ONLY
[email protected]("value, nvml_first", [("1", False), ("0", True)])
+def test_the_skip_nvml_switch(monkeypatch, value, nvml_first):
+ # "1" is the CUDA-only retry after a child NVML held past its deadline: NVML is never loaded
+ # or initialised, and the driver API is. Any other value still reads NVML first.
+ monkeypatch.setenv("UNSLOTH_NVIDIA_PROBE_SKIP_NVML", value)
+ nvml, cuda = _nvml(), _cuda()
+ out = _run({LINUX_NVML: nvml, LINUX_CUDA: cuda})
+ assert ("nvmlInit_v2" in nvml.calls) is nvml_first
+ assert out == (NVML_OUT if nvml_first else CUDA_OUT)
+ if not nvml_first:
+ assert "cuInit" in cuda.calls and "cuDriverGetVersion" in cuda.calls
+
+
+@POSIX_ONLY
+ "mask", ["1", None], ids = ["mask_hidden_then_restored", "absent_mask_not_invented"]
+)
+def test_cuinit_sees_the_physical_inventory(monkeypatch, mask):
+ # A hidden pre-Turing card still caps the family, so cuInit must see no mask.
+ if mask is None:
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ else:
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ seen: list = []
+ _run({LINUX_NVML: None, LINUX_CUDA: _cuda(seen = seen)})
+ assert seen == [None]
+ assert os.environ.get("CUDA_VISIBLE_DEVICES") == mask
+
+
+@POSIX_ONLY
+def test_the_capability_attributes_are_75_and_76():
+ # A wrong attribute number returns a plausible integer rather than an error.
+ attrs: list = []
+ _run({LINUX_NVML: None, LINUX_CUDA: _cuda(attrs = attrs)})
+ assert set(attrs) == {75, 76}
+
+
+@POSIX_ONLY
+def test_the_output_matches_what_the_powershell_side_parses():
+ """Get-NvidiaLibraryInventory splits on ';' and wants exactly four fields."""
+ parts = _run({LINUX_NVML: _nvml()}).split(";")
+ assert len(parts) == 4
+ assert parts[0] in ("nvml", "cuda")
+ assert int(parts[1]) >= 1
+ assert all(re.match(r"^\d+\.\d+$", cap) for cap in parts[3].split(","))
+
+
+def _run_windows(monkeypatch, libs, hints, loaded):
+ monkeypatch.setattr(os, "name", "nt")
+ monkeypatch.setattr(os, "path", ntpath)
+ try:
+ return _run(libs, hints = hints, loaded = loaded)
+ finally:
+ monkeypatch.undo()
+
+
+SYSTEM32_NVML = r"C:\Windows\System32\nvml.dll"
+SYSTEM32_CUDA = r"C:\Windows\System32\nvcuda.dll"
+
+
+def test_windows_loads_only_the_driver_paths_it_is_handed(monkeypatch):
+ order: list[str] = []
+ _run_windows(monkeypatch, {}, (SYSTEM32_NVML, SYSTEM32_CUDA), order)
+ assert order == [SYSTEM32_NVML, SYSTEM32_CUDA]
+
+
+def test_a_cuda_stand_in_beside_python_is_not_an_nvidia_gpu(monkeypatch):
+ # ZLUDA ships nvcuda.dll and nvml.dll that answer as a driver on an AMD host (#11736):
+ # without the driver in System32 the probe must say nothing, not report a GPU.
+ order: list[str] = []
+ stand_in = {"nvml.dll": _nvml(), "nvcuda.dll": _cuda()}
+ assert _run_windows(monkeypatch, stand_in, (SYSTEM32_NVML, SYSTEM32_CUDA), order) == ""
+ assert "nvml.dll" not in order and "nvcuda.dll" not in order
+
+
+def test_a_relative_hint_is_never_loaded(monkeypatch):
+ order: list[str] = []
+ stand_in = {"nvml.dll": _nvml(), "nvcuda.dll": _cuda()}
+ assert _run_windows(monkeypatch, stand_in, ("nvml.dll", "nvcuda.dll"), order) == ""
+ assert order == []
+
+
+def test_the_real_driver_in_system32_still_answers(monkeypatch):
+ order: list[str] = []
+ out = _run_windows(
+ monkeypatch,
+ {SYSTEM32_NVML: None, SYSTEM32_CUDA: _cuda()},
+ (SYSTEM32_NVML, SYSTEM32_CUDA),
+ order,
+ )
+ assert out == CUDA_OUT
diff --git a/tests/studio/test_nvidia_library_inventory.ps1 b/tests/studio/test_nvidia_library_inventory.ps1
index 8377e1f6b97..b2f09880350 100644
--- a/tests/studio/test_nvidia_library_inventory.ps1
+++ b/tests/studio/test_nvidia_library_inventory.ps1
@@ -249,6 +249,13 @@ $HasROCm = $true
Check "a Vulkan prebuilt is kept for any GPU vendor" ((Get-GpuPrebuiltToKeepOverSourceBuild -InstallDir $install) -eq "vulkan")
$HasROCm = $false
Check "a Vulkan prebuilt with no GPU at all is not kept" ((Get-GpuPrebuiltToKeepOverSourceBuild -InstallDir $install) -eq "")
+# A driver library below CUDA 11 does not promote NVIDIA, but the card still runs Vulkan.
+$script:NvidiaDriverLibraryOnly = $true
+Check "a Vulkan prebuilt is kept for an NVIDIA GPU only the driver library found" ((Get-GpuPrebuiltToKeepOverSourceBuild -InstallDir $install) -eq "vulkan")
+$script:NvidiaDriverLibraryOnly = $false
+# Read on every keep, so each run resets it: Set-StrictMode, and a rerun in the same session.
+Check "setup.ps1 resets the driver-library-only flag with the other per-run state" (
+ (Get-Content -LiteralPath $setupPs1 -Raw) -match '(?m)^\$script:NvidiaSmiRejected = \$false\r?\n\$script:NvidiaDriverLibraryOnly = \$false')
$script:FakeIntelAdapters = @("Intel(R) UHD Graphics 770")
Check "a Vulkan prebuilt is kept for an Intel adapter that is not an XPU part" ((Get-GpuPrebuiltToKeepOverSourceBuild -InstallDir $install) -eq "vulkan")
$script:FakeIntelAdapters = @()
@@ -301,6 +308,46 @@ $script:FakeSmiTimeouts = 0
$script:FakeSmiStdout = "| CUDA Version: 12.6 |"
Check "a readable banner is still authoritative" ((Get-TorchIndexUrl) -eq "https://download.pytorch.org/whl/cu126")
+# The promotion runs as written in each file: a driver below CUDA 11 must not claim the host from Intel/AMD.
+function Get-PromotionBlock([string]$Text) {
+ $at = $Text.IndexOf('if (-not $HasNvidiaSmi -and (Get-NvidiaLibraryInventory)) {')
+ $ast = [System.Management.Automation.Language.Parser]::ParseInput($Text.Substring($at), [ref]$null, [ref]$null)
+ return $ast.EndBlock.Statements[0].Extent.Text
+}
+foreach ($file in @($installPs1, $setupPs1)) {
+ $block = Get-PromotionBlock (Get-Content -LiteralPath $file -Raw)
+ foreach ($case in @(@{ Major = 10; Want = $false }, @{ Major = 11; Want = $true }, @{ Major = 12; Want = $true })) {
+ $script:FakeInventory = @{ Source = "nvml"; CudaMajor = $case.Major; CudaMinor = 2; ComputeCaps = @("6.1"); Count = 1 }
+ function Get-NvidiaLibraryInventory { param([int]$TimeoutSec = 10) return $script:FakeInventory }
+ $HasNvidiaSmi = $false
+ $script:NvidiaDriverLibraryOnly = $false
+ Invoke-Expression $block
+ Check "$(Split-Path -Leaf $file): a CUDA $($case.Major) driver library $(if ($case.Want) { 'promotes' } else { 'does not promote' }) NVIDIA" ($HasNvidiaSmi -eq $case.Want)
+ if ($file -eq $setupPs1) {
+ Check "setup.ps1: a CUDA $($case.Major) driver library still counts as an NVIDIA GPU for keeping a prebuilt" (
+ ($HasNvidiaSmi -or $script:NvidiaDriverLibraryOnly) -eq $true)
+ }
+ }
+}
+
+# The update's kind guard runs before the keeper and deleted the bundle the keeper would have kept.
+$setupText = Get-Content -LiteralPath $setupPs1 -Raw
+$from = $setupText.IndexOf('$_nvidiaKinds = if (Test-WinArm64Venv) {')
+$to = $setupText.IndexOf('if ($existingKind -and ($existingKind -notin $expectedKinds))', $from)
+$kindBlock = $setupText.Substring($from, $to - $from)
+function Test-WinArm64Venv { $false }
+function Get-PinnedTorchIndexUrl { $null }
+function Get-PersistedWoaTorchIndex { $null }
+function Get-WoaTorchIndexMarker { $null }
+$WinArm64EffectiveTorchIndexUrl = $null; $HasROCm = $false; $script:ROCmGfxArch = $null; $HasNvidiaSmi = $false
+foreach ($case in @(@{ Flag = $true; Want = $true }, @{ Flag = $false; Want = $false })) {
+ $script:NvidiaDriverLibraryOnly = $case.Flag
+ Invoke-Expression $kindBlock
+ Check "the update's kind guard $(if ($case.Want) { 'keeps' } else { 'does not keep' }) a CUDA bundle $(if ($case.Flag) { 'for an NVIDIA GPU only a pre-CUDA 11 driver library found' } else { 'with no NVIDIA evidence' })" (
+ ($expectedKinds -contains "windows-cuda") -eq $case.Want)
+}
+$script:NvidiaDriverLibraryOnly = $false
+
Write-Host ""
if ($failures -gt 0) { Write-Host "$failures check(s) failed" -ForegroundColor Red; exit 1 }
Write-Host "All checks passed"
diff --git a/tests/studio/test_nvidia_python_probe_parity.ps1 b/tests/studio/test_nvidia_python_probe_parity.ps1
new file mode 100644
index 00000000000..c8a3146da20
--- /dev/null
+++ b/tests/studio/test_nvidia_python_probe_parity.ps1
@@ -0,0 +1,476 @@
+#!/usr/bin/env pwsh
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+# The NVIDIA inventory's Python rung: CPython's ctypes makes the same NVML and CUDA driver calls
+# the emitted type makes, so a host that cannot emit a type (Constrained Language Mode, WDAC,
+# Dynamic Code Security) still gets the CUDA version AND the per-device compute capabilities.
+# Capabilities are the part WMI cannot supply: they feed $CudaArch into
+# -DCMAKE_CUDA_ARCHITECTURES for the llama.cpp source build (#5854) and the pre-Turing cap.
+#
+# This file exists to stop three kinds of drift:
+# 1. the two copies of the shared region drifting apart,
+# 2. the embedded probe drifting from studio/nvidia_probe.py, which makes the same calls,
+# 3. the Python rung being promoted ahead of the emitted one, or spending a second budget.
+# Run: pwsh -NoProfile -File tests/studio/test_nvidia_python_probe_parity.ps1
+
+$ErrorActionPreference = "Stop"
+$root = (Resolve-Path ([System.IO.Path]::Combine($PSScriptRoot, "..", ".."))).Path
+
+$failures = 0
+function Check($name, $cond) {
+ if ($cond) { Write-Host " PASS $name" }
+ else { Write-Host " FAIL $name" -ForegroundColor Red; $script:failures++ }
+}
+
+function Get-HelperSources($path, $names) {
+ $tokens = $null; $errors = $null
+ $ast = [System.Management.Automation.Language.Parser]::ParseFile($path, [ref]$tokens, [ref]$errors)
+ if ($errors) { $errors | ForEach-Object { $_.ToString() }; throw "$path has parse errors" }
+ $out = @()
+ foreach ($name in $names) {
+ $fn = $ast.FindAll({ param($n)
+ $n -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $n.Name -eq $name
+ }, $true)
+ if ($fn.Count -lt 1) { throw "expected $name in $path, found none" }
+ $out += $fn[0].Extent.Text
+ }
+ # A single-element return unrolls to a bare string, and [0] would then be its first
+ # character. Callers wrap in @(), and this comment is why.
+ return $out
+}
+
+$installPs1 = Join-Path $root "install.ps1"
+$setupPs1 = Join-Path $root "studio\setup.ps1"
+$probePy = Join-Path $root "studio\nvidia_probe.py"
+
+# The embedded Python is the text between @' and '@ inside Read-NvidiaLibraryRawViaPython.
+# Both files carry other here-strings, so anchor on the function rather than the first match.
+# The variable is $probeSource, not $probe: install.ps1 and setup.ps1 each already carry a
+# $probe here-string for the emit probe, and reusing the name made an older suite read this one.
+function Get-EmbeddedProbe($path) {
+ $text = [System.IO.File]::ReadAllText($path)
+ $at = $text.IndexOf("function Read-NvidiaLibraryRawViaPython")
+ if ($at -lt 0) { throw "no Read-NvidiaLibraryRawViaPython in $path" }
+ $open = $text.IndexOf("`$probeSource = @'", $at)
+ if ($open -lt 0) { throw "no probe here-string in $path" }
+ $bodyStart = $text.IndexOf("`n", $open) + 1
+ $close = $text.IndexOf("`n'@", $bodyStart)
+ if ($close -lt 0) { throw "unterminated probe here-string in $path" }
+ return $text.Substring($bodyStart, $close - $bodyStart)
+}
+
+Write-Host ""
+Write-Host "=== the two copies agree ==="
+
+$blockNames = @(
+ "Get-NvidiaNvmlLibraryPath",
+ "Get-NvidiaLibraryProbeType",
+ "Read-NvidiaLibraryRawViaPython",
+ "Read-NvidiaLibraryRaw",
+ "Get-NvidiaLibraryInventory",
+ "Get-NvidiaSystem32Dir"
+)
+$installParts = @(Get-HelperSources $installPs1 $blockNames)
+$setupParts = @(Get-HelperSources $setupPs1 $blockNames)
+# install.ps1 nests its helpers one level deeper; compare the two copies without indentation.
+$strip = { param($text) ($text -split "`n" | ForEach-Object { $_.TrimStart() }) -join "`n" }
+for ($k = 0; $k -lt $blockNames.Count; $k++) {
+ Check "install.ps1 and setup.ps1 carry the same $($blockNames[$k])" `
+ ((& $strip $installParts[$k]) -eq (& $strip $setupParts[$k]))
+}
+
+$installProbe = Get-EmbeddedProbe $installPs1
+$setupProbe = Get-EmbeddedProbe $setupPs1
+# Stripping indentation would hide a real defect here: leading whitespace IS the Python syntax,
+# so the two copies must match byte for byte, not merely after a TrimStart.
+Check "the embedded Python is byte-identical in both files" ($installProbe -ceq $setupProbe)
+Check "the embedded Python sits at column 0, so its indentation survives both files" `
+ (($installProbe -split "`n" | Where-Object { $_ -match '^import ctypes' }).Count -eq 1)
+
+Write-Host ""
+Write-Host "=== the embedded probe matches studio/nvidia_probe.py ==="
+
+$referenceProbe = [System.IO.File]::ReadAllText($probePy)
+
+# Every native symbol the emitted type imports, and the two attribute numbers. If the embedded
+# probe and nvidia_probe.py ever disagree on one of these, they are probing different things.
+$symbols = @(
+ "nvmlInit_v2", "nvmlShutdown", "nvmlSystemGetCudaDriverVersion_v2",
+ "nvmlDeviceGetCount_v2", "nvmlDeviceGetHandleByIndex_v2", "nvmlDeviceGetCudaComputeCapability",
+ "cuInit", "cuDriverGetVersion", "cuDeviceGetCount", "cuDeviceGet", "cuDeviceGetAttribute"
+)
+foreach ($symbol in $symbols) {
+ Check "both the embedded probe and nvidia_probe.py call $symbol" `
+ (($installProbe -match [regex]::Escape($symbol)) -and ($referenceProbe -match [regex]::Escape($symbol)))
+ Check "the emitted type imports $symbol too, so all three agree" `
+ ($installParts[1] -match [regex]::Escape($symbol))
+}
+
+# CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR / _MINOR. A wrong number here returns a
+# plausible integer rather than an error, which is exactly the drift a test has to catch.
+Check "the embedded probe reads attribute 75 for the capability major" ($installProbe -match 'byref\(major\), 75,')
+Check "the embedded probe reads attribute 76 for the capability minor" ($installProbe -match 'byref\(minor\), 76,')
+Check "nvidia_probe.py reads the same two attribute numbers" `
+ (($referenceProbe -match 'byref\(major\), 75,') -and ($referenceProbe -match 'byref\(minor\), 76,'))
+
+# The packed CUDA version arithmetic, shared by all three implementations.
+Check "the embedded probe unpacks the version as major*1000 + minor*10" `
+ ($installProbe -match 'packed // 1000, \(packed % 1000\) // 10')
+Check "nvidia_probe.py unpacks the version identically" `
+ ($referenceProbe -match 'packed // 1000, \(packed % 1000\) // 10')
+Check "the emitted rung unpacks the version identically" `
+ (($installParts[3] -match 'Floor\(\$ver / 1000\)') -and ($installParts[3] -match 'Floor\(\(\$ver % 1000\) / 10\)'))
+
+Check "the embedded probe is stdlib-only" `
+ (($installProbe -match '(?m)^import ctypes, os, sys$') -and ($installProbe -notmatch '(?m)^\s*(import|from)\s+(?!ctypes|os|sys)'))
+
+Write-Host ""
+Write-Host "=== the rung stays underneath the emitted one ==="
+
+$rawBlock = & $strip $setupParts[3]
+Check "the emitted rung is attempted first" `
+ ($rawBlock.IndexOf("Get-NvidiaLibraryProbeType") -lt $rawBlock.IndexOf("Read-NvidiaLibraryRawViaPython"))
+Check "Python is only reached when the emitted rung returned nothing" `
+ ($rawBlock -match 'if \(\$native\) \{ return \$native \}')
+# One budget, not two: a driver that wedges both emitted readers must not then buy a further
+# child process. The budget is one per-reader bound for NVML plus one for CUDA.
+Check "both rungs share one deadline" ($rawBlock -match '\$deadline = \(Get-Date\)\.AddMilliseconds\(\$TimeoutMs \* 2\)')
+Check "each emitted reader has its own deadline" `
+ (($rawBlock -match 'foreach \(\$which in @\("nvml", "cuda"\)\)') -and ($rawBlock -match 'WaitOne\(\$TimeoutMs\)'))
+Check "the per-reader bound defaults to 30s" ($rawBlock -match 'param\(\[int\]\$TimeoutMs = 30000\)')
+# The Python rung spends what the emitted rung left, at most one per-reader bound per child: one
+# child reads NVML then CUDA, and only a child killed at its bound earns a second, CUDA-only child
+# with what is still left. The behaviour is driven below; these pin the shape.
+Check "Python gets only what the emitted rung left, capped at one bound" `
+ (($rawBlock -match '\$childMs = \$remainingMs - 2000; if \(\$childMs -gt \$TimeoutMs\) \{ \$childMs = \$TimeoutMs \}') -and
+ ($rawBlock -match '\$raw = Read-NvidiaLibraryRawViaPython -TimeoutMs \$childMs\b'))
+Check "the reader itself avoids [math], which CLM refuses" ($rawBlock -notmatch '\[math\]::Min')
+Check "only a timed-out first child earns the CUDA-only retry" `
+ (($rawBlock -match 'if \(-not \$script:NvidiaPythonProbeTimedOut\) \{ return \$raw \}') -and
+ ($rawBlock -match 'Read-NvidiaLibraryRawViaPython -TimeoutMs \$childMs -SkipNvml'))
+Check "an exhausted budget skips the child process entirely" ($rawBlock -match 'if \(\$remainingMs -lt 3000\) \{ return "" \}')
+
+$pyBlock = & $strip $setupParts[2]
+# The banned constructs are NAMED in this rung's own comments, explaining why they are absent.
+# Matching the raw text would fail on the explanation rather than on the code, so the CLM rows
+# below run against comment-free source. (Found the honest way: they failed on the comments.)
+$pyCode = ($pyBlock -split "`n" | Where-Object { $_.TrimStart() -notmatch '^#' }) -join "`n"
+Check "the rung compiles nothing" ($pyCode -notmatch 'Add-Type')
+Check "the rung emits no type" ($pyCode -notmatch 'New-StudioEmittedNativeType|DefinePInvokeMethod')
+# Constrained Language Mode is one of the two policies that make the emitted rung decline, so a
+# launcher this rung cannot use on a CLM host would be absent from the population it exists for.
+Check "the launcher avoids ProcessStartInfo, which CLM refuses" ($pyCode -notmatch 'ProcessStartInfo')
+Check "the launcher avoids [Process]::Start, which CLM refuses" ($pyCode -notmatch '\[(System\.)?Diagnostics\.Process\]::Start|\[Process\]::Start')
+Check "the launcher avoids [System.IO.Path], which CLM refuses" ($pyCode -notmatch '\[System\.IO\.Path\]')
+Check "the launcher avoids [math], which CLM refuses" ($pyCode -notmatch '\[math\]::')
+# Constrained Language Mode permits property reads only on its documented allowed-type list, and
+# System.Diagnostics.Process and System.IO.FileInfo are on neither. Reading .Id, .HasExited,
+# .ExitCode or a path off a FileInfo throws on precisely the hosts where the emitted rung already
+# declined, which is the whole population this rung serves. Handing the object to a cmdlet keeps
+# the access inside compiled code. This is not hypothetical: the sibling early-python launcher
+# shipped with exactly these reads and failed three checks on windows-latest under 5.1.
+foreach ($blocked in @("\.Id\b", "\.HasExited\b", "\.ExitCode\b", "\.FullName\b", "New-TemporaryFile")) {
+ Check "the launcher avoids $($blocked -replace '\\[.b]', '') , which CLM refuses on a Process or FileInfo" (
+ $pyCode -notmatch $blocked)
+}
+Check "the launcher waits by object, not by pid" ($pyCode -match 'Wait-Process -InputObject \$proc')
+Check "the launcher kills by object too" ($pyCode -match 'Stop-Process -InputObject \$proc')
+Check "a killed child is waited for before its files are deleted" (
+ $pyCode -match 'Stop-Process -InputObject \$proc[^\n]*\n(\s*#[^\n]*\n)?\s*Wait-Process -InputObject \$proc -Timeout \d+')
+Check "the timeout is detected through an error variable" ($pyCode -match '-ErrorVariable waitError')
+# "[ref] - Casting an object to type [ref] ... is not permitted."
+Check "the launcher casts nothing to [ref]" ($pyCode -notmatch '\[ref\]\s*\$')
+# The stripping must not be so eager that it deletes the launcher itself.
+Check "the comment-free source still contains the launcher" ($pyCode -match 'Start-Process -FilePath \$exe')
+Check "the interpreter comes from a per-file hook, not from the shared region" `
+ (($pyBlock -match 'Get-NvidiaProbePythonExe') -and ($pyBlock -notmatch 'function Get-NvidiaProbePythonExe'))
+foreach ($file in @($installPs1, $setupPs1)) {
+ Check "$(Split-Path -Leaf $file) defines its own Get-NvidiaProbePythonExe" `
+ ([System.IO.File]::ReadAllText($file) -match 'function Get-NvidiaProbePythonExe')
+}
+Check "the probe is switchable off" ($pyBlock -match 'UNSLOTH_NVIDIA_PYTHON_PROBE')
+Check "the temp files are always cleaned up" ($pyBlock -match 'finally \{[\s\S]*Remove-Item -LiteralPath \$stale')
+# Windows PowerShell 5.1 appends each native argument verbatim, so anything with a space in it
+# splits and the child silently runs something else. Nothing with a space may reach argv.
+Check "only space-free arguments reach the command line" ($pyCode -match 'ArgumentList @\("-I", "-S", "-"\)')
+Check "the script arrives on stdin, not as a path argument" ($pyCode -match '-RedirectStandardInput \$scriptFile')
+Check "the library hints travel in the environment" `
+ (($pyCode -match '\$env:UNSLOTH_NVML_HINT = \$nvmlHint') -and ($pyCode -match '\$env:UNSLOTH_CUDA_HINT = \$cudaHint'))
+Check "the hint variables are restored afterwards" `
+ ($pyCode -match 'finally \{[\s\S]*Remove-Item Env:UNSLOTH_NVML_HINT')
+Check "the embedded probe reads the hints from the environment" `
+ (($installProbe -match 'os\.environ\.get\("UNSLOTH_NVML_HINT"') -and ($installProbe -match 'os\.environ\.get\("UNSLOTH_CUDA_HINT"'))
+
+Write-Host ""
+Write-Host "=== the installer uses the interpreter this run installed ==="
+
+# Get-StudioEarlyPython memoises, including a MISS. On a fresh host it is first called by the
+# install-lock path, finds no Python, and caches $null for the rest of the run. The inventory is
+# not read until thousands of lines later, by which point this install has created a managed
+# interpreter and a venv. Asking only the cache would decline on exactly the fresh install where
+# nvidia-smi is also most likely absent, and the host would take CPU wheels with a working card.
+$hookSrc = @(Get-HelperSources $installPs1 @("Get-NvidiaProbePythonExe"))[0]
+Invoke-Expression $hookSrc
+$script:EarlyCalled = $false
+function Get-StudioEarlyPython { $script:EarlyCalled = $true; return "/early/python3" }
+
+$hookDir = Join-Path ([System.IO.Path]::GetTempPath()) ("unsloth-hook-" + [guid]::NewGuid().ToString("N"))
+New-Item -ItemType Directory -Path $hookDir | Out-Null
+try {
+ $fakeVenv = Join-Path $hookDir "venv-python"
+ $fakeManaged = Join-Path $hookDir "managed-python"
+ [System.IO.File]::WriteAllText($fakeVenv, "")
+ [System.IO.File]::WriteAllText($fakeManaged, "")
+
+ $VenvPython = $fakeVenv; $ManagedPythonPath = $fakeManaged
+ $script:EarlyCalled = $false
+ Check "the venv interpreter wins when it exists" ((Get-NvidiaProbePythonExe) -eq $fakeVenv)
+ Check "and the memoising ladder is not consulted at all" ($script:EarlyCalled -eq $false)
+
+ # Before the venv is built but after the managed interpreter is installed.
+ $VenvPython = Join-Path $hookDir "not-created-yet"
+ $script:EarlyCalled = $false
+ Check "the managed interpreter is next" ((Get-NvidiaProbePythonExe) -eq $fakeManaged)
+ Check "and it too skips the ladder" ($script:EarlyCalled -eq $false)
+
+ # Neither present: the ladder is the fallback, which is the pre-existing behaviour.
+ $VenvPython = Join-Path $hookDir "nope-1"; $ManagedPythonPath = Join-Path $hookDir "nope-2"
+ $script:EarlyCalled = $false
+ Check "with neither on disk the ladder still answers" ((Get-NvidiaProbePythonExe) -eq "/early/python3")
+ Check "and the ladder really was the source" ($script:EarlyCalled -eq $true)
+
+ # A path recorded but never created must not be handed out: Test-Path is the discriminator,
+ # not whether the variable happens to be set.
+ $VenvPython = $null; $ManagedPythonPath = $null
+ Check "unset variables fall through rather than returning empty" (
+ (Get-NvidiaProbePythonExe) -eq "/early/python3")
+
+ $saved = $env:UNSLOTH_EARLY_PYTHON_PROBE
+ try {
+ $env:UNSLOTH_EARLY_PYTHON_PROBE = "0"
+ $VenvPython = $fakeVenv
+ Check "the kill switch still wins over the venv interpreter" ((Get-NvidiaProbePythonExe) -eq "")
+ } finally {
+ if ($null -eq $saved) { Remove-Item Env:UNSLOTH_EARLY_PYTHON_PROBE -ErrorAction SilentlyContinue }
+ else { $env:UNSLOTH_EARLY_PYTHON_PROBE = $saved }
+ }
+} finally {
+ Remove-Item -LiteralPath $hookDir -Recurse -Force -ErrorAction SilentlyContinue
+}
+
+# The ordering that makes the above reachable at all: both variables are assigned before the
+# first Get-NvidiaLibraryInventory call, or the hook would read $null however it is written.
+$installText = [System.IO.File]::ReadAllText($installPs1)
+$venvAt = $installText.IndexOf('$VenvPython = Join-Path $VenvDir')
+$managedAt = $installText.IndexOf('$ManagedPythonPath = (Resolve-Path')
+$firstInventoryAt = $installText.IndexOf('if (-not $HasNvidiaSmi -and (Get-NvidiaLibraryInventory))')
+Check "the venv interpreter is named before the inventory is first read" (
+ $venvAt -gt 0 -and $firstInventoryAt -gt $venvAt)
+Check "the managed interpreter is named before it too" (
+ $managedAt -gt 0 -and $firstInventoryAt -gt $managedAt)
+
+Write-Host ""
+Write-Host "=== the timeout is whole seconds, rounded up, without [math]::Ceiling ==="
+
+# PowerShell's / is floating point and [int] rounds to NEAREST, so the C "+999" ceiling idiom
+# turns 10000ms into 11s. Run the shipped expression rather than a retyped copy of it.
+$ceilLines = @($pyBlock -split "`n" | Where-Object {
+ $_ -match '^\$seconds = ' -or $_ -match '^if \(\(\$TimeoutMs % 1000\) -ne 0\)' -or $_ -match '^if \(\$seconds -lt 1\)'
+})
+Check "the ceiling is four lines of integer arithmetic" ($ceilLines.Count -eq 4)
+$ceilExpr = [scriptblock]::Create(($ceilLines -join "`n") + "`nreturn `$seconds")
+foreach ($row in @(
+ @{ Ms = 10000; Want = 10 }, @{ Ms = 10001; Want = 11 }, @{ Ms = 9999; Want = 10 },
+ @{ Ms = 1; Want = 1 }, @{ Ms = 0; Want = 1 }, @{ Ms = 2500; Want = 3 }, @{ Ms = 20000; Want = 20 }
+)) {
+ $TimeoutMs = $row.Ms
+ $got = & $ceilExpr
+ Check "$($row.Ms) ms is $($row.Want) s" ($got -eq $row.Want)
+}
+
+Write-Host ""
+Write-Host "=== behaviour, driven through the real functions ==="
+
+Invoke-Expression ($setupParts[0]) # Get-NvidiaNvmlLibraryPath
+Invoke-Expression ($setupParts[5]) # Get-NvidiaSystem32Dir: Windows names the driver libraries under it
+Invoke-Expression ($setupParts[2]) # Read-NvidiaLibraryRawViaPython
+Invoke-Expression ($setupParts[3]) # Read-NvidiaLibraryRaw
+Invoke-Expression ($setupParts[4]) # Get-NvidiaLibraryInventory
+# The emitted rung declines, which is the whole point: this is what a Constrained Language Mode
+# or WDAC host sees, and the capabilities below are recovered with nothing emitted.
+function Get-NvidiaLibraryProbeType { return $null }
+
+$script:PythonExe = ""
+function Get-NvidiaProbePythonExe { return $script:PythonExe }
+
+Check "no interpreter means no answer, not an error" ((Read-NvidiaLibraryRawViaPython -TimeoutMs 5000) -eq "")
+
+$python = $null
+foreach ($name in @("python3", "python")) {
+ $cmd = Get-Command $name -ErrorAction SilentlyContinue
+ if ($cmd) { $python = $cmd.Source; break }
+}
+
+if (-not $python) {
+ Write-Host " SKIP no Python on this runner; the behavioural rows need one"
+} else {
+ $script:PythonExe = $python
+ $saved = $env:UNSLOTH_NVIDIA_PYTHON_PROBE
+ try {
+ $env:UNSLOTH_NVIDIA_PYTHON_PROBE = "0"
+ Check "UNSLOTH_NVIDIA_PYTHON_PROBE=0 declines before spawning anything" `
+ ((Read-NvidiaLibraryRawViaPython -TimeoutMs 5000) -eq "")
+ } finally {
+ if ($null -eq $saved) { Remove-Item Env:UNSLOTH_NVIDIA_PYTHON_PROBE -ErrorAction SilentlyContinue }
+ else { $env:UNSLOTH_NVIDIA_PYTHON_PROBE = $saved }
+ }
+
+ $answer = Read-NvidiaLibraryRawViaPython -TimeoutMs 30000
+ # A runner without an NVIDIA driver is the common case and must answer "" rather than throw.
+ if (-not $answer) {
+ Check "no driver on this host answers empty, not an error" ($answer -eq "")
+ } else {
+ Check "the answer names a source the inventory understands" ($answer -match '^(nvml|cuda);')
+ Check "the answer carries a version and at least one capability" `
+ ($answer -match '^(nvml|cuda);\d+;\d+;\d+\.\d+(,\d+\.\d+)*$')
+ # Through the real Read-NvidiaLibraryRaw, with the emitted rung declining. Every row
+ # below is guarded on a non-null inventory: $null -eq $null would pass them all.
+ $inventory = Get-NvidiaLibraryInventory -TimeoutSec 30
+ Check "the inventory parses the answer with nothing emitted" ($null -ne $inventory)
+ Check "the inventory recovered the compute capabilities" `
+ ($null -ne $inventory -and $inventory.ComputeCaps.Count -ge 1 -and $inventory.ComputeCaps[0] -match '^\d+\.\d+$')
+ Check "the inventory device count matches the capability list" `
+ ($null -ne $inventory -and $inventory.Count -eq $inventory.ComputeCaps.Count)
+ Check "the CUDA major version is plausible" ($null -ne $inventory -and $inventory.CudaMajor -ge 1)
+ Check "the inventory names the source the raw answer named" `
+ ($null -ne $inventory -and $answer.StartsWith("$($inventory.Source);"))
+ }
+
+ # The launcher must not leave the interpreter or its scratch files behind.
+ $tempRoot = if ($env:TEMP) { $env:TEMP } elseif ($env:TMPDIR) { $env:TMPDIR } else { "/tmp" }
+ $litter = @(Get-ChildItem -LiteralPath $tempRoot -Filter "unsloth-nvprobe-*" -ErrorAction SilentlyContinue)
+ Check "the launcher leaves no scratch files behind" ($litter.Count -eq 0)
+}
+
+Write-Host ""
+Write-Host "=== a hung NVML earns one CUDA-only child, and nothing else does ==="
+
+# The embedded probe honours the switch, so the retry really skips NVML.
+Check "the embedded probe skips NVML only when the switch is 1" `
+ ($installProbe -match 'if os\.environ\.get\("UNSLOTH_NVIDIA_PROBE_SKIP_NVML", ""\) != "1":')
+Check "the launcher sets the switch only for a -SkipNvml child" `
+ (($pyBlock -match 'if \(\$SkipNvml\) \{ \$env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = "1" \}') -and
+ ($pyBlock -match 'finally \{[\s\S]*Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML'))
+
+# Stubbed launcher: the real Read-NvidiaLibraryRaw (emitted rung declining, as under CLM/WDAC)
+# resolves the stub through dynamic scope. A TIMEOUT outcome may also burn some of the budget.
+function Invoke-RawWithStub($outcomes, [int]$TimeoutMs = 30000, [int]$TimeoutSleepMs = 0) {
+ $script:StubCalls = @()
+ $script:StubOutcomes = [System.Collections.ArrayList]@($outcomes)
+ $script:StubSleepMs = $TimeoutSleepMs
+ return & {
+ function Read-NvidiaLibraryRawViaPython {
+ param([int]$TimeoutMs = 10000, [switch]$SkipNvml)
+ $script:StubCalls += , @{ TimeoutMs = $TimeoutMs; SkipNvml = [bool]$SkipNvml }
+ $next = $script:StubOutcomes[0]; $script:StubOutcomes.RemoveAt(0)
+ $script:NvidiaPythonProbeTimedOut = ($next -eq "TIMEOUT")
+ if ($next -eq "TIMEOUT") {
+ if ($script:StubSleepMs -gt 0) { Start-Sleep -Milliseconds $script:StubSleepMs }
+ return ""
+ }
+ return $next
+ }
+ Read-NvidiaLibraryRaw -TimeoutMs $TimeoutMs
+ }
+}
+$got = Invoke-RawWithStub @("TIMEOUT", "cuda;12;8;8.9")
+Check "stub: a timed-out first child is followed by exactly one more" ($script:StubCalls.Count -eq 2)
+Check "stub: the first child reads NVML, the second skips it" `
+ ($script:StubCalls.Count -eq 2 -and -not $script:StubCalls[0].SkipNvml -and $script:StubCalls[1].SkipNvml)
+Check "stub: each child is capped at the 30s per-reader bound" `
+ ($script:StubCalls.Count -eq 2 -and $script:StubCalls[0].TimeoutMs -eq 30000 -and
+ $script:StubCalls[1].TimeoutMs -le 30000 -and $script:StubCalls[1].TimeoutMs -ge 25000)
+Check "stub: the CUDA-only answer is returned" ($got -eq "cuda;12;8;8.9")
+$got = Invoke-RawWithStub @("")
+Check "stub: an empty answer spawns no second child" ($script:StubCalls.Count -eq 1 -and $got -eq "")
+$got = Invoke-RawWithStub @("nvml;13;0;12.0")
+Check "stub: an answer spawns no second child" ($script:StubCalls.Count -eq 1 -and $got -eq "nvml;13;0;12.0")
+$got = Invoke-RawWithStub @("TIMEOUT", "TIMEOUT")
+Check "stub: a CUDA-only child that also hangs ends the search" ($script:StubCalls.Count -eq 2 -and $got -eq "")
+# Shared deadline 2 x 1500 ms; the first child burns 1200 ms of it, leaving under 3 s.
+$got = Invoke-RawWithStub @("TIMEOUT", "cuda;12;8;8.9") -TimeoutMs 1500 -TimeoutSleepMs 1200
+Check "stub: no CUDA-only child when under 3 s of the shared budget remain" ($script:StubCalls.Count -eq 1 -and $got -eq "")
+# Shared deadline 2 x 4000 ms; the first child hangs for 3000 ms. Each child's bound leaves 2 s to
+# reap it, so the CUDA-only child gets what is left minus that, and the pair ends by the deadline.
+$got = Invoke-RawWithStub @("TIMEOUT", "cuda;12;8;8.9") -TimeoutMs 4000 -TimeoutSleepMs 3000
+Check "stub: every child's bound leaves 2 s of the shared deadline to reap it" (
+ $script:StubCalls.Count -eq 2 -and $script:StubCalls[0].TimeoutMs -eq 4000 -and
+ $script:StubCalls[1].TimeoutMs -le 3000 -and $script:StubCalls[1].TimeoutMs -ge 2000)
+# Shared deadline 2 x 5000 ms; the first child burns 5000 ms, so the retry gets only the rest.
+$got = Invoke-RawWithStub @("TIMEOUT", "cuda;12;8;8.9") -TimeoutMs 5000 -TimeoutSleepMs 5000
+Check "stub: the CUDA-only child gets what is left of the shared budget, not a fresh bound" `
+ ($script:StubCalls.Count -eq 2 -and $script:StubCalls[0].TimeoutMs -eq 5000 -and
+ $script:StubCalls[1].TimeoutMs -lt 5000 -and $script:StubCalls[1].TimeoutMs -ge 2000 -and $got -eq "cuda;12;8;8.9")
+
+# End to end through the real launcher, with a fake interpreter that logs the switch it sees and
+# hangs like a wedged NVML unless the switch is set. POSIX sh only: a .cmd stand-in would leave its
+# sleeping child holding the output file on Windows.
+if ($IsWindows -or $env:OS -eq "Windows_NT") {
+ Write-Host " SKIP the fake-interpreter rows need a POSIX shell"
+} else {
+ $fakeDir = Join-Path ([System.IO.Path]::GetTempPath()) ("unsloth-fakepy-" + [guid]::NewGuid().ToString("N"))
+ New-Item -ItemType Directory -Path $fakeDir | Out-Null
+ $fakePy = Join-Path $fakeDir "python"
+ $fakeLog = Join-Path $fakeDir "calls.log"
+ $fakeBody = @(
+ '#!/bin/sh'
+ 'cat > /dev/null'
+ 'echo "skip=${UNSLOTH_NVIDIA_PROBE_SKIP_NVML:-unset}" >> "$UNSLOTH_FAKE_PY_LOG"'
+ 'if [ "$UNSLOTH_NVIDIA_PROBE_SKIP_NVML" = "1" ]; then printf "cuda;12;8;8.9"; exit 0; fi'
+ 'if [ "$UNSLOTH_FAKE_PY_MODE" = "hang" ]; then exec sleep 30; fi'
+ 'exit 0'
+ ) -join "`n"
+ [System.IO.File]::WriteAllText($fakePy, "$fakeBody`n")
+ & chmod +x $fakePy
+ $savedPy = $script:PythonExe
+ $savedSkipEnv = $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML
+ try {
+ $script:PythonExe = $fakePy
+ $env:UNSLOTH_FAKE_PY_LOG = $fakeLog
+ # A value already in this shell must reach no first child and must survive the calls.
+ $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = "caller"
+
+ # Shared deadline 2 x 5000 ms: the first child is killed at 5 s and reaped, leaving the retry
+ # its share less the 2 s kept back for reaping it.
+ $env:UNSLOTH_FAKE_PY_MODE = "hang"
+ Remove-Item -LiteralPath $fakeLog -ErrorAction SilentlyContinue
+ $got = Read-NvidiaLibraryRaw -TimeoutMs 5000
+ $seen = @(Get-Content -LiteralPath $fakeLog -ErrorAction SilentlyContinue)
+ Check "live: a first child killed at its bound triggers exactly one more" ($seen.Count -eq 2)
+ Check "live: the first child saw no switch, the second saw 1" (($seen -join ",") -eq "skip=unset,skip=1")
+ Check "live: the CUDA-only answer is returned" ($got -eq "cuda;12;8;8.9")
+ Check "live: the caller's switch value is restored" ($env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -eq "caller")
+
+ $env:UNSLOTH_FAKE_PY_MODE = "empty"
+ Remove-Item -LiteralPath $fakeLog -ErrorAction SilentlyContinue
+ $got = Read-NvidiaLibraryRaw -TimeoutMs 3000
+ $seen = @(Get-Content -LiteralPath $fakeLog -ErrorAction SilentlyContinue)
+ Check "live: a first child that answers empty spawns no second" ($seen.Count -eq 1 -and $got -eq "")
+ Check "live: the switch is still restored after a single child" ($env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -eq "caller")
+
+ Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue
+ $env:UNSLOTH_FAKE_PY_MODE = "hang"
+ $null = Read-NvidiaLibraryRaw -TimeoutMs 5000
+ Check "live: an unset switch is left unset" ($null -eq $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML)
+ } finally {
+ $script:PythonExe = $savedPy
+ if ($null -eq $savedSkipEnv) { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }
+ else { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = $savedSkipEnv }
+ Remove-Item Env:UNSLOTH_FAKE_PY_LOG, Env:UNSLOTH_FAKE_PY_MODE -ErrorAction SilentlyContinue
+ Remove-Item -LiteralPath $fakeDir -Recurse -Force -ErrorAction SilentlyContinue
+ }
+}
+
+Write-Host ""
+if ($failures -gt 0) { Write-Host "$failures check(s) failed" -ForegroundColor Red; exit 1 }
+Write-Host "All checks passed"