Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Recover CUDA compute capabilities without emitting a P/Invoke type (#11173)
Daniel Hanb7d2b33f73a9
diff --git a/.github/workflows/cross-platform-parity-ci.yml b/.github/workflows/cross-platform-parity-ci.ymlindex d0be9c3ea21..69db28eaa72 100644--- a/.github/workflows/cross-platform-parity-ci.yml+++ b/.github/workflows/cross-platform-parity-ci.yml@@ -76,6 +76,7 @@ on:- 'tests/python/test_windows_python_venv_hardening.py'- 'tests/python/test_windows_setup_output_encoding.py'- 'tests/python/test_windows_installer_native_type_fallback.py'+ - 'tests/python/test_installer_embedded_nvidia_probe.py'- 'tests/python/test_pwsh_runner_encoding.py'# The deliberate unreadable-directory and traversal-limit cases for the# no-compiler detector's temp scan live in this job, and only here. The@@ -105,6 +106,7 @@ on:- 'tests/studio/test_amd_name_arch_r9700.ps1'- 'tests/studio/test_nvidia_library_inventory.ps1'- 'tests/studio/test_nvidia_smi_discovery.ps1'+ - 'tests/studio/test_nvidia_python_probe_parity.ps1'- 'tests/studio/test_install_lock_excludes_aliases.ps1'- 'tests/studio/test_early_python_path_resolver.ps1'- 'tests/studio/test_early_python_process_image.ps1'@@ -191,6 +193,7 @@ on:- 'tests/python/test_windows_python_venv_hardening.py'- 'tests/python/test_windows_setup_output_encoding.py'- 'tests/python/test_windows_installer_native_type_fallback.py'+ - 'tests/python/test_installer_embedded_nvidia_probe.py'- 'tests/python/test_pwsh_runner_encoding.py'# The deliberate unreadable-directory and traversal-limit cases for the# no-compiler detector's temp scan live in this job, and only here. The@@ -220,6 +223,7 @@ on:- 'tests/studio/test_amd_name_arch_r9700.ps1'- 'tests/studio/test_nvidia_library_inventory.ps1'- 'tests/studio/test_nvidia_smi_discovery.ps1'+ - 'tests/studio/test_nvidia_python_probe_parity.ps1'- 'tests/studio/test_install_lock_excludes_aliases.ps1'- 'tests/studio/test_early_python_path_resolver.ps1'- 'tests/studio/test_early_python_process_image.ps1'@@ -327,6 +331,7 @@ jobs:tests/python/test_windows_installer_native_type_fallback.pytests/python/test_pwsh_runner_encoding.pytests/python/test_compiler_watch_temp_scan.py+ tests/python/test_installer_embedded_nvidia_probe.pytests/test_installer_skip_autostart.pytests/test_installer_system32_guard.pytests/test_installer_interactive_prompts.py@@ -390,6 +395,17 @@ jobs:if: runner.os == 'Windows'shell: powershellrun: powershell -NoProfile -File tests\studio\test_nvidia_smi_discovery.ps1+ # The inventory's Python rung, which recovers the CUDA version and the per-device compute+ # capabilities on hosts that cannot emit a P/Invoke type. Pins the embedded probe against+ # studio/nvidia_probe.py and against the other copy; the 5.1 leg matters because+ # Constrained Language Mode is one of the policies that make the emitted rung decline.+ - name: NVIDIA Python probe parity tests+ shell: pwsh+ run: pwsh -NoProfile -File tests/studio/test_nvidia_python_probe_parity.ps1+ - name: NVIDIA Python probe parity tests (Windows PowerShell 5.1)+ if: runner.os == 'Windows'+ shell: powershell+ run: powershell -NoProfile -File tests\studio\test_nvidia_python_probe_parity.ps1# Spawns a real holder process and takes the lock through a junction (Windows) or a# symbolic link (Linux), so it needs both legs: the alias mechanisms differ per OS and# .NET enforces FileShare.None through a different mechanism on each.diff --git a/install.ps1 b/install.ps1index 8b01b97ffb1..241599b2077 100644--- a/install.ps1+++ b/install.ps1@@ -8793,6 +8793,27 @@ exit 0return ($LASTEXITCODE -eq 0 -and $out -match '(?m)^GPU\s+\d+:')}+ # Interpreter for the shared inventory's Python rung, deliberately NOT part of the shared+ # region: the two files find Python in different places. Here it is the early read-only+ # ladder from the process-image rung, which never installs anything.+ function Get-NvidiaProbePythonExe {+ if ("$($env:UNSLOTH_EARLY_PYTHON_PROBE)".Trim() -eq "0") { return "" }+ # The interpreter this run installed comes FIRST, and Get-StudioEarlyPython is the+ # fallback rather than the source.+ #+ # That ladder memoises, including a miss: on a fresh host with no Python of its own it+ # is first called by the install-lock path, finds nothing, and caches $null for the rest+ # of the run. The inventory is not read until much later, by which point this install+ # has created a managed interpreter and then a venv, so asking the cache would decline+ # on exactly the fresh install where nvidia-smi is also most likely to be missing. The+ # host would take CPU wheels while holding a working NVIDIA card.+ foreach ($candidate in @($VenvPython, $ManagedPythonPath)) {+ if ([string]::IsNullOrWhiteSpace($candidate)) { continue }+ if (Test-Path -LiteralPath $candidate -PathType Leaf) { return $candidate }+ }+ try { return "$(Get-StudioEarlyPython)" } catch { return "" }+ }+# ── BEGIN SHARED WITH studio/setup.ps1 (Get-NvidiaLibraryInventory) ──# Full paths only: a bare name searches PATH, where ZLUDA's nvcuda.dll and nvml.dll pass for NVIDIA.function Get-NvidiaNvmlLibraryPath {@@ -8845,72 +8866,311 @@ exit 0return ($name -as [type])}+ # The same inventory with nothing emitted: CPython's ctypes makes the identical NVML and CUDA+ # driver calls, and the interop leaves the scanned surface rather than moving within it.+ # A second source BENEATH the emitted one, never ahead of it: "" whenever no interpreter is+ # available or the probe itself says nothing.+ # Get-NvidiaProbePythonExe is deliberately per-file. The installer has its early read-only+ # interpreter ladder; setup.ps1 has the venv a previous run already built.+ function Read-NvidiaLibraryRawViaPython {+ param([int]$TimeoutMs = 10000, [switch]$SkipNvml)+ # Set only when the child is killed at the deadline, so a caller can tell a hung driver from+ # an empty answer. Reset first: every early return below is an answer, not a timeout.+ $script:NvidiaPythonProbeTimedOut = $false+ if ("$($env:UNSLOTH_NVIDIA_PYTHON_PROBE)".Trim() -eq "0") { return "" }+ $exe = ""+ try { $exe = "$(Get-NvidiaProbePythonExe)" } catch { return "" }+ if (-not $exe) { return "" }+ $windows = ($env:OS -eq "Windows_NT")+ $nvmlHint = if ($windows) { Get-NvidiaNvmlLibraryPath } else { "libnvidia-ml.so.1" }+ $cudaHint = if ($windows) { Join-Path (Get-NvidiaSystem32Dir) "nvcuda.dll" } else { "libcuda.so.1" }+ # Kept byte-identical with studio/nvidia_probe.py's readers by+ # tests/studio/test_nvidia_python_probe_parity.ps1. Column 0 on purpose: this is Python.+ $probeSource = @'+import ctypes, os, sys+++def _names(kind, hint):+ if os.name == "nt":+ # The driver's full path only: a bare name also finds a CUDA stand-in such as ZLUDA+ # beside the interpreter, and that is not an NVIDIA GPU (#11736).+ return [hint] if hint and os.path.isabs(hint) else []+ if kind == "nvml":+ return ["libnvidia-ml.so.1", "libnvidia-ml.so"]+ return ["libcuda.so.1", "libcuda.so"]+++def _load(kind, hint):+ for name in _names(kind, hint):+ if not name:+ continue+ try:+ return ctypes.CDLL(name)+ except Exception:+ continue+ return None+++def _unpack(packed):+ return "%d;%d" % (packed // 1000, (packed % 1000) // 10)+++def read_nvml(hint):+ lib = _load("nvml", hint)+ if lib is None:+ return ""+ try:+ if lib.nvmlInit_v2() != 0:+ return ""+ except Exception:+ return ""+ try:+ count = ctypes.c_uint(0)+ if lib.nvmlDeviceGetCount_v2(ctypes.byref(count)) != 0 or count.value == 0:+ return ""+ packed = ctypes.c_int(0)+ if lib.nvmlSystemGetCudaDriverVersion_v2(ctypes.byref(packed)) != 0 or packed.value < 1000:+ return ""+ # ctypes defaults every return and every pointer argument to a C int, which truncates a+ # 64-bit nvmlDevice_t handle. Declare both before the first call, not after.+ handle_of = lib.nvmlDeviceGetHandleByIndex_v2+ handle_of.restype = ctypes.c_int+ handle_of.argtypes = [ctypes.c_uint, ctypes.POINTER(ctypes.c_void_p)]+ cap_of = lib.nvmlDeviceGetCudaComputeCapability+ cap_of.restype = ctypes.c_int+ cap_of.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)]+ caps = []+ for index in range(count.value):+ device = ctypes.c_void_p()+ # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.+ if handle_of(index, ctypes.byref(device)) != 0:+ return ""+ major = ctypes.c_int(0)+ minor = ctypes.c_int(0)+ if cap_of(device, ctypes.byref(major), ctypes.byref(minor)) != 0:+ return ""+ caps.append("%d.%d" % (major.value, minor.value))+ return "nvml;%s;%s" % (_unpack(packed.value), ",".join(caps))+ finally:+ try:+ lib.nvmlShutdown()+ except Exception:+ pass+++def read_cuda(hint):+ lib = _load("cuda", hint)+ if lib is None:+ return ""+ # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one, so a+ # hidden pre-Turing card still caps the family. cuInit reads the mask once.+ saved = os.environ.pop("CUDA_VISIBLE_DEVICES", None)+ try:+ init = lib.cuInit(0)+ except Exception:+ return ""+ finally:+ if saved is not None:+ os.environ["CUDA_VISIBLE_DEVICES"] = saved+ if init != 0:+ return ""+ count = ctypes.c_int(0)+ if lib.cuDeviceGetCount(ctypes.byref(count)) != 0 or count.value == 0:+ return ""+ packed = ctypes.c_int(0)+ if lib.cuDriverGetVersion(ctypes.byref(packed)) != 0 or packed.value < 1000:+ return ""+ caps = []+ for index in range(count.value):+ device = ctypes.c_int(0)+ if lib.cuDeviceGet(ctypes.byref(device), index) != 0:+ return ""+ major = ctypes.c_int(0)+ minor = ctypes.c_int(0)+ # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.+ if lib.cuDeviceGetAttribute(ctypes.byref(major), 75, device) != 0:+ return ""+ if lib.cuDeviceGetAttribute(ctypes.byref(minor), 76, device) != 0:+ return ""+ caps.append("%d.%d" % (major.value, minor.value))+ return "cuda;%s;%s" % (_unpack(packed.value), ",".join(caps))+++def main():+ nvml_hint = os.environ.get("UNSLOTH_NVML_HINT", "")+ cuda_hint = os.environ.get("UNSLOTH_CUDA_HINT", "")+ answer = ""+ # Set only for the CUDA-only retry that follows a child NVML held past its deadline.+ if os.environ.get("UNSLOTH_NVIDIA_PROBE_SKIP_NVML", "") != "1":+ try:+ answer = read_nvml(nvml_hint)+ except Exception:+ answer = ""+ if not answer:+ try:+ answer = read_cuda(cuda_hint)+ except Exception:+ answer = ""+ sys.stdout.write(answer)+++main()+'@+ # Cmdlets only. Constrained Language Mode refuses New-Object ProcessStartInfo and+ # [Process]::Start, and CLM is one of the two policies that make the emitted rung decline,+ # so this launcher has to work on exactly the hosts that need it most.+ $tempRoot = if ($env:TEMP) { $env:TEMP } elseif ($env:TMPDIR) { $env:TMPDIR } else { "/tmp" }+ $stem = Join-Path $tempRoot ("unsloth-nvprobe-" + [guid]::NewGuid().ToString("N"))+ $scriptFile = "$stem.py"+ $outFile = "$stem.out"+ $errFile = "$stem.err"+ $raw = ""+ try {+ Set-Content -LiteralPath $scriptFile -Value $probeSource -Encoding UTF8 -ErrorAction Stop+ # Whole seconds, rounded up, without [math]::Ceiling: CLM blocks it. PowerShell's / is+ # floating point and [int] rounds to nearest, so 10000ms must not become 11s.+ $seconds = ($TimeoutMs - ($TimeoutMs % 1000)) / 1000+ if (($TimeoutMs % 1000) -ne 0) { $seconds = $seconds + 1 }+ $seconds = [int]$seconds+ if ($seconds -lt 1) { $seconds = 1 }+ # Nothing with a space in it reaches the command line. Windows PowerShell 5.1 appends+ # each native argument verbatim, so a script under "C:\Users\First Last\AppData\Local\+ # Temp" or a hint under "C:\Program Files\NVIDIA Corporation\NVSMI" would split on its+ # spaces and the child would run something else. The script arrives on stdin and the two+ # library hints in the environment; the only arguments left are -I -S and a bare dash.+ $savedNvml = $env:UNSLOTH_NVML_HINT+ $savedCuda = $env:UNSLOTH_CUDA_HINT+ $savedSkip = $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML+ $env:UNSLOTH_NVML_HINT = $nvmlHint+ $env:UNSLOTH_CUDA_HINT = $cudaHint+ # The switch reaches this child only. An inherited value must not make a first child skip NVML.+ if ($SkipNvml) { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = "1" }+ else { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }+ try {+ $proc = Start-Process -FilePath $exe -ArgumentList @("-I", "-S", "-") -NoNewWindow -PassThru `+ -RedirectStandardInput $scriptFile -RedirectStandardOutput $outFile -RedirectStandardError $errFile -ErrorAction Stop+ } finally {+ if ($null -eq $savedNvml) { Remove-Item Env:UNSLOTH_NVML_HINT -ErrorAction SilentlyContinue }+ else { $env:UNSLOTH_NVML_HINT = $savedNvml }+ if ($null -eq $savedCuda) { Remove-Item Env:UNSLOTH_CUDA_HINT -ErrorAction SilentlyContinue }+ else { $env:UNSLOTH_CUDA_HINT = $savedCuda }+ if ($null -eq $savedSkip) { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }+ else { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = $savedSkip }+ }+ if (-not $proc) { return "" }+ # -InputObject and an error variable, never $proc.Id or $proc.HasExited. Constrained+ # Language Mode permits property reads only on its allowed type list and+ # System.Diagnostics.Process is not on it, so reading either one throws on exactly the+ # hosts this rung exists for. Handing the object to a cmdlet keeps the access inside+ # compiled code, where the language mode does not reach.+ $waitError = $null+ Wait-Process -InputObject $proc -Timeout $seconds -ErrorAction SilentlyContinue -ErrorVariable waitError+ if ($waitError) {+ $script:NvidiaPythonProbeTimedOut = $true+ try { Stop-Process -InputObject $proc -Force -ErrorAction SilentlyContinue } catch { }+ # Let the killed child release its redirected files before the finally deletes them.+ Wait-Process -InputObject $proc -Timeout 2 -ErrorAction SilentlyContinue+ return ""+ }+ $raw = "$(Get-Content -LiteralPath $outFile -Raw -ErrorAction SilentlyContinue)"+ } catch { return "" }+ finally {+ foreach ($stale in @($scriptFile, $outFile, $errFile)) {+ Remove-Item -LiteralPath $stale -Force -ErrorAction SilentlyContinue+ }+ }+ return "$raw".Trim()+ }+# "source;cudaMajor;cudaMinor;cap,cap" from NVML, else the CUDA driver API; "" when neither# answers. Versions are major*1000 + minor*10. One runspace + deadline per reader: a shared one let slow NVML starve CUDA.+ # A wedged driver can block inside the library, and each deadline leaves its runspace behind.+ # The Python rung below spends what the emitted rung left of ONE budget (a per-reader bound for+ # NVML plus one for CUDA), so the worst-case wall clock is the emitted rung's own. Its first child+ # reads NVML then the CUDA driver API under at most one per-reader bound; only when that child is+ # killed at its bound (a hung NVML) and at least 2 s remain does a second child read the CUDA+ # driver API alone with the rest, so a hung NVML no longer starves a healthy CUDA driver API.function Read-NvidiaLibraryRaw {param([int]$TimeoutMs = 30000)+ $deadline = (Get-Date).AddMilliseconds($TimeoutMs * 2)+ $native = ""$type = Get-NvidiaLibraryProbeType- if (-not $type) { return "" }- $reader = {- param($T, $Which)- function Read-Nvml {- if ($T::nvmlInit_v2() -ne 0) { return "" }- try {- [uint32]$count = 0- if ($T::nvmlDeviceGetCount_v2([ref]$count) -ne 0 -or $count -eq 0) { return "" }+ if ($type) {+ $reader = {+ param($T, $Which)+ function Read-Nvml {+ if ($T::nvmlInit_v2() -ne 0) { return "" }+ try {+ [uint32]$count = 0+ if ($T::nvmlDeviceGetCount_v2([ref]$count) -ne 0 -or $count -eq 0) { return "" }+ [int]$ver = 0+ if ($T::nvmlSystemGetCudaDriverVersion_v2([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }+ $caps = @()+ for ([uint32]$i = 0; $i -lt $count; $i++) {+ [IntPtr]$dev = [IntPtr]::Zero; [int]$major = 0; [int]$minor = 0+ # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.+ if ($T::nvmlDeviceGetHandleByIndex_v2($i, [ref]$dev) -ne 0) { return "" }+ if ($T::nvmlDeviceGetCudaComputeCapability($dev, [ref]$major, [ref]$minor) -ne 0) { return "" }+ $caps += "$major.$minor"+ }+ return "nvml;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"+ } finally { $null = $T::nvmlShutdown() }+ }+ function Read-Cuda {+ # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one,+ # so a hidden pre-Turing card still caps the family. cuInit reads the mask once.+ $saved = $env:CUDA_VISIBLE_DEVICES+ Remove-Item Env:CUDA_VISIBLE_DEVICES -ErrorAction SilentlyContinue+ try { $init = $T::cuInit([uint32]0) } finally { if ($null -ne $saved) { $env:CUDA_VISIBLE_DEVICES = $saved } }+ if ($init -ne 0) { return "" }+ [int]$count = 0+ if ($T::cuDeviceGetCount([ref]$count) -ne 0 -or $count -eq 0) { return "" }[int]$ver = 0- if ($T::nvmlSystemGetCudaDriverVersion_v2([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }+ if ($T::cuDriverGetVersion([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }$caps = @()- for ([uint32]$i = 0; $i -lt $count; $i++) {- [IntPtr]$dev = [IntPtr]::Zero; [int]$major = 0; [int]$minor = 0- # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.- if ($T::nvmlDeviceGetHandleByIndex_v2($i, [ref]$dev) -ne 0) { return "" }- if ($T::nvmlDeviceGetCudaComputeCapability($dev, [ref]$major, [ref]$minor) -ne 0) { return "" }+ for ($i = 0; $i -lt $count; $i++) {+ [int]$dev = 0; [int]$major = 0; [int]$minor = 0+ if ($T::cuDeviceGet([ref]$dev, $i) -ne 0) { return "" }+ # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.+ if ($T::cuDeviceGetAttribute([ref]$major, 75, $dev) -ne 0) { return "" }+ if ($T::cuDeviceGetAttribute([ref]$minor, 76, $dev) -ne 0) { return "" }$caps += "$major.$minor"}- return "nvml;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"- } finally { $null = $T::nvmlShutdown() }- }- function Read-Cuda {- # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one,- # so a hidden pre-Turing card still caps the family. cuInit reads the mask once.- $saved = $env:CUDA_VISIBLE_DEVICES- Remove-Item Env:CUDA_VISIBLE_DEVICES -ErrorAction SilentlyContinue- try { $init = $T::cuInit([uint32]0) } finally { if ($null -ne $saved) { $env:CUDA_VISIBLE_DEVICES = $saved } }- if ($init -ne 0) { return "" }- [int]$count = 0- if ($T::cuDeviceGetCount([ref]$count) -ne 0 -or $count -eq 0) { return "" }- [int]$ver = 0- if ($T::cuDriverGetVersion([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }- $caps = @()- for ($i = 0; $i -lt $count; $i++) {- [int]$dev = 0; [int]$major = 0; [int]$minor = 0- if ($T::cuDeviceGet([ref]$dev, $i) -ne 0) { return "" }- # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.- if ($T::cuDeviceGetAttribute([ref]$major, 75, $dev) -ne 0) { return "" }- if ($T::cuDeviceGetAttribute([ref]$minor, 76, $dev) -ne 0) { return "" }- $caps += "$major.$minor"- }- return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"- }- $r = ""- try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }- return "$r"- }- foreach ($which in @("nvml", "cuda")) {- $ps = $null; $handle = $null; $r = ""- try {- $ps = [powershell]::Create()- $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)- $handle = $ps.BeginInvoke()- if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {- $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"+ return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"}- } catch { $r = "" }- finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }- if ($r) { return $r }- }- return ""+ $r = ""+ try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }+ return "$r"+ }+ foreach ($which in @("nvml", "cuda")) {+ $ps = $null; $handle = $null; $r = ""+ try {+ $ps = [powershell]::Create()+ $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)+ $handle = $ps.BeginInvoke()+ if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {+ $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"+ }+ } catch { $r = "" }+ finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }+ if ($r) { $native = $r; break }+ }+ }+ if ($native) { return $native }+ $remainingMs = [int]($deadline - (Get-Date)).TotalMilliseconds+ # Not worth a child process we cannot wait out; the emitted rung already spent the budget.+ if ($remainingMs -lt 3000) { return "" }+ # Each child gets at most one per-reader bound, compared by hand: CLM refuses [math]. 2 s of+# every child's share is kept back for reaping it if it hangs, so the pair ends inside the deadline.+ $childMs = $remainingMs - 2000; if ($childMs -gt $TimeoutMs) { $childMs = $TimeoutMs }+ $raw = ""+ try { $raw = Read-NvidiaLibraryRawViaPython -TimeoutMs $childMs } catch { return "" }+ # Only a first child killed at its bound (a hung NVML) earns a CUDA-only child, with what is left.+ if (-not $script:NvidiaPythonProbeTimedOut) { return $raw }+ $remainingMs = [int]($deadline - (Get-Date)).TotalMilliseconds+ if ($remainingMs -lt 3000) { return "" }+ $childMs = $remainingMs - 2000; if ($childMs -gt $TimeoutMs) { $childMs = $TimeoutMs }+ try { return (Read-NvidiaLibraryRawViaPython -TimeoutMs $childMs -SkipNvml) } catch { return "" }}# NVIDIA inventory from the driver's own libraries (NVML, then the CUDA driver API), for a@@ -9029,9 +9289,13 @@ exit 0}}if (-not $HasNvidiaSmi -and (Get-NvidiaLibraryInventory)) {- # Same promotion as setup.ps1: the gates below read $HasNvidiaSmi as "NVIDIA GPU present".- $HasNvidiaSmi = $true- Write-StudioLine " NVIDIA GPU found through the driver library; nvidia-smi is unavailable" -ForegroundColor Gray+ # A driver too old for any CUDA wheel (below 11) is no GPU this route can serve: the Intel and+ # AMD routes still get their turn instead of the install falling to CPU.+ if ((Get-NvidiaLibraryInventory).CudaMajor -ge 11) {+ # Same promotion as setup.ps1: the gates below read $HasNvidiaSmi as "NVIDIA GPU present".+ $HasNvidiaSmi = $true+ Write-StudioLine " NVIDIA GPU found through the driver library; nvidia-smi is unavailable" -ForegroundColor Gray+ }}# nvidia-smi was already resolved above and never asked which card it found, so the# banner said "NVIDIA GPU detected" on every NVIDIA host alike. compute_cap is thediff --git a/studio/setup.ps1 b/studio/setup.ps1index 1703eb6c021..4c2e16dc69c 100644--- a/studio/setup.ps1+++ b/studio/setup.ps1@@ -155,6 +155,7 @@ if ($script:UnslothVerbose) {$script:LlamaCppDegraded = $false$script:LlamaKeptGpuPrebuilt = $null$script:NvidiaSmiRejected = $false+$script:NvidiaDriverLibraryOnly = $false$script:NvidiaLibraryInventoryProbed = $false$script:NvidiaLibraryInventory = $null# Set by the offline keep, read unconditionally by the sidecar and legacy-migration blocks:@@ -1184,7 +1185,7 @@ function Get-GpuPrebuiltToKeepOverSourceBuild {if ("$($env:UNSLOTH_LLAMA_TAG)".Trim() -notin @("", "latest")) { return "" }$backend = Get-PrebuiltMarkerBackend -Marker (Join-Path $InstallDir "UNSLOTH_PREBUILT_INFO.json")if (-not $backend) { return "" }- $nvidia = $HasNvidiaSmi+ $nvidia = $HasNvidiaSmi -or [bool]$script:NvidiaDriverLibraryOnly$amd = $HasROCm -or [bool]$script:ROCmGfxArch$present = switch ($backend) {"cuda" { $nvidia }@@ -1485,6 +1486,20 @@ function Get-CudaFamilyCappedForPreTuring {return $Family}+# Interpreter for the shared inventory's Python rung, deliberately NOT part of the shared+# region: the two files find Python in different places. setup.ps1 runs after the installer,+# so the venv a previous run built is the interpreter, and $VenvDir is set well before the+# first Get-NvidiaLibraryInventory call. No interpreter simply means the rung declines.+function Get-NvidiaProbePythonExe {+ if ("$($env:UNSLOTH_EARLY_PYTHON_PROBE)".Trim() -eq "0") { return "" }+ if (-not $VenvDir) { return "" }+ foreach ($leaf in @("Scripts\python.exe", "bin/python3", "bin/python")) {+ $candidate = Join-Path $VenvDir $leaf+ if (Test-Path -LiteralPath $candidate -PathType Leaf) { return $candidate }+ }+ return ""+}+# ── BEGIN SHARED WITH install.ps1 (Get-NvidiaLibraryInventory) ──# Full paths only: a bare name searches PATH, where ZLUDA's nvcuda.dll and nvml.dll pass for NVIDIA.function Get-NvidiaNvmlLibraryPath {@@ -1537,72 +1552,311 @@ function Get-NvidiaLibraryProbeType {return ($name -as [type])}+# The same inventory with nothing emitted: CPython's ctypes makes the identical NVML and CUDA+# driver calls, and the interop leaves the scanned surface rather than moving within it.+# A second source BENEATH the emitted one, never ahead of it: "" whenever no interpreter is+# available or the probe itself says nothing.+# Get-NvidiaProbePythonExe is deliberately per-file. The installer has its early read-only+# interpreter ladder; setup.ps1 has the venv a previous run already built.+function Read-NvidiaLibraryRawViaPython {+ param([int]$TimeoutMs = 10000, [switch]$SkipNvml)+ # Set only when the child is killed at the deadline, so a caller can tell a hung driver from+ # an empty answer. Reset first: every early return below is an answer, not a timeout.+ $script:NvidiaPythonProbeTimedOut = $false+ if ("$($env:UNSLOTH_NVIDIA_PYTHON_PROBE)".Trim() -eq "0") { return "" }+ $exe = ""+ try { $exe = "$(Get-NvidiaProbePythonExe)" } catch { return "" }+ if (-not $exe) { return "" }+ $windows = ($env:OS -eq "Windows_NT")+ $nvmlHint = if ($windows) { Get-NvidiaNvmlLibraryPath } else { "libnvidia-ml.so.1" }+ $cudaHint = if ($windows) { Join-Path (Get-NvidiaSystem32Dir) "nvcuda.dll" } else { "libcuda.so.1" }+ # Kept byte-identical with studio/nvidia_probe.py's readers by+ # tests/studio/test_nvidia_python_probe_parity.ps1. Column 0 on purpose: this is Python.+ $probeSource = @'+import ctypes, os, sys+++def _names(kind, hint):+ if os.name == "nt":+ # The driver's full path only: a bare name also finds a CUDA stand-in such as ZLUDA+ # beside the interpreter, and that is not an NVIDIA GPU (#11736).+ return [hint] if hint and os.path.isabs(hint) else []+ if kind == "nvml":+ return ["libnvidia-ml.so.1", "libnvidia-ml.so"]+ return ["libcuda.so.1", "libcuda.so"]+++def _load(kind, hint):+ for name in _names(kind, hint):+ if not name:+ continue+ try:+ return ctypes.CDLL(name)+ except Exception:+ continue+ return None+++def _unpack(packed):+ return "%d;%d" % (packed // 1000, (packed % 1000) // 10)+++def read_nvml(hint):+ lib = _load("nvml", hint)+ if lib is None:+ return ""+ try:+ if lib.nvmlInit_v2() != 0:+ return ""+ except Exception:+ return ""+ try:+ count = ctypes.c_uint(0)+ if lib.nvmlDeviceGetCount_v2(ctypes.byref(count)) != 0 or count.value == 0:+ return ""+ packed = ctypes.c_int(0)+ if lib.nvmlSystemGetCudaDriverVersion_v2(ctypes.byref(packed)) != 0 or packed.value < 1000:+ return ""+ # ctypes defaults every return and every pointer argument to a C int, which truncates a+ # 64-bit nvmlDevice_t handle. Declare both before the first call, not after.+ handle_of = lib.nvmlDeviceGetHandleByIndex_v2+ handle_of.restype = ctypes.c_int+ handle_of.argtypes = [ctypes.c_uint, ctypes.POINTER(ctypes.c_void_p)]+ cap_of = lib.nvmlDeviceGetCudaComputeCapability+ cap_of.restype = ctypes.c_int+ cap_of.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)]+ caps = []+ for index in range(count.value):+ device = ctypes.c_void_p()+ # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.+ if handle_of(index, ctypes.byref(device)) != 0:+ return ""+ major = ctypes.c_int(0)+ minor = ctypes.c_int(0)+ if cap_of(device, ctypes.byref(major), ctypes.byref(minor)) != 0:+ return ""+ caps.append("%d.%d" % (major.value, minor.value))+ return "nvml;%s;%s" % (_unpack(packed.value), ",".join(caps))+ finally:+ try:+ lib.nvmlShutdown()+ except Exception:+ pass+++def read_cuda(hint):+ lib = _load("cuda", hint)+ if lib is None:+ return ""+ # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one, so a+ # hidden pre-Turing card still caps the family. cuInit reads the mask once.+ saved = os.environ.pop("CUDA_VISIBLE_DEVICES", None)+ try:+ init = lib.cuInit(0)+ except Exception:+ return ""+ finally:+ if saved is not None:+ os.environ["CUDA_VISIBLE_DEVICES"] = saved+ if init != 0:+ return ""+ count = ctypes.c_int(0)+ if lib.cuDeviceGetCount(ctypes.byref(count)) != 0 or count.value == 0:+ return ""+ packed = ctypes.c_int(0)+ if lib.cuDriverGetVersion(ctypes.byref(packed)) != 0 or packed.value < 1000:+ return ""+ caps = []+ for index in range(count.value):+ device = ctypes.c_int(0)+ if lib.cuDeviceGet(ctypes.byref(device), index) != 0:+ return ""+ major = ctypes.c_int(0)+ minor = ctypes.c_int(0)+ # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.+ if lib.cuDeviceGetAttribute(ctypes.byref(major), 75, device) != 0:+ return ""+ if lib.cuDeviceGetAttribute(ctypes.byref(minor), 76, device) != 0:+ return ""+ caps.append("%d.%d" % (major.value, minor.value))+ return "cuda;%s;%s" % (_unpack(packed.value), ",".join(caps))+++def main():+ nvml_hint = os.environ.get("UNSLOTH_NVML_HINT", "")+ cuda_hint = os.environ.get("UNSLOTH_CUDA_HINT", "")+ answer = ""+ # Set only for the CUDA-only retry that follows a child NVML held past its deadline.+ if os.environ.get("UNSLOTH_NVIDIA_PROBE_SKIP_NVML", "") != "1":+ try:+ answer = read_nvml(nvml_hint)+ except Exception:+ answer = ""+ if not answer:+ try:+ answer = read_cuda(cuda_hint)+ except Exception:+ answer = ""+ sys.stdout.write(answer)+++main()+'@+ # Cmdlets only. Constrained Language Mode refuses New-Object ProcessStartInfo and+ # [Process]::Start, and CLM is one of the two policies that make the emitted rung decline,+ # so this launcher has to work on exactly the hosts that need it most.+ $tempRoot = if ($env:TEMP) { $env:TEMP } elseif ($env:TMPDIR) { $env:TMPDIR } else { "/tmp" }+ $stem = Join-Path $tempRoot ("unsloth-nvprobe-" + [guid]::NewGuid().ToString("N"))+ $scriptFile = "$stem.py"+ $outFile = "$stem.out"+ $errFile = "$stem.err"+ $raw = ""+ try {+ Set-Content -LiteralPath $scriptFile -Value $probeSource -Encoding UTF8 -ErrorAction Stop+ # Whole seconds, rounded up, without [math]::Ceiling: CLM blocks it. PowerShell's / is+ # floating point and [int] rounds to nearest, so 10000ms must not become 11s.+ $seconds = ($TimeoutMs - ($TimeoutMs % 1000)) / 1000+ if (($TimeoutMs % 1000) -ne 0) { $seconds = $seconds + 1 }+ $seconds = [int]$seconds+ if ($seconds -lt 1) { $seconds = 1 }+ # Nothing with a space in it reaches the command line. Windows PowerShell 5.1 appends+ # each native argument verbatim, so a script under "C:\Users\First Last\AppData\Local\+ # Temp" or a hint under "C:\Program Files\NVIDIA Corporation\NVSMI" would split on its+ # spaces and the child would run something else. The script arrives on stdin and the two+ # library hints in the environment; the only arguments left are -I -S and a bare dash.+ $savedNvml = $env:UNSLOTH_NVML_HINT+ $savedCuda = $env:UNSLOTH_CUDA_HINT+ $savedSkip = $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML+ $env:UNSLOTH_NVML_HINT = $nvmlHint+ $env:UNSLOTH_CUDA_HINT = $cudaHint+ # The switch reaches this child only. An inherited value must not make a first child skip NVML.+ if ($SkipNvml) { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = "1" }+ else { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }+ try {+ $proc = Start-Process -FilePath $exe -ArgumentList @("-I", "-S", "-") -NoNewWindow -PassThru `+ -RedirectStandardInput $scriptFile -RedirectStandardOutput $outFile -RedirectStandardError $errFile -ErrorAction Stop+ } finally {+ if ($null -eq $savedNvml) { Remove-Item Env:UNSLOTH_NVML_HINT -ErrorAction SilentlyContinue }+ else { $env:UNSLOTH_NVML_HINT = $savedNvml }+ if ($null -eq $savedCuda) { Remove-Item Env:UNSLOTH_CUDA_HINT -ErrorAction SilentlyContinue }+ else { $env:UNSLOTH_CUDA_HINT = $savedCuda }+ if ($null -eq $savedSkip) { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }+ else { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = $savedSkip }+ }+ if (-not $proc) { return "" }+ # -InputObject and an error variable, never $proc.Id or $proc.HasExited. Constrained+ # Language Mode permits property reads only on its allowed type list and+ # System.Diagnostics.Process is not on it, so reading either one throws on exactly the+ # hosts this rung exists for. Handing the object to a cmdlet keeps the access inside+ # compiled code, where the language mode does not reach.+ $waitError = $null+ Wait-Process -InputObject $proc -Timeout $seconds -ErrorAction SilentlyContinue -ErrorVariable waitError+ if ($waitError) {+ $script:NvidiaPythonProbeTimedOut = $true+ try { Stop-Process -InputObject $proc -Force -ErrorAction SilentlyContinue } catch { }+ # Let the killed child release its redirected files before the finally deletes them.+ Wait-Process -InputObject $proc -Timeout 2 -ErrorAction SilentlyContinue+ return ""+ }+ $raw = "$(Get-Content -LiteralPath $outFile -Raw -ErrorAction SilentlyContinue)"+ } catch { return "" }+ finally {+ foreach ($stale in @($scriptFile, $outFile, $errFile)) {+ Remove-Item -LiteralPath $stale -Force -ErrorAction SilentlyContinue+ }+ }+ return "$raw".Trim()+}+# "source;cudaMajor;cudaMinor;cap,cap" from NVML, else the CUDA driver API; "" when neither# answers. Versions are major*1000 + minor*10. One runspace + deadline per reader: a shared one let slow NVML starve CUDA.+# A wedged driver can block inside the library, and each deadline leaves its runspace behind.+# The Python rung below spends what the emitted rung left of ONE budget (a per-reader bound for+# NVML plus one for CUDA), so the worst-case wall clock is the emitted rung's own. Its first child+# reads NVML then the CUDA driver API under at most one per-reader bound; only when that child is+# killed at its bound (a hung NVML) and at least 2 s remain does a second child read the CUDA+# driver API alone with the rest, so a hung NVML no longer starves a healthy CUDA driver API.function Read-NvidiaLibraryRaw {param([int]$TimeoutMs = 30000)+ $deadline = (Get-Date).AddMilliseconds($TimeoutMs * 2)+ $native = ""$type = Get-NvidiaLibraryProbeType- if (-not $type) { return "" }- $reader = {- param($T, $Which)- function Read-Nvml {- if ($T::nvmlInit_v2() -ne 0) { return "" }- try {- [uint32]$count = 0- if ($T::nvmlDeviceGetCount_v2([ref]$count) -ne 0 -or $count -eq 0) { return "" }+ if ($type) {+ $reader = {+ param($T, $Which)+ function Read-Nvml {+ if ($T::nvmlInit_v2() -ne 0) { return "" }+ try {+ [uint32]$count = 0+ if ($T::nvmlDeviceGetCount_v2([ref]$count) -ne 0 -or $count -eq 0) { return "" }+ [int]$ver = 0+ if ($T::nvmlSystemGetCudaDriverVersion_v2([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }+ $caps = @()+ for ([uint32]$i = 0; $i -lt $count; $i++) {+ [IntPtr]$dev = [IntPtr]::Zero; [int]$major = 0; [int]$minor = 0+ # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.+ if ($T::nvmlDeviceGetHandleByIndex_v2($i, [ref]$dev) -ne 0) { return "" }+ if ($T::nvmlDeviceGetCudaComputeCapability($dev, [ref]$major, [ref]$minor) -ne 0) { return "" }+ $caps += "$major.$minor"+ }+ return "nvml;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"+ } finally { $null = $T::nvmlShutdown() }+ }+ function Read-Cuda {+ # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one,+ # so a hidden pre-Turing card still caps the family. cuInit reads the mask once.+ $saved = $env:CUDA_VISIBLE_DEVICES+ Remove-Item Env:CUDA_VISIBLE_DEVICES -ErrorAction SilentlyContinue+ try { $init = $T::cuInit([uint32]0) } finally { if ($null -ne $saved) { $env:CUDA_VISIBLE_DEVICES = $saved } }+ if ($init -ne 0) { return "" }+ [int]$count = 0+ if ($T::cuDeviceGetCount([ref]$count) -ne 0 -or $count -eq 0) { return "" }[int]$ver = 0- if ($T::nvmlSystemGetCudaDriverVersion_v2([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }+ if ($T::cuDriverGetVersion([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }$caps = @()- for ([uint32]$i = 0; $i -lt $count; $i++) {- [IntPtr]$dev = [IntPtr]::Zero; [int]$major = 0; [int]$minor = 0- # One unreadable GPU voids the source: a partial list misleads the pre-Turing cap.- if ($T::nvmlDeviceGetHandleByIndex_v2($i, [ref]$dev) -ne 0) { return "" }- if ($T::nvmlDeviceGetCudaComputeCapability($dev, [ref]$major, [ref]$minor) -ne 0) { return "" }+ for ($i = 0; $i -lt $count; $i++) {+ [int]$dev = 0; [int]$major = 0; [int]$minor = 0+ if ($T::cuDeviceGet([ref]$dev, $i) -ne 0) { return "" }+ # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.+ if ($T::cuDeviceGetAttribute([ref]$major, 75, $dev) -ne 0) { return "" }+ if ($T::cuDeviceGetAttribute([ref]$minor, 76, $dev) -ne 0) { return "" }$caps += "$major.$minor"}- return "nvml;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"- } finally { $null = $T::nvmlShutdown() }- }- function Read-Cuda {- # The driver API honours CUDA_VISIBLE_DEVICES; the inventory must be the physical one,- # so a hidden pre-Turing card still caps the family. cuInit reads the mask once.- $saved = $env:CUDA_VISIBLE_DEVICES- Remove-Item Env:CUDA_VISIBLE_DEVICES -ErrorAction SilentlyContinue- try { $init = $T::cuInit([uint32]0) } finally { if ($null -ne $saved) { $env:CUDA_VISIBLE_DEVICES = $saved } }- if ($init -ne 0) { return "" }- [int]$count = 0- if ($T::cuDeviceGetCount([ref]$count) -ne 0 -or $count -eq 0) { return "" }- [int]$ver = 0- if ($T::cuDriverGetVersion([ref]$ver) -ne 0 -or $ver -lt 1000) { return "" }- $caps = @()- for ($i = 0; $i -lt $count; $i++) {- [int]$dev = 0; [int]$major = 0; [int]$minor = 0- if ($T::cuDeviceGet([ref]$dev, $i) -ne 0) { return "" }- # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.- if ($T::cuDeviceGetAttribute([ref]$major, 75, $dev) -ne 0) { return "" }- if ($T::cuDeviceGetAttribute([ref]$minor, 76, $dev) -ne 0) { return "" }- $caps += "$major.$minor"- }- return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"- }- $r = ""- try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }- return "$r"- }- foreach ($which in @("nvml", "cuda")) {- $ps = $null; $handle = $null; $r = ""- try {- $ps = [powershell]::Create()- $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)- $handle = $ps.BeginInvoke()- if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {- $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"+ return "cuda;$([int][math]::Floor($ver / 1000));$([int][math]::Floor(($ver % 1000) / 10));$($caps -join ',')"}- } catch { $r = "" }- finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }- if ($r) { return $r }- }- return ""+ $r = ""+ try { if ($Which -eq "nvml") { $r = Read-Nvml } else { $r = Read-Cuda } } catch { $r = "" }+ return "$r"+ }+ foreach ($which in @("nvml", "cuda")) {+ $ps = $null; $handle = $null; $r = ""+ try {+ $ps = [powershell]::Create()+ $null = $ps.AddScript($reader.ToString()).AddArgument($type).AddArgument($which)+ $handle = $ps.BeginInvoke()+ if ($handle.AsyncWaitHandle.WaitOne($TimeoutMs)) {+ $r = "$(@($ps.EndInvoke($handle)) | Select-Object -Last 1)"+ }+ } catch { $r = "" }+ finally { if ($ps -and $handle -and $handle.IsCompleted) { $ps.Dispose() } }+ if ($r) { $native = $r; break }+ }+ }+ if ($native) { return $native }+ $remainingMs = [int]($deadline - (Get-Date)).TotalMilliseconds+ # Not worth a child process we cannot wait out; the emitted rung already spent the budget.+ if ($remainingMs -lt 3000) { return "" }+ # Each child gets at most one per-reader bound, compared by hand: CLM refuses [math]. 2 s of+# every child's share is kept back for reaping it if it hangs, so the pair ends inside the deadline.+ $childMs = $remainingMs - 2000; if ($childMs -gt $TimeoutMs) { $childMs = $TimeoutMs }+ $raw = ""+ try { $raw = Read-NvidiaLibraryRawViaPython -TimeoutMs $childMs } catch { return "" }+ # Only a first child killed at its bound (a hung NVML) earns a CUDA-only child, with what is left.+ if (-not $script:NvidiaPythonProbeTimedOut) { return $raw }+ $remainingMs = [int]($deadline - (Get-Date)).TotalMilliseconds+ if ($remainingMs -lt 3000) { return "" }+ $childMs = $remainingMs - 2000; if ($childMs -gt $TimeoutMs) { $childMs = $TimeoutMs }+ try { return (Read-NvidiaLibraryRawViaPython -TimeoutMs $childMs -SkipNvml) } catch { return "" }}# NVIDIA inventory from the driver's own libraries (NVML, then the CUDA driver API), for a@@ -3495,11 +3749,18 @@ if (-not $HasNvidiaSmi) {}}if (-not $HasNvidiaSmi -and (Get-NvidiaLibraryInventory)) {- # The driver lists a GPU nvidia-smi cannot: the gates below read this as "GPU present",- # and the consumers stop asking nvidia-smi, absent or rejected, for the rest of the run.- $HasNvidiaSmi = $true- $script:NvidiaSmiRejected = $true- Write-StudioLine " NVIDIA GPU found through the driver library; nvidia-smi is unavailable" -ForegroundColor Gray+ # A driver too old for any CUDA wheel (below 11) is no GPU this route can serve: the Intel and+ # AMD routes still get their turn instead of the install falling to CPU.+ if ((Get-NvidiaLibraryInventory).CudaMajor -ge 11) {+ # The driver lists a GPU nvidia-smi cannot: the gates below read this as "GPU present",+ # and the consumers stop asking nvidia-smi, absent or rejected, for the rest of the run.+ $HasNvidiaSmi = $true+ $script:NvidiaSmiRejected = $true+ Write-StudioLine " NVIDIA GPU found through the driver library; nvidia-smi is unavailable" -ForegroundColor Gray+ } else {+ # Still a physical NVIDIA GPU: an installed llama.cpp prebuilt it runs is kept, as before.+ $script:NvidiaDriverLibraryOnly = $true+ }}# nvidia-smi was already resolved above and never asked which card it found, so the banner# said "NVIDIA GPU detected" on every NVIDIA host alike. compute_cap is the counterpart of the@@ -9270,6 +9531,8 @@ if ($LocalLlamaCppLinked) {@("windows-rocm", "windows-hip", "windows-arm64", "windows-vulkan")} else { @("windows-rocm", "windows-hip", "windows-vulkan") }$expectedKinds = if ($HasROCm -or $script:ROCmGfxArch) { $_rocmKinds } elseif ($_nvidiaEvidence) { $_nvidiaKinds } else { @("windows-cpu", "windows-arm64", "windows-vulkan") }+ # A pre-CUDA 11 library still proves physical NVIDIA presence, even when AMD/Intel selects torch.+ if ($script:NvidiaDriverLibraryOnly) { $expectedKinds += $_nvidiaKinds }if ($existingKind -and ($existingKind -notin $expectedKinds)) {substep "Removing mismatched llama.cpp install (found '$existingKind', need one of: $($expectedKinds -join ', '))..."Remove-Item -Recurse -Force -LiteralPath $LlamaCppDir -ErrorAction SilentlyContinuediff --git a/tests/python/test_installer_embedded_nvidia_probe.py b/tests/python/test_installer_embedded_nvidia_probe.pynew file mode 100644index 00000000000..0dfb6dfdd95--- /dev/null+++ b/tests/python/test_installer_embedded_nvidia_probe.py@@ -0,0 +1,322 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+"""Behaviour of the ctypes NVIDIA probe embedded in install.ps1 and studio/setup.ps1.++The probe recovers the CUDA version AND the per-device compute capabilities on hosts where the+installer cannot emit a P/Invoke type (Constrained Language Mode, WDAC, Dynamic Code Security).+Capabilities are the part WMI cannot supply: they feed ``$CudaArch`` into+``-DCMAKE_CUDA_ARCHITECTURES`` for the llama.cpp source build (#5854) and the pre-Turing cap.++``tests/studio/test_nvidia_python_probe_parity.ps1`` pins the probe's TEXT against+``studio/nvidia_probe.py`` and against the other copy. This file pins its BEHAVIOUR, by running+it against a fake NVML / CUDA library. The two are complementary: a defect edited identically+into both copies is invisible to a parity check and caught here.+"""++from __future__ import annotations++import ctypes+import io+import ntpath+import os+import re+import sys+import types+from contextlib import redirect_stdout+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[2]+POSIX_ONLY = pytest.mark.skipif(+ os.name == "nt", reason = "the library names below are the POSIX ones"+)+LINUX_NVML = "libnvidia-ml.so.1"+LINUX_CUDA = "libcuda.so.1"+NVML_OUT = "nvml;12;6;8.9,8.9"+CUDA_OUT = "cuda;12;6;9.0,9.0"+++def _embedded(path: Path) -> str:+ # Anchored on the function: both files carry other here-strings, and the first one is a+ # different probe.+ text = path.read_text(encoding = "utf-8")+ at = text.index("function Read-NvidiaLibraryRawViaPython")+ match = re.search(r"\$probeSource = @'\n(.*?)\n'@\n", text[at:], re.S)+ assert match, f"no probe here-string in {path}"+ return match.group(1)+++INSTALL_PROBE = _embedded(ROOT / "install.ps1")+SETUP_PROBE = _embedded(ROOT / "studio" / "setup.ps1")+++def test_both_copies_are_byte_identical():+ # Leading whitespace IS Python syntax, so this is a byte comparison, not a stripped one.+ assert INSTALL_PROBE == SETUP_PROBE+++def test_the_probe_is_stdlib_only():+ imports = set(re.findall(r"(?m)^\s*(?:import|from)\s+([A-Za-z_][\w.]*)", INSTALL_PROBE))+ assert imports <= {"ctypes", "os", "sys"}, imports+++def test_the_probe_compiles():+ compile(INSTALL_PROBE, "embedded_probe.py", "exec")+++class _Fn:+ # The probe sets .restype / .argtypes on every export.+ def __init__(self, impl):+ self._impl, self.restype, self.argtypes = impl, None, None++ def __call__(self, *args):+ return self._impl(*args)+++class _Lib:+ def __init__(self, table):+ self._table = table+ self.calls: list[str] = []++ def __getattr__(self, name):+ if name not in self._table:+ raise AttributeError(name)+ return _Fn(lambda *args: (self.calls.append(name), self._table[name](*args))[1])+++def _run(+ libs,+ hints = ("", ""),+ loaded = None,+) -> str:+ """Execute the probe with ctypes.CDLL replaced, and return what it wrote to stdout."""++ def fake_cdll(name):+ if loaded is not None:+ loaded.append(name)+ if libs.get(name) is None:+ raise OSError(f"cannot load {name}")+ return libs[name]++ shim = types.ModuleType("ctypes")+ for attr in ("c_int", "c_uint", "c_void_p", "POINTER"):+ setattr(shim, attr, getattr(ctypes, attr))+ shim.CDLL = fake_cdll+ # Identity byref, so a fake export can write through to the caller's c_int.+ shim.byref = lambda obj: obj+ buffer = io.StringIO()+ with pytest.MonkeyPatch.context() as mp, redirect_stdout(buffer):+ mp.setitem(sys.modules, "ctypes", shim)+ mp.setenv("UNSLOTH_NVML_HINT", hints[0])+ mp.setenv("UNSLOTH_CUDA_HINT", hints[1])+ exec(compile(INSTALL_PROBE, "embedded_probe.py", "exec"), {})+ return buffer.getvalue()+++def _put(value):+ def write(out):+ out.value = value+ return 0++ return write+++def _nvml(+ count = 2,+ packed = 12060,+ caps = ((8, 9), (8, 9)),+ bad_handle = -1,+ bad_cap = -1,+ init = 0,+):+ """A fake libnvidia-ml. bad_handle / bad_cap make that device index fail."""++ def get_handle(index, out):+ if index == bad_handle:+ return 999+ out.value = 0x1000 + index+ return 0++ def get_cap(handle, major, minor):+ index = handle.value - 0x1000 if handle.value else 0+ if index == bad_cap:+ return 999+ major.value, minor.value = caps[index]+ return 0++ return _Lib(+ {+ "nvmlInit_v2": lambda: init,+ "nvmlShutdown": lambda: 0,+ "nvmlDeviceGetCount_v2": _put(count),+ "nvmlSystemGetCudaDriverVersion_v2": _put(packed),+ "nvmlDeviceGetHandleByIndex_v2": get_handle,+ "nvmlDeviceGetCudaComputeCapability": get_cap,+ }+ )+++def _cuda(+ count = 2,+ packed = 12060,+ caps = ((9, 0), (9, 0)),+ init = 0,+ seen = None,+ attrs = None,+):+ def cu_init(flags):+ if seen is not None:+ # What the driver would see: the mask at the moment cuInit reads it.+ seen.append(os.environ.get("CUDA_VISIBLE_DEVICES"))+ return init++ def get_device(out, index):+ out.value = index+ return 0++ def get_attr(out, attr, device):+ if attrs is not None:+ attrs.append(attr)+ out.value = caps[device.value][0 if attr == 75 else 1]+ return 0++ return _Lib(+ {+ "cuInit": cu_init,+ "cuDriverGetVersion": _put(packed),+ "cuDeviceGetCount": _put(count),+ "cuDeviceGet": get_device,+ "cuDeviceGetAttribute": get_attr,+ }+ )+++@POSIX_ONLY+ "nvml, cuda, expected",+ [+ pytest.param({}, None, NVML_OUT, id = "healthy_two_gpu_nvml"),+ pytest.param(+ {"packed": 13010}, None, "nvml;13;1;8.9,8.9", id = "version_is_major_1000_minor_10"+ ),+ # A partial list would misreport the lowest capability and so the pre-Turing cap.+ pytest.param({"bad_handle": 1}, None, "", id = "unreadable_handle_voids_source"),+ pytest.param({"bad_cap": 1}, None, "", id = "unreadable_capability_voids_source"),+ pytest.param({"count": 0}, None, "", id = "no_devices"),+ # Below 1000 the packed form cannot encode a real CUDA version.+ pytest.param({"packed": 999}, None, "", id = "nonsense_driver_version"),+ pytest.param({"init": 1}, {}, CUDA_OUT, id = "failed_nvml_init_falls_through"),+ pytest.param(None, {}, CUDA_OUT, id = "nvml_absent_uses_driver_api"),+ pytest.param({}, {}, NVML_OUT, id = "nvml_preferred_when_both_answer"),+ pytest.param(None, None, "", id = "neither_library"),+ pytest.param(None, {"init": 1}, "", id = "failed_cuinit"),+ ],+)+def test_probe_output(nvml, cuda, expected):+ libs = {+ LINUX_NVML: None if nvml is None else _nvml(**nvml),+ LINUX_CUDA: None if cuda is None else _cuda(**cuda),+ }+ assert _run(libs) == expected+++@POSIX_ONLY+def test_a_partial_read_still_shuts_nvml_down():+ lib = _nvml(bad_cap = 1)+ _run({LINUX_NVML: lib, LINUX_CUDA: None})+ assert "nvmlShutdown" in lib.calls+++@POSIX_ONLY[email protected]("value, nvml_first", [("1", False), ("0", True)])+def test_the_skip_nvml_switch(monkeypatch, value, nvml_first):+ # "1" is the CUDA-only retry after a child NVML held past its deadline: NVML is never loaded+ # or initialised, and the driver API is. Any other value still reads NVML first.+ monkeypatch.setenv("UNSLOTH_NVIDIA_PROBE_SKIP_NVML", value)+ nvml, cuda = _nvml(), _cuda()+ out = _run({LINUX_NVML: nvml, LINUX_CUDA: cuda})+ assert ("nvmlInit_v2" in nvml.calls) is nvml_first+ assert out == (NVML_OUT if nvml_first else CUDA_OUT)+ if not nvml_first:+ assert "cuInit" in cuda.calls and "cuDriverGetVersion" in cuda.calls+++@POSIX_ONLY+ "mask", ["1", None], ids = ["mask_hidden_then_restored", "absent_mask_not_invented"]+)+def test_cuinit_sees_the_physical_inventory(monkeypatch, mask):+ # A hidden pre-Turing card still caps the family, so cuInit must see no mask.+ if mask is None:+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ else:+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ seen: list = []+ _run({LINUX_NVML: None, LINUX_CUDA: _cuda(seen = seen)})+ assert seen == [None]+ assert os.environ.get("CUDA_VISIBLE_DEVICES") == mask+++@POSIX_ONLY+def test_the_capability_attributes_are_75_and_76():+ # A wrong attribute number returns a plausible integer rather than an error.+ attrs: list = []+ _run({LINUX_NVML: None, LINUX_CUDA: _cuda(attrs = attrs)})+ assert set(attrs) == {75, 76}+++@POSIX_ONLY+def test_the_output_matches_what_the_powershell_side_parses():+ """Get-NvidiaLibraryInventory splits on ';' and wants exactly four fields."""+ parts = _run({LINUX_NVML: _nvml()}).split(";")+ assert len(parts) == 4+ assert parts[0] in ("nvml", "cuda")+ assert int(parts[1]) >= 1+ assert all(re.match(r"^\d+\.\d+$", cap) for cap in parts[3].split(","))+++def _run_windows(monkeypatch, libs, hints, loaded):+ monkeypatch.setattr(os, "name", "nt")+ monkeypatch.setattr(os, "path", ntpath)+ try:+ return _run(libs, hints = hints, loaded = loaded)+ finally:+ monkeypatch.undo()+++SYSTEM32_NVML = r"C:\Windows\System32\nvml.dll"+SYSTEM32_CUDA = r"C:\Windows\System32\nvcuda.dll"+++def test_windows_loads_only_the_driver_paths_it_is_handed(monkeypatch):+ order: list[str] = []+ _run_windows(monkeypatch, {}, (SYSTEM32_NVML, SYSTEM32_CUDA), order)+ assert order == [SYSTEM32_NVML, SYSTEM32_CUDA]+++def test_a_cuda_stand_in_beside_python_is_not_an_nvidia_gpu(monkeypatch):+ # ZLUDA ships nvcuda.dll and nvml.dll that answer as a driver on an AMD host (#11736):+ # without the driver in System32 the probe must say nothing, not report a GPU.+ order: list[str] = []+ stand_in = {"nvml.dll": _nvml(), "nvcuda.dll": _cuda()}+ assert _run_windows(monkeypatch, stand_in, (SYSTEM32_NVML, SYSTEM32_CUDA), order) == ""+ assert "nvml.dll" not in order and "nvcuda.dll" not in order+++def test_a_relative_hint_is_never_loaded(monkeypatch):+ order: list[str] = []+ stand_in = {"nvml.dll": _nvml(), "nvcuda.dll": _cuda()}+ assert _run_windows(monkeypatch, stand_in, ("nvml.dll", "nvcuda.dll"), order) == ""+ assert order == []+++def test_the_real_driver_in_system32_still_answers(monkeypatch):+ order: list[str] = []+ out = _run_windows(+ monkeypatch,+ {SYSTEM32_NVML: None, SYSTEM32_CUDA: _cuda()},+ (SYSTEM32_NVML, SYSTEM32_CUDA),+ order,+ )+ assert out == CUDA_OUTdiff --git a/tests/studio/test_nvidia_library_inventory.ps1 b/tests/studio/test_nvidia_library_inventory.ps1index 8377e1f6b97..b2f09880350 100644--- a/tests/studio/test_nvidia_library_inventory.ps1+++ b/tests/studio/test_nvidia_library_inventory.ps1@@ -249,6 +249,13 @@ $HasROCm = $trueCheck "a Vulkan prebuilt is kept for any GPU vendor" ((Get-GpuPrebuiltToKeepOverSourceBuild -InstallDir $install) -eq "vulkan")$HasROCm = $falseCheck "a Vulkan prebuilt with no GPU at all is not kept" ((Get-GpuPrebuiltToKeepOverSourceBuild -InstallDir $install) -eq "")+# A driver library below CUDA 11 does not promote NVIDIA, but the card still runs Vulkan.+$script:NvidiaDriverLibraryOnly = $true+Check "a Vulkan prebuilt is kept for an NVIDIA GPU only the driver library found" ((Get-GpuPrebuiltToKeepOverSourceBuild -InstallDir $install) -eq "vulkan")+$script:NvidiaDriverLibraryOnly = $false+# Read on every keep, so each run resets it: Set-StrictMode, and a rerun in the same session.+Check "setup.ps1 resets the driver-library-only flag with the other per-run state" (+ (Get-Content -LiteralPath $setupPs1 -Raw) -match '(?m)^\$script:NvidiaSmiRejected = \$false\r?\n\$script:NvidiaDriverLibraryOnly = \$false')$script:FakeIntelAdapters = @("Intel(R) UHD Graphics 770")Check "a Vulkan prebuilt is kept for an Intel adapter that is not an XPU part" ((Get-GpuPrebuiltToKeepOverSourceBuild -InstallDir $install) -eq "vulkan")$script:FakeIntelAdapters = @()@@ -301,6 +308,46 @@ $script:FakeSmiTimeouts = 0$script:FakeSmiStdout = "| CUDA Version: 12.6 |"Check "a readable banner is still authoritative" ((Get-TorchIndexUrl) -eq "https://download.pytorch.org/whl/cu126")+# The promotion runs as written in each file: a driver below CUDA 11 must not claim the host from Intel/AMD.+function Get-PromotionBlock([string]$Text) {+ $at = $Text.IndexOf('if (-not $HasNvidiaSmi -and (Get-NvidiaLibraryInventory)) {')+ $ast = [System.Management.Automation.Language.Parser]::ParseInput($Text.Substring($at), [ref]$null, [ref]$null)+ return $ast.EndBlock.Statements[0].Extent.Text+}+foreach ($file in @($installPs1, $setupPs1)) {+ $block = Get-PromotionBlock (Get-Content -LiteralPath $file -Raw)+ foreach ($case in @(@{ Major = 10; Want = $false }, @{ Major = 11; Want = $true }, @{ Major = 12; Want = $true })) {+ $script:FakeInventory = @{ Source = "nvml"; CudaMajor = $case.Major; CudaMinor = 2; ComputeCaps = @("6.1"); Count = 1 }+ function Get-NvidiaLibraryInventory { param([int]$TimeoutSec = 10) return $script:FakeInventory }+ $HasNvidiaSmi = $false+ $script:NvidiaDriverLibraryOnly = $false+ Invoke-Expression $block+ Check "$(Split-Path -Leaf $file): a CUDA $($case.Major) driver library $(if ($case.Want) { 'promotes' } else { 'does not promote' }) NVIDIA" ($HasNvidiaSmi -eq $case.Want)+ if ($file -eq $setupPs1) {+ Check "setup.ps1: a CUDA $($case.Major) driver library still counts as an NVIDIA GPU for keeping a prebuilt" (+ ($HasNvidiaSmi -or $script:NvidiaDriverLibraryOnly) -eq $true)+ }+ }+}++# The update's kind guard runs before the keeper and deleted the bundle the keeper would have kept.+$setupText = Get-Content -LiteralPath $setupPs1 -Raw+$from = $setupText.IndexOf('$_nvidiaKinds = if (Test-WinArm64Venv) {')+$to = $setupText.IndexOf('if ($existingKind -and ($existingKind -notin $expectedKinds))', $from)+$kindBlock = $setupText.Substring($from, $to - $from)+function Test-WinArm64Venv { $false }+function Get-PinnedTorchIndexUrl { $null }+function Get-PersistedWoaTorchIndex { $null }+function Get-WoaTorchIndexMarker { $null }+$WinArm64EffectiveTorchIndexUrl = $null; $HasROCm = $false; $script:ROCmGfxArch = $null; $HasNvidiaSmi = $false+foreach ($case in @(@{ Flag = $true; Want = $true }, @{ Flag = $false; Want = $false })) {+ $script:NvidiaDriverLibraryOnly = $case.Flag+ Invoke-Expression $kindBlock+ Check "the update's kind guard $(if ($case.Want) { 'keeps' } else { 'does not keep' }) a CUDA bundle $(if ($case.Flag) { 'for an NVIDIA GPU only a pre-CUDA 11 driver library found' } else { 'with no NVIDIA evidence' })" (+ ($expectedKinds -contains "windows-cuda") -eq $case.Want)+}+$script:NvidiaDriverLibraryOnly = $false+Write-Host ""if ($failures -gt 0) { Write-Host "$failures check(s) failed" -ForegroundColor Red; exit 1 }Write-Host "All checks passed"diff --git a/tests/studio/test_nvidia_python_probe_parity.ps1 b/tests/studio/test_nvidia_python_probe_parity.ps1new file mode 100644index 00000000000..c8a3146da20--- /dev/null+++ b/tests/studio/test_nvidia_python_probe_parity.ps1@@ -0,0 +1,476 @@+#!/usr/bin/env pwsh+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+# The NVIDIA inventory's Python rung: CPython's ctypes makes the same NVML and CUDA driver calls+# the emitted type makes, so a host that cannot emit a type (Constrained Language Mode, WDAC,+# Dynamic Code Security) still gets the CUDA version AND the per-device compute capabilities.+# Capabilities are the part WMI cannot supply: they feed $CudaArch into+# -DCMAKE_CUDA_ARCHITECTURES for the llama.cpp source build (#5854) and the pre-Turing cap.+#+# This file exists to stop three kinds of drift:+# 1. the two copies of the shared region drifting apart,+# 2. the embedded probe drifting from studio/nvidia_probe.py, which makes the same calls,+# 3. the Python rung being promoted ahead of the emitted one, or spending a second budget.+# Run: pwsh -NoProfile -File tests/studio/test_nvidia_python_probe_parity.ps1++$ErrorActionPreference = "Stop"+$root = (Resolve-Path ([System.IO.Path]::Combine($PSScriptRoot, "..", ".."))).Path++$failures = 0+function Check($name, $cond) {+ if ($cond) { Write-Host " PASS $name" }+ else { Write-Host " FAIL $name" -ForegroundColor Red; $script:failures++ }+}++function Get-HelperSources($path, $names) {+ $tokens = $null; $errors = $null+ $ast = [System.Management.Automation.Language.Parser]::ParseFile($path, [ref]$tokens, [ref]$errors)+ if ($errors) { $errors | ForEach-Object { $_.ToString() }; throw "$path has parse errors" }+ $out = @()+ foreach ($name in $names) {+ $fn = $ast.FindAll({ param($n)+ $n -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $n.Name -eq $name+ }, $true)+ if ($fn.Count -lt 1) { throw "expected $name in $path, found none" }+ $out += $fn[0].Extent.Text+ }+ # A single-element return unrolls to a bare string, and [0] would then be its first+ # character. Callers wrap in @(), and this comment is why.+ return $out+}++$installPs1 = Join-Path $root "install.ps1"+$setupPs1 = Join-Path $root "studio\setup.ps1"+$probePy = Join-Path $root "studio\nvidia_probe.py"++# The embedded Python is the text between @' and '@ inside Read-NvidiaLibraryRawViaPython.+# Both files carry other here-strings, so anchor on the function rather than the first match.+# The variable is $probeSource, not $probe: install.ps1 and setup.ps1 each already carry a+# $probe here-string for the emit probe, and reusing the name made an older suite read this one.+function Get-EmbeddedProbe($path) {+ $text = [System.IO.File]::ReadAllText($path)+ $at = $text.IndexOf("function Read-NvidiaLibraryRawViaPython")+ if ($at -lt 0) { throw "no Read-NvidiaLibraryRawViaPython in $path" }+ $open = $text.IndexOf("`$probeSource = @'", $at)+ if ($open -lt 0) { throw "no probe here-string in $path" }+ $bodyStart = $text.IndexOf("`n", $open) + 1+ $close = $text.IndexOf("`n'@", $bodyStart)+ if ($close -lt 0) { throw "unterminated probe here-string in $path" }+ return $text.Substring($bodyStart, $close - $bodyStart)+}++Write-Host ""+Write-Host "=== the two copies agree ==="++$blockNames = @(+ "Get-NvidiaNvmlLibraryPath",+ "Get-NvidiaLibraryProbeType",+ "Read-NvidiaLibraryRawViaPython",+ "Read-NvidiaLibraryRaw",+ "Get-NvidiaLibraryInventory",+ "Get-NvidiaSystem32Dir"+)+$installParts = @(Get-HelperSources $installPs1 $blockNames)+$setupParts = @(Get-HelperSources $setupPs1 $blockNames)+# install.ps1 nests its helpers one level deeper; compare the two copies without indentation.+$strip = { param($text) ($text -split "`n" | ForEach-Object { $_.TrimStart() }) -join "`n" }+for ($k = 0; $k -lt $blockNames.Count; $k++) {+ Check "install.ps1 and setup.ps1 carry the same $($blockNames[$k])" `+ ((& $strip $installParts[$k]) -eq (& $strip $setupParts[$k]))+}++$installProbe = Get-EmbeddedProbe $installPs1+$setupProbe = Get-EmbeddedProbe $setupPs1+# Stripping indentation would hide a real defect here: leading whitespace IS the Python syntax,+# so the two copies must match byte for byte, not merely after a TrimStart.+Check "the embedded Python is byte-identical in both files" ($installProbe -ceq $setupProbe)+Check "the embedded Python sits at column 0, so its indentation survives both files" `+ (($installProbe -split "`n" | Where-Object { $_ -match '^import ctypes' }).Count -eq 1)++Write-Host ""+Write-Host "=== the embedded probe matches studio/nvidia_probe.py ==="++$referenceProbe = [System.IO.File]::ReadAllText($probePy)++# Every native symbol the emitted type imports, and the two attribute numbers. If the embedded+# probe and nvidia_probe.py ever disagree on one of these, they are probing different things.+$symbols = @(+ "nvmlInit_v2", "nvmlShutdown", "nvmlSystemGetCudaDriverVersion_v2",+ "nvmlDeviceGetCount_v2", "nvmlDeviceGetHandleByIndex_v2", "nvmlDeviceGetCudaComputeCapability",+ "cuInit", "cuDriverGetVersion", "cuDeviceGetCount", "cuDeviceGet", "cuDeviceGetAttribute"+)+foreach ($symbol in $symbols) {+ Check "both the embedded probe and nvidia_probe.py call $symbol" `+ (($installProbe -match [regex]::Escape($symbol)) -and ($referenceProbe -match [regex]::Escape($symbol)))+ Check "the emitted type imports $symbol too, so all three agree" `+ ($installParts[1] -match [regex]::Escape($symbol))+}++# CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR / _MINOR. A wrong number here returns a+# plausible integer rather than an error, which is exactly the drift a test has to catch.+Check "the embedded probe reads attribute 75 for the capability major" ($installProbe -match 'byref\(major\), 75,')+Check "the embedded probe reads attribute 76 for the capability minor" ($installProbe -match 'byref\(minor\), 76,')+Check "nvidia_probe.py reads the same two attribute numbers" `+ (($referenceProbe -match 'byref\(major\), 75,') -and ($referenceProbe -match 'byref\(minor\), 76,'))++# The packed CUDA version arithmetic, shared by all three implementations.+Check "the embedded probe unpacks the version as major*1000 + minor*10" `+ ($installProbe -match 'packed // 1000, \(packed % 1000\) // 10')+Check "nvidia_probe.py unpacks the version identically" `+ ($referenceProbe -match 'packed // 1000, \(packed % 1000\) // 10')+Check "the emitted rung unpacks the version identically" `+ (($installParts[3] -match 'Floor\(\$ver / 1000\)') -and ($installParts[3] -match 'Floor\(\(\$ver % 1000\) / 10\)'))++Check "the embedded probe is stdlib-only" `+ (($installProbe -match '(?m)^import ctypes, os, sys$') -and ($installProbe -notmatch '(?m)^\s*(import|from)\s+(?!ctypes|os|sys)'))++Write-Host ""+Write-Host "=== the rung stays underneath the emitted one ==="++$rawBlock = & $strip $setupParts[3]+Check "the emitted rung is attempted first" `+ ($rawBlock.IndexOf("Get-NvidiaLibraryProbeType") -lt $rawBlock.IndexOf("Read-NvidiaLibraryRawViaPython"))+Check "Python is only reached when the emitted rung returned nothing" `+ ($rawBlock -match 'if \(\$native\) \{ return \$native \}')+# One budget, not two: a driver that wedges both emitted readers must not then buy a further+# child process. The budget is one per-reader bound for NVML plus one for CUDA.+Check "both rungs share one deadline" ($rawBlock -match '\$deadline = \(Get-Date\)\.AddMilliseconds\(\$TimeoutMs \* 2\)')+Check "each emitted reader has its own deadline" `+ (($rawBlock -match 'foreach \(\$which in @\("nvml", "cuda"\)\)') -and ($rawBlock -match 'WaitOne\(\$TimeoutMs\)'))+Check "the per-reader bound defaults to 30s" ($rawBlock -match 'param\(\[int\]\$TimeoutMs = 30000\)')+# The Python rung spends what the emitted rung left, at most one per-reader bound per child: one+# child reads NVML then CUDA, and only a child killed at its bound earns a second, CUDA-only child+# with what is still left. The behaviour is driven below; these pin the shape.+Check "Python gets only what the emitted rung left, capped at one bound" `+ (($rawBlock -match '\$childMs = \$remainingMs - 2000; if \(\$childMs -gt \$TimeoutMs\) \{ \$childMs = \$TimeoutMs \}') -and+ ($rawBlock -match '\$raw = Read-NvidiaLibraryRawViaPython -TimeoutMs \$childMs\b'))+Check "the reader itself avoids [math], which CLM refuses" ($rawBlock -notmatch '\[math\]::Min')+Check "only a timed-out first child earns the CUDA-only retry" `+ (($rawBlock -match 'if \(-not \$script:NvidiaPythonProbeTimedOut\) \{ return \$raw \}') -and+ ($rawBlock -match 'Read-NvidiaLibraryRawViaPython -TimeoutMs \$childMs -SkipNvml'))+Check "an exhausted budget skips the child process entirely" ($rawBlock -match 'if \(\$remainingMs -lt 3000\) \{ return "" \}')++$pyBlock = & $strip $setupParts[2]+# The banned constructs are NAMED in this rung's own comments, explaining why they are absent.+# Matching the raw text would fail on the explanation rather than on the code, so the CLM rows+# below run against comment-free source. (Found the honest way: they failed on the comments.)+$pyCode = ($pyBlock -split "`n" | Where-Object { $_.TrimStart() -notmatch '^#' }) -join "`n"+Check "the rung compiles nothing" ($pyCode -notmatch 'Add-Type')+Check "the rung emits no type" ($pyCode -notmatch 'New-StudioEmittedNativeType|DefinePInvokeMethod')+# Constrained Language Mode is one of the two policies that make the emitted rung decline, so a+# launcher this rung cannot use on a CLM host would be absent from the population it exists for.+Check "the launcher avoids ProcessStartInfo, which CLM refuses" ($pyCode -notmatch 'ProcessStartInfo')+Check "the launcher avoids [Process]::Start, which CLM refuses" ($pyCode -notmatch '\[(System\.)?Diagnostics\.Process\]::Start|\[Process\]::Start')+Check "the launcher avoids [System.IO.Path], which CLM refuses" ($pyCode -notmatch '\[System\.IO\.Path\]')+Check "the launcher avoids [math], which CLM refuses" ($pyCode -notmatch '\[math\]::')+# Constrained Language Mode permits property reads only on its documented allowed-type list, and+# System.Diagnostics.Process and System.IO.FileInfo are on neither. Reading .Id, .HasExited,+# .ExitCode or a path off a FileInfo throws on precisely the hosts where the emitted rung already+# declined, which is the whole population this rung serves. Handing the object to a cmdlet keeps+# the access inside compiled code. This is not hypothetical: the sibling early-python launcher+# shipped with exactly these reads and failed three checks on windows-latest under 5.1.+foreach ($blocked in @("\.Id\b", "\.HasExited\b", "\.ExitCode\b", "\.FullName\b", "New-TemporaryFile")) {+ Check "the launcher avoids $($blocked -replace '\\[.b]', '') , which CLM refuses on a Process or FileInfo" (+ $pyCode -notmatch $blocked)+}+Check "the launcher waits by object, not by pid" ($pyCode -match 'Wait-Process -InputObject \$proc')+Check "the launcher kills by object too" ($pyCode -match 'Stop-Process -InputObject \$proc')+Check "a killed child is waited for before its files are deleted" (+ $pyCode -match 'Stop-Process -InputObject \$proc[^\n]*\n(\s*#[^\n]*\n)?\s*Wait-Process -InputObject \$proc -Timeout \d+')+Check "the timeout is detected through an error variable" ($pyCode -match '-ErrorVariable waitError')+# "[ref] - Casting an object to type [ref] ... is not permitted."+Check "the launcher casts nothing to [ref]" ($pyCode -notmatch '\[ref\]\s*\$')+# The stripping must not be so eager that it deletes the launcher itself.+Check "the comment-free source still contains the launcher" ($pyCode -match 'Start-Process -FilePath \$exe')+Check "the interpreter comes from a per-file hook, not from the shared region" `+ (($pyBlock -match 'Get-NvidiaProbePythonExe') -and ($pyBlock -notmatch 'function Get-NvidiaProbePythonExe'))+foreach ($file in @($installPs1, $setupPs1)) {+ Check "$(Split-Path -Leaf $file) defines its own Get-NvidiaProbePythonExe" `+ ([System.IO.File]::ReadAllText($file) -match 'function Get-NvidiaProbePythonExe')+}+Check "the probe is switchable off" ($pyBlock -match 'UNSLOTH_NVIDIA_PYTHON_PROBE')+Check "the temp files are always cleaned up" ($pyBlock -match 'finally \{[\s\S]*Remove-Item -LiteralPath \$stale')+# Windows PowerShell 5.1 appends each native argument verbatim, so anything with a space in it+# splits and the child silently runs something else. Nothing with a space may reach argv.+Check "only space-free arguments reach the command line" ($pyCode -match 'ArgumentList @\("-I", "-S", "-"\)')+Check "the script arrives on stdin, not as a path argument" ($pyCode -match '-RedirectStandardInput \$scriptFile')+Check "the library hints travel in the environment" `+ (($pyCode -match '\$env:UNSLOTH_NVML_HINT = \$nvmlHint') -and ($pyCode -match '\$env:UNSLOTH_CUDA_HINT = \$cudaHint'))+Check "the hint variables are restored afterwards" `+ ($pyCode -match 'finally \{[\s\S]*Remove-Item Env:UNSLOTH_NVML_HINT')+Check "the embedded probe reads the hints from the environment" `+ (($installProbe -match 'os\.environ\.get\("UNSLOTH_NVML_HINT"') -and ($installProbe -match 'os\.environ\.get\("UNSLOTH_CUDA_HINT"'))++Write-Host ""+Write-Host "=== the installer uses the interpreter this run installed ==="++# Get-StudioEarlyPython memoises, including a MISS. On a fresh host it is first called by the+# install-lock path, finds no Python, and caches $null for the rest of the run. The inventory is+# not read until thousands of lines later, by which point this install has created a managed+# interpreter and a venv. Asking only the cache would decline on exactly the fresh install where+# nvidia-smi is also most likely absent, and the host would take CPU wheels with a working card.+$hookSrc = @(Get-HelperSources $installPs1 @("Get-NvidiaProbePythonExe"))[0]+Invoke-Expression $hookSrc+$script:EarlyCalled = $false+function Get-StudioEarlyPython { $script:EarlyCalled = $true; return "/early/python3" }++$hookDir = Join-Path ([System.IO.Path]::GetTempPath()) ("unsloth-hook-" + [guid]::NewGuid().ToString("N"))+New-Item -ItemType Directory -Path $hookDir | Out-Null+try {+ $fakeVenv = Join-Path $hookDir "venv-python"+ $fakeManaged = Join-Path $hookDir "managed-python"+ [System.IO.File]::WriteAllText($fakeVenv, "")+ [System.IO.File]::WriteAllText($fakeManaged, "")++ $VenvPython = $fakeVenv; $ManagedPythonPath = $fakeManaged+ $script:EarlyCalled = $false+ Check "the venv interpreter wins when it exists" ((Get-NvidiaProbePythonExe) -eq $fakeVenv)+ Check "and the memoising ladder is not consulted at all" ($script:EarlyCalled -eq $false)++ # Before the venv is built but after the managed interpreter is installed.+ $VenvPython = Join-Path $hookDir "not-created-yet"+ $script:EarlyCalled = $false+ Check "the managed interpreter is next" ((Get-NvidiaProbePythonExe) -eq $fakeManaged)+ Check "and it too skips the ladder" ($script:EarlyCalled -eq $false)++ # Neither present: the ladder is the fallback, which is the pre-existing behaviour.+ $VenvPython = Join-Path $hookDir "nope-1"; $ManagedPythonPath = Join-Path $hookDir "nope-2"+ $script:EarlyCalled = $false+ Check "with neither on disk the ladder still answers" ((Get-NvidiaProbePythonExe) -eq "/early/python3")+ Check "and the ladder really was the source" ($script:EarlyCalled -eq $true)++ # A path recorded but never created must not be handed out: Test-Path is the discriminator,+ # not whether the variable happens to be set.+ $VenvPython = $null; $ManagedPythonPath = $null+ Check "unset variables fall through rather than returning empty" (+ (Get-NvidiaProbePythonExe) -eq "/early/python3")++ $saved = $env:UNSLOTH_EARLY_PYTHON_PROBE+ try {+ $env:UNSLOTH_EARLY_PYTHON_PROBE = "0"+ $VenvPython = $fakeVenv+ Check "the kill switch still wins over the venv interpreter" ((Get-NvidiaProbePythonExe) -eq "")+ } finally {+ if ($null -eq $saved) { Remove-Item Env:UNSLOTH_EARLY_PYTHON_PROBE -ErrorAction SilentlyContinue }+ else { $env:UNSLOTH_EARLY_PYTHON_PROBE = $saved }+ }+} finally {+ Remove-Item -LiteralPath $hookDir -Recurse -Force -ErrorAction SilentlyContinue+}++# The ordering that makes the above reachable at all: both variables are assigned before the+# first Get-NvidiaLibraryInventory call, or the hook would read $null however it is written.+$installText = [System.IO.File]::ReadAllText($installPs1)+$venvAt = $installText.IndexOf('$VenvPython = Join-Path $VenvDir')+$managedAt = $installText.IndexOf('$ManagedPythonPath = (Resolve-Path')+$firstInventoryAt = $installText.IndexOf('if (-not $HasNvidiaSmi -and (Get-NvidiaLibraryInventory))')+Check "the venv interpreter is named before the inventory is first read" (+ $venvAt -gt 0 -and $firstInventoryAt -gt $venvAt)+Check "the managed interpreter is named before it too" (+ $managedAt -gt 0 -and $firstInventoryAt -gt $managedAt)++Write-Host ""+Write-Host "=== the timeout is whole seconds, rounded up, without [math]::Ceiling ==="++# PowerShell's / is floating point and [int] rounds to NEAREST, so the C "+999" ceiling idiom+# turns 10000ms into 11s. Run the shipped expression rather than a retyped copy of it.+$ceilLines = @($pyBlock -split "`n" | Where-Object {+ $_ -match '^\$seconds = ' -or $_ -match '^if \(\(\$TimeoutMs % 1000\) -ne 0\)' -or $_ -match '^if \(\$seconds -lt 1\)'+})+Check "the ceiling is four lines of integer arithmetic" ($ceilLines.Count -eq 4)+$ceilExpr = [scriptblock]::Create(($ceilLines -join "`n") + "`nreturn `$seconds")+foreach ($row in @(+ @{ Ms = 10000; Want = 10 }, @{ Ms = 10001; Want = 11 }, @{ Ms = 9999; Want = 10 },+ @{ Ms = 1; Want = 1 }, @{ Ms = 0; Want = 1 }, @{ Ms = 2500; Want = 3 }, @{ Ms = 20000; Want = 20 }+)) {+ $TimeoutMs = $row.Ms+ $got = & $ceilExpr+ Check "$($row.Ms) ms is $($row.Want) s" ($got -eq $row.Want)+}++Write-Host ""+Write-Host "=== behaviour, driven through the real functions ==="++Invoke-Expression ($setupParts[0]) # Get-NvidiaNvmlLibraryPath+Invoke-Expression ($setupParts[5]) # Get-NvidiaSystem32Dir: Windows names the driver libraries under it+Invoke-Expression ($setupParts[2]) # Read-NvidiaLibraryRawViaPython+Invoke-Expression ($setupParts[3]) # Read-NvidiaLibraryRaw+Invoke-Expression ($setupParts[4]) # Get-NvidiaLibraryInventory+# The emitted rung declines, which is the whole point: this is what a Constrained Language Mode+# or WDAC host sees, and the capabilities below are recovered with nothing emitted.+function Get-NvidiaLibraryProbeType { return $null }++$script:PythonExe = ""+function Get-NvidiaProbePythonExe { return $script:PythonExe }++Check "no interpreter means no answer, not an error" ((Read-NvidiaLibraryRawViaPython -TimeoutMs 5000) -eq "")++$python = $null+foreach ($name in @("python3", "python")) {+ $cmd = Get-Command $name -ErrorAction SilentlyContinue+ if ($cmd) { $python = $cmd.Source; break }+}++if (-not $python) {+ Write-Host " SKIP no Python on this runner; the behavioural rows need one"+} else {+ $script:PythonExe = $python+ $saved = $env:UNSLOTH_NVIDIA_PYTHON_PROBE+ try {+ $env:UNSLOTH_NVIDIA_PYTHON_PROBE = "0"+ Check "UNSLOTH_NVIDIA_PYTHON_PROBE=0 declines before spawning anything" `+ ((Read-NvidiaLibraryRawViaPython -TimeoutMs 5000) -eq "")+ } finally {+ if ($null -eq $saved) { Remove-Item Env:UNSLOTH_NVIDIA_PYTHON_PROBE -ErrorAction SilentlyContinue }+ else { $env:UNSLOTH_NVIDIA_PYTHON_PROBE = $saved }+ }++ $answer = Read-NvidiaLibraryRawViaPython -TimeoutMs 30000+ # A runner without an NVIDIA driver is the common case and must answer "" rather than throw.+ if (-not $answer) {+ Check "no driver on this host answers empty, not an error" ($answer -eq "")+ } else {+ Check "the answer names a source the inventory understands" ($answer -match '^(nvml|cuda);')+ Check "the answer carries a version and at least one capability" `+ ($answer -match '^(nvml|cuda);\d+;\d+;\d+\.\d+(,\d+\.\d+)*$')+ # Through the real Read-NvidiaLibraryRaw, with the emitted rung declining. Every row+ # below is guarded on a non-null inventory: $null -eq $null would pass them all.+ $inventory = Get-NvidiaLibraryInventory -TimeoutSec 30+ Check "the inventory parses the answer with nothing emitted" ($null -ne $inventory)+ Check "the inventory recovered the compute capabilities" `+ ($null -ne $inventory -and $inventory.ComputeCaps.Count -ge 1 -and $inventory.ComputeCaps[0] -match '^\d+\.\d+$')+ Check "the inventory device count matches the capability list" `+ ($null -ne $inventory -and $inventory.Count -eq $inventory.ComputeCaps.Count)+ Check "the CUDA major version is plausible" ($null -ne $inventory -and $inventory.CudaMajor -ge 1)+ Check "the inventory names the source the raw answer named" `+ ($null -ne $inventory -and $answer.StartsWith("$($inventory.Source);"))+ }++ # The launcher must not leave the interpreter or its scratch files behind.+ $tempRoot = if ($env:TEMP) { $env:TEMP } elseif ($env:TMPDIR) { $env:TMPDIR } else { "/tmp" }+ $litter = @(Get-ChildItem -LiteralPath $tempRoot -Filter "unsloth-nvprobe-*" -ErrorAction SilentlyContinue)+ Check "the launcher leaves no scratch files behind" ($litter.Count -eq 0)+}++Write-Host ""+Write-Host "=== a hung NVML earns one CUDA-only child, and nothing else does ==="++# The embedded probe honours the switch, so the retry really skips NVML.+Check "the embedded probe skips NVML only when the switch is 1" `+ ($installProbe -match 'if os\.environ\.get\("UNSLOTH_NVIDIA_PROBE_SKIP_NVML", ""\) != "1":')+Check "the launcher sets the switch only for a -SkipNvml child" `+ (($pyBlock -match 'if \(\$SkipNvml\) \{ \$env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = "1" \}') -and+ ($pyBlock -match 'finally \{[\s\S]*Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML'))++# Stubbed launcher: the real Read-NvidiaLibraryRaw (emitted rung declining, as under CLM/WDAC)+# resolves the stub through dynamic scope. A TIMEOUT outcome may also burn some of the budget.+function Invoke-RawWithStub($outcomes, [int]$TimeoutMs = 30000, [int]$TimeoutSleepMs = 0) {+ $script:StubCalls = @()+ $script:StubOutcomes = [System.Collections.ArrayList]@($outcomes)+ $script:StubSleepMs = $TimeoutSleepMs+ return & {+ function Read-NvidiaLibraryRawViaPython {+ param([int]$TimeoutMs = 10000, [switch]$SkipNvml)+ $script:StubCalls += , @{ TimeoutMs = $TimeoutMs; SkipNvml = [bool]$SkipNvml }+ $next = $script:StubOutcomes[0]; $script:StubOutcomes.RemoveAt(0)+ $script:NvidiaPythonProbeTimedOut = ($next -eq "TIMEOUT")+ if ($next -eq "TIMEOUT") {+ if ($script:StubSleepMs -gt 0) { Start-Sleep -Milliseconds $script:StubSleepMs }+ return ""+ }+ return $next+ }+ Read-NvidiaLibraryRaw -TimeoutMs $TimeoutMs+ }+}+$got = Invoke-RawWithStub @("TIMEOUT", "cuda;12;8;8.9")+Check "stub: a timed-out first child is followed by exactly one more" ($script:StubCalls.Count -eq 2)+Check "stub: the first child reads NVML, the second skips it" `+ ($script:StubCalls.Count -eq 2 -and -not $script:StubCalls[0].SkipNvml -and $script:StubCalls[1].SkipNvml)+Check "stub: each child is capped at the 30s per-reader bound" `+ ($script:StubCalls.Count -eq 2 -and $script:StubCalls[0].TimeoutMs -eq 30000 -and+ $script:StubCalls[1].TimeoutMs -le 30000 -and $script:StubCalls[1].TimeoutMs -ge 25000)+Check "stub: the CUDA-only answer is returned" ($got -eq "cuda;12;8;8.9")+$got = Invoke-RawWithStub @("")+Check "stub: an empty answer spawns no second child" ($script:StubCalls.Count -eq 1 -and $got -eq "")+$got = Invoke-RawWithStub @("nvml;13;0;12.0")+Check "stub: an answer spawns no second child" ($script:StubCalls.Count -eq 1 -and $got -eq "nvml;13;0;12.0")+$got = Invoke-RawWithStub @("TIMEOUT", "TIMEOUT")+Check "stub: a CUDA-only child that also hangs ends the search" ($script:StubCalls.Count -eq 2 -and $got -eq "")+# Shared deadline 2 x 1500 ms; the first child burns 1200 ms of it, leaving under 3 s.+$got = Invoke-RawWithStub @("TIMEOUT", "cuda;12;8;8.9") -TimeoutMs 1500 -TimeoutSleepMs 1200+Check "stub: no CUDA-only child when under 3 s of the shared budget remain" ($script:StubCalls.Count -eq 1 -and $got -eq "")+# Shared deadline 2 x 4000 ms; the first child hangs for 3000 ms. Each child's bound leaves 2 s to+# reap it, so the CUDA-only child gets what is left minus that, and the pair ends by the deadline.+$got = Invoke-RawWithStub @("TIMEOUT", "cuda;12;8;8.9") -TimeoutMs 4000 -TimeoutSleepMs 3000+Check "stub: every child's bound leaves 2 s of the shared deadline to reap it" (+ $script:StubCalls.Count -eq 2 -and $script:StubCalls[0].TimeoutMs -eq 4000 -and+ $script:StubCalls[1].TimeoutMs -le 3000 -and $script:StubCalls[1].TimeoutMs -ge 2000)+# Shared deadline 2 x 5000 ms; the first child burns 5000 ms, so the retry gets only the rest.+$got = Invoke-RawWithStub @("TIMEOUT", "cuda;12;8;8.9") -TimeoutMs 5000 -TimeoutSleepMs 5000+Check "stub: the CUDA-only child gets what is left of the shared budget, not a fresh bound" `+ ($script:StubCalls.Count -eq 2 -and $script:StubCalls[0].TimeoutMs -eq 5000 -and+ $script:StubCalls[1].TimeoutMs -lt 5000 -and $script:StubCalls[1].TimeoutMs -ge 2000 -and $got -eq "cuda;12;8;8.9")++# End to end through the real launcher, with a fake interpreter that logs the switch it sees and+# hangs like a wedged NVML unless the switch is set. POSIX sh only: a .cmd stand-in would leave its+# sleeping child holding the output file on Windows.+if ($IsWindows -or $env:OS -eq "Windows_NT") {+ Write-Host " SKIP the fake-interpreter rows need a POSIX shell"+} else {+ $fakeDir = Join-Path ([System.IO.Path]::GetTempPath()) ("unsloth-fakepy-" + [guid]::NewGuid().ToString("N"))+ New-Item -ItemType Directory -Path $fakeDir | Out-Null+ $fakePy = Join-Path $fakeDir "python"+ $fakeLog = Join-Path $fakeDir "calls.log"+ $fakeBody = @(+ '#!/bin/sh'+ 'cat > /dev/null'+ 'echo "skip=${UNSLOTH_NVIDIA_PROBE_SKIP_NVML:-unset}" >> "$UNSLOTH_FAKE_PY_LOG"'+ 'if [ "$UNSLOTH_NVIDIA_PROBE_SKIP_NVML" = "1" ]; then printf "cuda;12;8;8.9"; exit 0; fi'+ 'if [ "$UNSLOTH_FAKE_PY_MODE" = "hang" ]; then exec sleep 30; fi'+ 'exit 0'+ ) -join "`n"+ [System.IO.File]::WriteAllText($fakePy, "$fakeBody`n")+ & chmod +x $fakePy+ $savedPy = $script:PythonExe+ $savedSkipEnv = $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML+ try {+ $script:PythonExe = $fakePy+ $env:UNSLOTH_FAKE_PY_LOG = $fakeLog+ # A value already in this shell must reach no first child and must survive the calls.+ $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = "caller"++ # Shared deadline 2 x 5000 ms: the first child is killed at 5 s and reaped, leaving the retry+ # its share less the 2 s kept back for reaping it.+ $env:UNSLOTH_FAKE_PY_MODE = "hang"+ Remove-Item -LiteralPath $fakeLog -ErrorAction SilentlyContinue+ $got = Read-NvidiaLibraryRaw -TimeoutMs 5000+ $seen = @(Get-Content -LiteralPath $fakeLog -ErrorAction SilentlyContinue)+ Check "live: a first child killed at its bound triggers exactly one more" ($seen.Count -eq 2)+ Check "live: the first child saw no switch, the second saw 1" (($seen -join ",") -eq "skip=unset,skip=1")+ Check "live: the CUDA-only answer is returned" ($got -eq "cuda;12;8;8.9")+ Check "live: the caller's switch value is restored" ($env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -eq "caller")++ $env:UNSLOTH_FAKE_PY_MODE = "empty"+ Remove-Item -LiteralPath $fakeLog -ErrorAction SilentlyContinue+ $got = Read-NvidiaLibraryRaw -TimeoutMs 3000+ $seen = @(Get-Content -LiteralPath $fakeLog -ErrorAction SilentlyContinue)+ Check "live: a first child that answers empty spawns no second" ($seen.Count -eq 1 -and $got -eq "")+ Check "live: the switch is still restored after a single child" ($env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -eq "caller")++ Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue+ $env:UNSLOTH_FAKE_PY_MODE = "hang"+ $null = Read-NvidiaLibraryRaw -TimeoutMs 5000+ Check "live: an unset switch is left unset" ($null -eq $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML)+ } finally {+ $script:PythonExe = $savedPy+ if ($null -eq $savedSkipEnv) { Remove-Item Env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML -ErrorAction SilentlyContinue }+ else { $env:UNSLOTH_NVIDIA_PROBE_SKIP_NVML = $savedSkipEnv }+ Remove-Item Env:UNSLOTH_FAKE_PY_LOG, Env:UNSLOTH_FAKE_PY_MODE -ErrorAction SilentlyContinue+ Remove-Item -LiteralPath $fakeDir -Recurse -Force -ErrorAction SilentlyContinue+ }+}++Write-Host ""+if ($failures -gt 0) { Write-Host "$failures check(s) failed" -ForegroundColor Red; exit 1 }+Write-Host "All checks passed"