Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop a CPU torch handover from disarming the flavor invariant (#10906)

Daniel Han24f06276f1d1
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index a3c535a33d4..247f8224b8f 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -2515,48 +2515,14 @@ def _has_usable_nvidia_gpu() -> bool:
if cvd is not None and cvd.strip() in ("", "-1"):
return False
- def _lists_a_gpu(exe: str) -> bool:
- try:
- result = subprocess.run(
- [exe, "-L"],
- stdout = subprocess.PIPE,
- stderr = subprocess.DEVNULL,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- timeout = 10,
- )
- except Exception:
- return False
- return result.returncode == 0 and "GPU " in result.stdout
-
# A stale nvidia-smi on PATH exits non-zero listing nothing, so try every
# candidate: install.ps1 / setup.ps1 also gate the fixed-location fallback
# on the GPU check failing, not on the PATH lookup missing.
- candidates = []
_path_exe = shutil.which("nvidia-smi")
- if _path_exe:
- candidates.append(_path_exe)
- if IS_WINDOWS:
- candidates.extend(
- (
- os.path.join(
- os.environ.get("ProgramFiles", r"C:\Program Files"),
- "NVIDIA Corporation",
- "NVSMI",
- "nvidia-smi.exe",
- ),
- os.path.join(
- os.environ.get("SystemRoot", r"C:\Windows"),
- "System32",
- "nvidia-smi.exe",
- ),
- )
- )
- for _candidate in candidates:
+ for _candidate in _nvidia_smi_candidates():
if _candidate != _path_exe and not os.path.isfile(_candidate):
continue
- if _lists_a_gpu(_candidate):
+ if _nvidia_smi_lists_a_gpu(_candidate):
return True
# Fallback: /proc/driver/nvidia/gpus/ has one subdir per GPU whatever nvidia-smi does.
if sys.platform != "win32":
@@ -3345,13 +3311,83 @@ def _install_bnb_windows_rocm() -> bool:
return True
-def _nvidia_smi_path() -> "str | None":
- """nvidia-smi from PATH, falling back to the canonical Linux install path a
- stripped-down PATH (systemd units, cron) can miss."""
+def _nvidia_smi_candidates() -> "list[str]":
+ """Every place nvidia-smi is worth looking for, PATH first.
+
+ One list, because two callers that disagree about where nvidia-smi lives disagree
+ about the host: _has_usable_nvidia_gpu would find the driver through the Windows
+ fixed locations while _detect_cuda_torch_index_url, reading PATH only, fell back to
+ its cu126 default and recorded a family the driver never reported.
+ """
+ candidates = []
exe = shutil.which("nvidia-smi")
- if not exe and os.path.isfile("/usr/bin/nvidia-smi"):
- exe = "/usr/bin/nvidia-smi"
- return exe
+ if exe:
+ candidates.append(exe)
+ if IS_WINDOWS:
+ # The locations install.ps1 / setup.ps1 use; nvidia-smi.exe is routinely off PATH.
+ candidates.extend(
+ (
+ os.path.join(
+ os.environ.get("ProgramFiles", r"C:\Program Files"),
+ "NVIDIA Corporation",
+ "NVSMI",
+ "nvidia-smi.exe",
+ ),
+ os.path.join(
+ os.environ.get("SystemRoot", r"C:\Windows"),
+ "System32",
+ "nvidia-smi.exe",
+ ),
+ )
+ )
+ else:
+ # The canonical Linux path a stripped-down PATH (systemd units, cron) can miss.
+ candidates.append("/usr/bin/nvidia-smi")
+ return candidates
+
+
+def _nvidia_smi_lists_a_gpu(exe: str) -> bool:
+ """Whether this nvidia-smi actually enumerates a GPU.
+
+ The predicate both probes have to agree on. A stale copy can exit 0 and print a
+ perfectly parseable "CUDA Version:" banner while `-L` lists nothing, which is the
+ state setup.ps1's Test-NvidiaSmiHasGpu rejects.
+ """
+ try:
+ result = subprocess.run(
+ [exe, "-L"],
+ stdout = subprocess.PIPE,
+ stderr = subprocess.DEVNULL,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 10,
+ )
+ except Exception:
+ return False
+ return result.returncode == 0 and "GPU " in result.stdout
+
+
+def _nvidia_smi_usable_candidates() -> "list[str]":
+ """Candidates worth actually running: the PATH result plus every file that exists.
+
+ `shutil.which` already proved its result runnable, so it is kept without an
+ os.path.isfile check -- that would reject a bare "nvidia-smi" relative to a CWD it
+ does not live in, which is both what a stubbed test double looks like and what a
+ GPU-free runner would turn into a silent cu126 default.
+ """
+ path_exe = shutil.which("nvidia-smi")
+ return [
+ candidate
+ for candidate in _nvidia_smi_candidates()
+ if candidate == path_exe or os.path.isfile(candidate)
+ ]
+
+
+def _nvidia_smi_path() -> "str | None":
+ """The first nvidia-smi worth running, or None."""
+ candidates = _nvidia_smi_usable_candidates()
+ return candidates[0] if candidates else None
def _nvidia_compute_sms(exe: str) -> "list[int] | None":
@@ -3464,9 +3500,19 @@ def _detect_cuda_torch_index_url() -> str:
_override_family = os.environ.get("UNSLOTH_TORCH_INDEX_FAMILY", "").strip()
if _override_family:
return f"{_PYTORCH_WHL_BASE}/{_override_family.strip('/')}"
- exe = _nvidia_smi_path()
tag = "cu126" # default when the driver CUDA version cannot be read
- if exe:
+ # Every candidate until one ANSWERS, the way _has_usable_nvidia_gpu does. Taking the
+ # first that merely exists loses to a stale nvidia-smi on PATH: the presence probe
+ # walks past it to the working Program Files copy and confirms the GPU, while this one
+ # stopped at the stale binary, read no version, and defaulted to cu126. On Blackwell
+ # that wheel has no kernels, so a repair would install one the GPU cannot use.
+ for exe in _nvidia_smi_usable_candidates():
+ # The same predicate the presence probe uses. A stale copy that exits 0 with a
+ # parseable banner but lists no GPU must not decide the family: _has_usable_nvidia_gpu
+ # walks past it to the working copy, and setup.ps1 keeps the executable that passes
+ # Test-NvidiaSmiHasGpu, so accepting it here picks a family off the wrong driver.
+ if not _nvidia_smi_lists_a_gpu(exe):
+ continue
try:
result = subprocess.run(
[exe],
@@ -3477,28 +3523,40 @@ def _detect_cuda_torch_index_url() -> str:
errors = "replace",
timeout = 10,
)
- if result.returncode == 0:
- m = re.search(r"CUDA(?: UMD)? Version:\s*(\d+)\.(\d+)", result.stdout)
- if m:
- major, minor = int(m.group(1)), int(m.group(2))
- if major >= 13:
- tag = "cu130"
- elif major == 12 and minor >= 8:
- tag = "cu128"
- elif major == 12 and minor >= 6:
- tag = "cu126"
- elif major >= 12:
- tag = "cu124"
- elif major >= 11:
- tag = "cu118"
- else:
- tag = "cpu" # ancient driver: no usable CUDA wheels
except Exception:
- pass
- tag = _cap_cuda_family_for_pre_turing(tag, exe)
+ continue
+ if result.returncode != 0:
+ continue
+ m = re.search(r"CUDA(?: UMD)? Version:\s*(\d+)\.(\d+)", result.stdout)
+ if m is None:
+ continue
+ major, minor = int(m.group(1)), int(m.group(2))
+ if major >= 13:
+ tag = "cu130"
+ elif major == 12 and minor >= 8:
+ tag = "cu128"
+ elif major == 12 and minor >= 6:
+ tag = "cu126"
+ elif major >= 12:
+ tag = "cu124"
+ elif major >= 11:
+ tag = "cu118"
+ else:
+ tag = "cpu" # ancient driver: no usable CUDA wheels
+ return f"{_PYTORCH_WHL_BASE}/{_cap_cuda_family_for_pre_turing(tag, exe)}"
return f"{_PYTORCH_WHL_BASE}/{tag}"
+def _driver_cuda_torch_flavor_tag() -> str:
+ """The CUDA wheel family this driver can actually run, or "" if it can run none.
+
+ _detect_cuda_torch_index_url mirrors setup.ps1::Get-PytorchCudaTag, ancient-driver "cpu"
+ and pre-Turing cap included, so reading its leaf asks the same question the handover did.
+ """
+ leaf = _detect_cuda_torch_index_url().rstrip("/").rsplit("/", 1)[-1].strip().lower()
+ return leaf if _is_cuda_family_leaf(leaf) else ""
+
+
def _explicit_torch_index_url() -> "str | None":
"""The wheel index URL pinned via UNSLOTH_TORCH_INDEX_URL / _FAMILY, else None.
@@ -3986,7 +4044,7 @@ def _ensure_xpu_triton() -> None:
"""
if NO_TORCH or IS_MACOS:
return
- if IS_WINDOWS and os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip():
+ if IS_WINDOWS and _handover_torch_flavor_tag():
return
pin = _explicit_xpu_torch_index_url()
if pin is None:
@@ -4296,6 +4354,15 @@ def _torch_build_is_gpu() -> bool:
return (not label) or _is_gpu_torch_label(label)
+def _handover_torch_flavor_tag() -> str:
+ """The flavor setup.sh / setup.ps1 published for this run, lowercased, or "".
+
+ Named rather than read inline: the invariant has to tell a handover apart from the other
+ sources of the same string.
+ """
+ return os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip().lower()
+
+
def _expected_torch_flavor_tag() -> str:
"""The torch flavor this venv is SUPPOSED to hold, or "" when nothing can say.
@@ -4317,8 +4384,36 @@ def _expected_torch_flavor_tag() -> str:
Only an NVIDIA host, or an explicit pin, can expect a GPU build -- otherwise
return "" rather than invent an expectation from an absent GPU.
"""
- env = os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip().lower()
+ env = _handover_torch_flavor_tag()
if env:
+ # One exception. The setup scripts publish "cpu" both when the user asked for it and
+ # when their GPU probe came back empty, and honouring the second reading is what let an
+ # update report a venv rebuilt as 2.11.0+cpu as a success. Unpinned, "cpu" is a probe
+ # ANSWER and does not outrank a CUDA manifest while the GPU is still there; both facts
+ # must hold, or a host that really lost its GPU gets a wheel it cannot load. Resolved
+ # here rather than in _ensure_expected_torch_flavor so the caller records the enforced
+ # tag too, instead of repairing the venv and then writing "cpu" to the manifest.
+ if env == "cpu" and not _explicit_cpu_torch_index_pin():
+ recorded = (_RECORDED_TORCH_TAG or "").strip().lower()
+ if _is_cuda_family_leaf(recorded) and _has_usable_nvidia_gpu():
+ # The driver's OWN family, not the recorded one. setup.ps1 also answers "cpu"
+ # legitimately, from Get-PytorchCudaTag, when the driver tops out below CUDA 11
+ # or Get-CudaFamilyCappedForPreTuring lowers the family; reinstating the record
+ # there would install a wheel this driver cannot load. Same probe as setup.ps1,
+ # mirrored in _detect_cuda_torch_index_url, so the two cannot disagree.
+ # An explicit CUDA pin outranks the probe: the repair helpers install from
+ # the pinned URL, so expecting anything else would flag the venv they just
+ # built correctly. _explicit_cpu_torch_index_pin already took the CPU pin.
+ pinned = _torch_index_leaf(_explicit_torch_index_url() or "")
+ driver = pinned if _is_cuda_family_leaf(pinned) else _driver_cuda_torch_flavor_tag()
+ if not driver:
+ return env
+ _safe_print(
+ f" [WARN] the installer handed over a CPU torch expectation, but this venv "
+ f"was recorded as {recorded} and an NVIDIA GPU is still present; "
+ f"enforcing {driver}."
+ )
+ return driver
return env
pin = _explicit_torch_index_url()
if pin is not None:
@@ -4366,7 +4461,7 @@ def _expected_torch_flavor_is_explicit() -> bool:
False when only the live hardware probe can answer, which is the one case a
visibility mask has any business overruling.
"""
- if os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip():
+ if _handover_torch_flavor_tag():
return True
if _explicit_torch_index_url() is not None:
return True
diff --git a/tests/studio/install/test_cuda_repair.py b/tests/studio/install/test_cuda_repair.py
index aa868f2f9e6..05c708aab61 100644
--- a/tests/studio/install/test_cuda_repair.py
+++ b/tests/studio/install/test_cuda_repair.py
@@ -86,6 +86,14 @@ def _make_run(
result.returncode = smi_rc
if len(cmd) > 1 and str(cmd[1]) == "--query-gpu=compute_cap":
out = "".join(f"{cap}\n" for cap in compute_caps)
+ elif len(cmd) > 1 and str(cmd[1]) == "-L":
+ # The GPU-presence predicate both probes now share. A driver that reports a
+ # CUDA version has a GPU to report it for, so this tracks cuda_version.
+ out = (
+ "GPU 0: NVIDIA GeForce RTX 4090 (UUID: GPU-x)\n"
+ if cuda_version
+ else "No devices were found\n"
+ )
else:
out = f"CUDA Version: {cuda_version}\n" if cuda_version else "No devices found\n"
result.stdout = out if kwargs.get("text") else out.encode()
@@ -768,6 +776,14 @@ def _run_flavor_invariant(
result.returncode = 0
if len(cmd) > 1 and str(cmd[1]) == "--query-gpu=compute_cap":
out = "8.6\n"
+ elif len(cmd) > 1 and str(cmd[1]) == "-L":
+ # The shared GPU-presence predicate; a driver reporting a CUDA version
+ # has a GPU to report it for.
+ out = (
+ "GPU 0: NVIDIA GeForce RTX 4090 (UUID: GPU-x)\n"
+ if cuda_version
+ else "No devices were found\n"
+ )
else:
out = f"CUDA Version: {cuda_version}\n" if cuda_version else "No devices\n"
result.stdout = out if kwargs.get("text") else out.encode()
@@ -2047,7 +2063,7 @@ class TestTheWindowsXpuTritonSwapReachesADirectRun:
source = inspect.getsource(stack_mod._ensure_xpu_triton)
assert "if NO_TORCH or IS_MACOS:" in source, "Windows must no longer be excluded outright"
assert (
- 'IS_WINDOWS and os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG"' in source
+ "IS_WINDOWS and _handover_torch_flavor_tag()" in source
), "under setup.ps1 the swap is still that script's job"
def test_the_windows_branch_runs_it_after_the_invariant(self):
@@ -2194,11 +2210,303 @@ class TestARepairedTorchThatCannotImport:
assert any("cannot be imported" in ln for ln in lines), lines
+class TestACpuHandoverDoesNotDisarmTheInvariant:
+ """setup.ps1 publishes UNSLOTH_EXPECTED_TORCH_TAG=cpu on an empty nvidia-smi probe, and
+ the invariant used to pass on that tag without ever reading the installed wheel: a venv
+ rebuilt as 2.11.0+cpu was reported as a successful update. The handover is the probe's
+ answer, not a stated choice, so it loses to the manifest but still beats a real pin and
+ is still honoured on a host whose GPU has genuinely gone."""
+
+ def test_a_cpu_handover_is_overruled_by_a_recorded_cuda_flavor(self):
+ # cuda_version has to agree with the record: the enforced family comes from the
+ # DRIVER, and a 12.4 driver under a cu128 record is the downgrade case below.
+ ok, mock_pip = _run_flavor_invariant(
+ expected_env = "cpu",
+ recorded = "cu128",
+ nvidia = True,
+ cuda_version = "12.8",
+ repaired = "2.11.0+cu128",
+ )
+ assert ok is True
+ assert mock_pip.call_count == 1
+ assert _index_url(mock_pip).endswith("/cu128")
+
+ def test_a_downgraded_driver_is_repaired_onto_its_own_family_end_to_end(self):
+ # Record says cu128, driver reports CUDA 12.4: install cu124, not the record.
+ ok, mock_pip = _run_flavor_invariant(
+ expected_env = "cpu",
+ recorded = "cu128",
+ nvidia = True,
+ cuda_version = "12.4",
+ repaired = "2.11.0+cu124",
+ )
+ assert ok is True
+ assert _index_url(mock_pip).endswith("/cu124")
+
+ def test_the_unrepaired_case_fails_the_install_rather_than_reporting_success(self):
+ # repaired = None: the reinstall left the venv on +cpu, the state this invariant exists
+ # to catch.
+ ok, _mock_pip = _run_flavor_invariant(
+ expected_env = "cpu",
+ recorded = "cu128",
+ nvidia = True,
+ repaired = None,
+ )
+ assert ok is False
+
+ def test_a_host_whose_gpu_really_went_away_is_left_alone(self):
+ # Repairing here would install a CUDA wheel onto a machine with no NVIDIA GPU.
+ ok, mock_pip = _run_flavor_invariant(
+ expected_env = "cpu",
+ recorded = "cu128",
+ nvidia = False,
+ )
+ assert ok is True
+ assert mock_pip.call_count == 0
+
+ def test_a_cpu_handover_with_nothing_recorded_still_skips(self):
+ ok, mock_pip = _run_flavor_invariant(
+ expected_env = "cpu",
+ recorded = None,
+ nvidia = True,
+ )
+ assert ok is True
+ assert mock_pip.call_count == 0
+
+ def test_an_explicit_cpu_index_pin_still_wins(self):
+ # A stated choice, not a probe result: a cu128 record must not override the CPU index.
+ ok, mock_pip = _run_flavor_invariant(
+ expected_env = "cpu",
+ index_url = f"{stack_mod._PYTORCH_WHL_BASE}/cpu",
+ recorded = "cu128",
+ nvidia = True,
+ installed = "2.11.0+cpu",
+ )
+ assert ok is True
+ assert mock_pip.call_count == 0
+
+ def test_the_enforced_tag_reaches_the_manifest(self, monkeypatch):
+ """One resolved tag feeds both the invariant and _recordable_torch_flavor_tag.
+
+ Overriding only the invariant's copy repaired the venv and then wrote "cpu" to the
+ manifest, leaving the next update no CUDA record to fire on: a one-shot fix.
+ """
+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)
+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu128")
+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack_mod, "_driver_cuda_torch_flavor_tag", lambda: "cu128")
+
+ resolved = stack_mod._expected_torch_flavor_tag()
+ assert resolved == "cu128"
+ assert stack_mod._recordable_torch_flavor_tag(resolved) == "cu128"
+
+ def test_the_driver_probe_finds_a_windows_nvidia_smi_off_path(self, monkeypatch, tmp_path):
+ """Both probes have to look in the same places.
+
+ _has_usable_nvidia_gpu already probed the Windows fixed locations, so a host with
+ nvidia-smi.exe off PATH answered "GPU present" while the family probe read PATH
+ only, fell back to its cu126 default, and recorded a family the driver never
+ reported. On Blackwell that wheel has no supported kernels.
+ """
+ program_files = tmp_path / "Program Files"
+ smi = program_files / "NVIDIA Corporation" / "NVSMI" / "nvidia-smi.exe"
+ smi.parent.mkdir(parents = True)
+ smi.write_text("")
+ monkeypatch.setattr(stack_mod, "IS_WINDOWS", True)
+ monkeypatch.setenv("ProgramFiles", str(program_files))
+ monkeypatch.setenv("SystemRoot", str(tmp_path / "Windows"))
+ monkeypatch.setattr(stack_mod.shutil, "which", lambda _name: None)
+
+ assert str(smi) in stack_mod._nvidia_smi_candidates()
+ assert stack_mod._nvidia_smi_path() == str(smi)
+
+ def test_the_candidate_list_is_shared_by_both_probes(self):
+ # Two callers that disagree about where nvidia-smi lives disagree about the host.
+ assert "_nvidia_smi_candidates()" in inspect.getsource(stack_mod._has_usable_nvidia_gpu)
+ assert "_nvidia_smi_candidates()" in inspect.getsource(
+ stack_mod._nvidia_smi_usable_candidates
+ )
+ assert "_nvidia_smi_usable_candidates()" in inspect.getsource(
+ stack_mod._detect_cuda_torch_index_url
+ )
+
+ def test_a_which_result_is_trusted_without_an_isfile_check(self, monkeypatch):
+ """shutil.which already proved its result runnable.
+
+ Gating it on os.path.isfile rejects a bare "nvidia-smi" relative to a CWD it does
+ not live in. That is both what a stubbed test double looks like and, on a GPU-free
+ runner with nothing at /usr/bin/nvidia-smi, a silent fall back to the cu126 default.
+ """
+ monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack_mod.shutil, "which", lambda name, *a, **k: "nvidia-smi")
+ monkeypatch.setattr(stack_mod.os.path, "isfile", lambda p: "nvidia-smi" not in str(p))
+ assert stack_mod._nvidia_smi_path() == "nvidia-smi"
+
+ def test_the_family_probe_walks_past_a_stale_candidate(self, monkeypatch, tmp_path):
+ """A stale nvidia-smi on PATH must not decide the family.
+
+ _has_usable_nvidia_gpu already walks past it to the working copy and confirms the
+ GPU; stopping at the stale one here read no version and defaulted to cu126, which
+ on Blackwell has no kernels.
+ """
+ working = tmp_path / "nvidia-smi.exe"
+ working.write_text("")
+ monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_FAMILY", raising = False)
+ monkeypatch.setattr(stack_mod.shutil, "which", lambda name, *a, **k: "stale-smi")
+ monkeypatch.setattr(
+ stack_mod, "_nvidia_smi_candidates", lambda: ["stale-smi", str(working)]
+ )
+ monkeypatch.setattr(stack_mod, "_cap_cuda_family_for_pre_turing", lambda f, e: f)
+
+ def _run(command, *a, **k):
+ if command[0] == "stale-smi":
+ return SimpleNamespace(returncode = 9, stdout = "", stderr = "")
+ if len(command) > 1 and command[1] == "-L":
+ return SimpleNamespace(
+ returncode = 0, stdout = "GPU 0: NVIDIA RTX (UUID: GPU-x)", stderr = ""
+ )
+ return SimpleNamespace(
+ returncode = 0, stdout = "| NVIDIA-SMI 591.86 CUDA Version: 12.8 |", stderr = ""
+ )
+
+ monkeypatch.setattr(stack_mod.subprocess, "run", _run)
+ assert stack_mod._detect_cuda_torch_index_url().endswith("/cu128")
+
+ def test_a_banner_without_a_gpu_does_not_decide_the_family(self, monkeypatch, tmp_path):
+ """Exiting 0 with a parseable banner is not enough; -L has to list a GPU.
+
+ A stale copy can print "CUDA Version: 12.6" and enumerate nothing. The presence
+ probe rejects it and walks on to the working copy, and setup.ps1 keeps whichever
+ executable passes Test-NvidiaSmiHasGpu, so accepting it here reads the family off
+ the wrong driver.
+ """
+ working = tmp_path / "nvidia-smi.exe"
+ working.write_text("")
+ monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_FAMILY", raising = False)
+ monkeypatch.setattr(stack_mod.shutil, "which", lambda name, *a, **k: "stale-smi")
+ monkeypatch.setattr(
+ stack_mod, "_nvidia_smi_candidates", lambda: ["stale-smi", str(working)]
+ )
+ monkeypatch.setattr(stack_mod, "_cap_cuda_family_for_pre_turing", lambda f, e: f)
+
+ def _run(command, *a, **k):
+ listing = len(command) > 1 and command[1] == "-L"
+ if command[0] == "stale-smi":
+ # Exits 0 either way, but lists nothing.
+ return SimpleNamespace(
+ returncode = 0,
+ stdout = "" if listing else "| NVIDIA-SMI 550.1 CUDA Version: 12.6 |",
+ stderr = "",
+ )
+ return SimpleNamespace(
+ returncode = 0,
+ stdout = (
+ "GPU 0: NVIDIA RTX (UUID: GPU-x)"
+ if listing
+ else "| NVIDIA-SMI 591.86 CUDA Version: 13.1 |"
+ ),
+ stderr = "",
+ )
+
+ monkeypatch.setattr(stack_mod.subprocess, "run", _run)
+ assert stack_mod._detect_cuda_torch_index_url().endswith("/cu130")
+
+ def test_both_probes_share_the_gpu_presence_predicate(self):
+ # Two callers that disagree about what counts as a working nvidia-smi disagree
+ # about the host, which is how the family came off the wrong driver twice.
+ assert "_nvidia_smi_lists_a_gpu" in inspect.getsource(stack_mod._has_usable_nvidia_gpu)
+ assert "_nvidia_smi_lists_a_gpu" in inspect.getsource(
+ stack_mod._detect_cuda_torch_index_url
+ )
+
+ def test_an_explicit_cuda_pin_outranks_the_driver_probe(self, monkeypatch):
+ # The repair helpers install from the pinned URL, so expecting the driver's family
+ # instead would flag the venv they just built correctly.
+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")
+ monkeypatch.setenv("UNSLOTH_TORCH_INDEX_URL", "https://download.pytorch.org/whl/cu128")
+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu124")
+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack_mod, "_driver_cuda_torch_flavor_tag", lambda: "cu126")
+
+ assert stack_mod._expected_torch_flavor_tag() == "cu128"
+
+ def test_a_driver_too_old_for_any_cuda_wheel_keeps_the_cpu_handover(self, monkeypatch):
+ # Get-PytorchCudaTag returns "cpu" for a driver below CUDA 11, so the handover is a
+ # STATEMENT here, not an empty probe. Reinstating cu128 would install a wheel this
+ # driver cannot load. _detect_cuda_torch_index_url mirrors that and yields "" .
+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)
+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu128")
+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack_mod, "_driver_cuda_torch_flavor_tag", lambda: "")
+
+ resolved = stack_mod._expected_torch_flavor_tag()
+ assert resolved == "cpu"
+ assert stack_mod._recordable_torch_flavor_tag(resolved) == "cpu"
+
+ def test_a_downgraded_driver_gets_its_own_family_not_the_recorded_one(self, monkeypatch):
+ # A driver downgrade, or Get-CudaFamilyCappedForPreTuring lowering the family, leaves
+ # the manifest naming a family the host outgrew. The record proves this venv is meant
+ # to be CUDA; it does not get to pick which CUDA.
+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)
+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu128")
+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack_mod, "_driver_cuda_torch_flavor_tag", lambda: "cu118")
+
+ resolved = stack_mod._expected_torch_flavor_tag()
+ assert resolved == "cu118"
+ assert stack_mod._recordable_torch_flavor_tag(resolved) == "cu118"
+
+ def test_the_driver_family_probe_mirrors_the_index_url(self, monkeypatch):
+ # The helper reads _detect_cuda_torch_index_url's leaf, so an ancient-driver "cpu"
+ # URL has to come back as "" rather than as a family named "cpu".
+ monkeypatch.setattr(
+ stack_mod,
+ "_detect_cuda_torch_index_url",
+ lambda: "https://download.pytorch.org/whl/cpu",
+ )
+ assert stack_mod._driver_cuda_torch_flavor_tag() == ""
+ monkeypatch.setattr(
+ stack_mod,
+ "_detect_cuda_torch_index_url",
+ lambda: "https://download.pytorch.org/whl/cu126/",
+ )
+ assert stack_mod._driver_cuda_torch_flavor_tag() == "cu126"
+
+ def test_a_host_that_lost_its_gpu_records_cpu(self, monkeypatch):
+ # Without the GPU the handover stands and the manifest records cpu.
+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)
+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu128")
+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: False)
+
+ assert stack_mod._expected_torch_flavor_tag() == "cpu"
+
+ def test_both_handover_readers_go_through_one_helper(self):
+ # Three call sites read this tag; an inline os.environ read drifting out of step with
+ # the helper is how the skip came back the first time.
+ source = inspect.getsource(stack_mod)
+ assert source.count('os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG"') == 1
+ assert "def _handover_torch_flavor_tag(" in source
+
+
# What a localized nvidia-smi writes, which -X utf8 decodes as UTF-8 (#10173).
_LOCALIZED_NVIDIA_SMI = (
"import sys\n"
"if sys.argv[1:] == ['--query-gpu=compute_cap', '--format=csv,noheader,nounits']:\n"
" sys.stdout.buffer.write(b'8.6\\n')\n"
+ # -L is the GPU-presence predicate the family probe now shares with
+ # _has_usable_nvidia_gpu. Without an arm for it this double prints the banner, which
+ # lists no GPU, and the candidate is skipped: the test then passes only on a host that
+ # happens to have a REAL nvidia-smi to fall through to, and fails on a GPU-free runner.
+ "elif sys.argv[1:] == ['-L']:\n"
+ " sys.stdout.buffer.write(b'GPU 0: NVIDIA GeForce RTX 4090 (UUID: GPU-x)\\n')\n"
"else:\n"
" sys.stdout.buffer.write(b'| NVIDIA-SMI 591.86 CUDA Version: 13.1 |\\n')\n"
" sys.stdout.buffer.write('\\u4e02\\u4fdd\\u7559\\u6240\\u6709\\u6743\\u5229\\u3002\\n'.encode('gbk'))\n"
diff --git a/tests/studio/test_xpu_triton_swap.py b/tests/studio/test_xpu_triton_swap.py
index 30c48213b52..51af2a51d36 100644
--- a/tests/studio/test_xpu_triton_swap.py
+++ b/tests/studio/test_xpu_triton_swap.py
@@ -197,6 +197,12 @@ def _load(
# pip check only runs when something said it changed the environment.
"_count_install_action": lambda: counted.append(1),
"_red": lambda s: s,
+ # _ensure_xpu_triton reads the setup-script handover through this helper rather
+ # than inline, so the slice needs it by name or the guard tests NameError at call
+ # time. Same semantics as the real one: the env var, lowercased.
+ "_handover_torch_flavor_tag": (
+ lambda: os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip().lower()
+ ),
# _safe_print, not print: the slice calls it by name, so stubbing "print" would leave _safe_print undefined at
# exec time.
"_safe_print": (