Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop a CPU torch handover from disarming the flavor invariant (#10906)
Daniel Han24f06276f1d1
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex a3c535a33d4..247f8224b8f 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -2515,48 +2515,14 @@ def _has_usable_nvidia_gpu() -> bool:if cvd is not None and cvd.strip() in ("", "-1"):return False- def _lists_a_gpu(exe: str) -> bool:- try:- result = subprocess.run(- [exe, "-L"],- stdout = subprocess.PIPE,- stderr = subprocess.DEVNULL,- text = True,- encoding = "utf-8",- errors = "replace",- timeout = 10,- )- except Exception:- return False- return result.returncode == 0 and "GPU " in result.stdout-# A stale nvidia-smi on PATH exits non-zero listing nothing, so try every# candidate: install.ps1 / setup.ps1 also gate the fixed-location fallback# on the GPU check failing, not on the PATH lookup missing.- candidates = []_path_exe = shutil.which("nvidia-smi")- if _path_exe:- candidates.append(_path_exe)- if IS_WINDOWS:- candidates.extend(- (- os.path.join(- os.environ.get("ProgramFiles", r"C:\Program Files"),- "NVIDIA Corporation",- "NVSMI",- "nvidia-smi.exe",- ),- os.path.join(- os.environ.get("SystemRoot", r"C:\Windows"),- "System32",- "nvidia-smi.exe",- ),- )- )- for _candidate in candidates:+ for _candidate in _nvidia_smi_candidates():if _candidate != _path_exe and not os.path.isfile(_candidate):continue- if _lists_a_gpu(_candidate):+ if _nvidia_smi_lists_a_gpu(_candidate):return True# Fallback: /proc/driver/nvidia/gpus/ has one subdir per GPU whatever nvidia-smi does.if sys.platform != "win32":@@ -3345,13 +3311,83 @@ def _install_bnb_windows_rocm() -> bool:return True-def _nvidia_smi_path() -> "str | None":- """nvidia-smi from PATH, falling back to the canonical Linux install path a- stripped-down PATH (systemd units, cron) can miss."""+def _nvidia_smi_candidates() -> "list[str]":+ """Every place nvidia-smi is worth looking for, PATH first.++ One list, because two callers that disagree about where nvidia-smi lives disagree+ about the host: _has_usable_nvidia_gpu would find the driver through the Windows+ fixed locations while _detect_cuda_torch_index_url, reading PATH only, fell back to+ its cu126 default and recorded a family the driver never reported.+ """+ candidates = []exe = shutil.which("nvidia-smi")- if not exe and os.path.isfile("/usr/bin/nvidia-smi"):- exe = "/usr/bin/nvidia-smi"- return exe+ if exe:+ candidates.append(exe)+ if IS_WINDOWS:+ # The locations install.ps1 / setup.ps1 use; nvidia-smi.exe is routinely off PATH.+ candidates.extend(+ (+ os.path.join(+ os.environ.get("ProgramFiles", r"C:\Program Files"),+ "NVIDIA Corporation",+ "NVSMI",+ "nvidia-smi.exe",+ ),+ os.path.join(+ os.environ.get("SystemRoot", r"C:\Windows"),+ "System32",+ "nvidia-smi.exe",+ ),+ )+ )+ else:+ # The canonical Linux path a stripped-down PATH (systemd units, cron) can miss.+ candidates.append("/usr/bin/nvidia-smi")+ return candidates+++def _nvidia_smi_lists_a_gpu(exe: str) -> bool:+ """Whether this nvidia-smi actually enumerates a GPU.++ The predicate both probes have to agree on. A stale copy can exit 0 and print a+ perfectly parseable "CUDA Version:" banner while `-L` lists nothing, which is the+ state setup.ps1's Test-NvidiaSmiHasGpu rejects.+ """+ try:+ result = subprocess.run(+ [exe, "-L"],+ stdout = subprocess.PIPE,+ stderr = subprocess.DEVNULL,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 10,+ )+ except Exception:+ return False+ return result.returncode == 0 and "GPU " in result.stdout+++def _nvidia_smi_usable_candidates() -> "list[str]":+ """Candidates worth actually running: the PATH result plus every file that exists.++ `shutil.which` already proved its result runnable, so it is kept without an+ os.path.isfile check -- that would reject a bare "nvidia-smi" relative to a CWD it+ does not live in, which is both what a stubbed test double looks like and what a+ GPU-free runner would turn into a silent cu126 default.+ """+ path_exe = shutil.which("nvidia-smi")+ return [+ candidate+ for candidate in _nvidia_smi_candidates()+ if candidate == path_exe or os.path.isfile(candidate)+ ]+++def _nvidia_smi_path() -> "str | None":+ """The first nvidia-smi worth running, or None."""+ candidates = _nvidia_smi_usable_candidates()+ return candidates[0] if candidates else Nonedef _nvidia_compute_sms(exe: str) -> "list[int] | None":@@ -3464,9 +3500,19 @@ def _detect_cuda_torch_index_url() -> str:_override_family = os.environ.get("UNSLOTH_TORCH_INDEX_FAMILY", "").strip()if _override_family:return f"{_PYTORCH_WHL_BASE}/{_override_family.strip('/')}"- exe = _nvidia_smi_path()tag = "cu126" # default when the driver CUDA version cannot be read- if exe:+ # Every candidate until one ANSWERS, the way _has_usable_nvidia_gpu does. Taking the+ # first that merely exists loses to a stale nvidia-smi on PATH: the presence probe+ # walks past it to the working Program Files copy and confirms the GPU, while this one+ # stopped at the stale binary, read no version, and defaulted to cu126. On Blackwell+ # that wheel has no kernels, so a repair would install one the GPU cannot use.+ for exe in _nvidia_smi_usable_candidates():+ # The same predicate the presence probe uses. A stale copy that exits 0 with a+ # parseable banner but lists no GPU must not decide the family: _has_usable_nvidia_gpu+ # walks past it to the working copy, and setup.ps1 keeps the executable that passes+ # Test-NvidiaSmiHasGpu, so accepting it here picks a family off the wrong driver.+ if not _nvidia_smi_lists_a_gpu(exe):+ continuetry:result = subprocess.run([exe],@@ -3477,28 +3523,40 @@ def _detect_cuda_torch_index_url() -> str:errors = "replace",timeout = 10,)- if result.returncode == 0:- m = re.search(r"CUDA(?: UMD)? Version:\s*(\d+)\.(\d+)", result.stdout)- if m:- major, minor = int(m.group(1)), int(m.group(2))- if major >= 13:- tag = "cu130"- elif major == 12 and minor >= 8:- tag = "cu128"- elif major == 12 and minor >= 6:- tag = "cu126"- elif major >= 12:- tag = "cu124"- elif major >= 11:- tag = "cu118"- else:- tag = "cpu" # ancient driver: no usable CUDA wheelsexcept Exception:- pass- tag = _cap_cuda_family_for_pre_turing(tag, exe)+ continue+ if result.returncode != 0:+ continue+ m = re.search(r"CUDA(?: UMD)? Version:\s*(\d+)\.(\d+)", result.stdout)+ if m is None:+ continue+ major, minor = int(m.group(1)), int(m.group(2))+ if major >= 13:+ tag = "cu130"+ elif major == 12 and minor >= 8:+ tag = "cu128"+ elif major == 12 and minor >= 6:+ tag = "cu126"+ elif major >= 12:+ tag = "cu124"+ elif major >= 11:+ tag = "cu118"+ else:+ tag = "cpu" # ancient driver: no usable CUDA wheels+ return f"{_PYTORCH_WHL_BASE}/{_cap_cuda_family_for_pre_turing(tag, exe)}"return f"{_PYTORCH_WHL_BASE}/{tag}"+def _driver_cuda_torch_flavor_tag() -> str:+ """The CUDA wheel family this driver can actually run, or "" if it can run none.++ _detect_cuda_torch_index_url mirrors setup.ps1::Get-PytorchCudaTag, ancient-driver "cpu"+ and pre-Turing cap included, so reading its leaf asks the same question the handover did.+ """+ leaf = _detect_cuda_torch_index_url().rstrip("/").rsplit("/", 1)[-1].strip().lower()+ return leaf if _is_cuda_family_leaf(leaf) else ""++def _explicit_torch_index_url() -> "str | None":"""The wheel index URL pinned via UNSLOTH_TORCH_INDEX_URL / _FAMILY, else None.@@ -3986,7 +4044,7 @@ def _ensure_xpu_triton() -> None:"""if NO_TORCH or IS_MACOS:return- if IS_WINDOWS and os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip():+ if IS_WINDOWS and _handover_torch_flavor_tag():returnpin = _explicit_xpu_torch_index_url()if pin is None:@@ -4296,6 +4354,15 @@ def _torch_build_is_gpu() -> bool:return (not label) or _is_gpu_torch_label(label)+def _handover_torch_flavor_tag() -> str:+ """The flavor setup.sh / setup.ps1 published for this run, lowercased, or "".++ Named rather than read inline: the invariant has to tell a handover apart from the other+ sources of the same string.+ """+ return os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip().lower()++def _expected_torch_flavor_tag() -> str:"""The torch flavor this venv is SUPPOSED to hold, or "" when nothing can say.@@ -4317,8 +4384,36 @@ def _expected_torch_flavor_tag() -> str:Only an NVIDIA host, or an explicit pin, can expect a GPU build -- otherwisereturn "" rather than invent an expectation from an absent GPU."""- env = os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip().lower()+ env = _handover_torch_flavor_tag()if env:+ # One exception. The setup scripts publish "cpu" both when the user asked for it and+ # when their GPU probe came back empty, and honouring the second reading is what let an+ # update report a venv rebuilt as 2.11.0+cpu as a success. Unpinned, "cpu" is a probe+ # ANSWER and does not outrank a CUDA manifest while the GPU is still there; both facts+ # must hold, or a host that really lost its GPU gets a wheel it cannot load. Resolved+ # here rather than in _ensure_expected_torch_flavor so the caller records the enforced+ # tag too, instead of repairing the venv and then writing "cpu" to the manifest.+ if env == "cpu" and not _explicit_cpu_torch_index_pin():+ recorded = (_RECORDED_TORCH_TAG or "").strip().lower()+ if _is_cuda_family_leaf(recorded) and _has_usable_nvidia_gpu():+ # The driver's OWN family, not the recorded one. setup.ps1 also answers "cpu"+ # legitimately, from Get-PytorchCudaTag, when the driver tops out below CUDA 11+ # or Get-CudaFamilyCappedForPreTuring lowers the family; reinstating the record+ # there would install a wheel this driver cannot load. Same probe as setup.ps1,+ # mirrored in _detect_cuda_torch_index_url, so the two cannot disagree.+ # An explicit CUDA pin outranks the probe: the repair helpers install from+ # the pinned URL, so expecting anything else would flag the venv they just+ # built correctly. _explicit_cpu_torch_index_pin already took the CPU pin.+ pinned = _torch_index_leaf(_explicit_torch_index_url() or "")+ driver = pinned if _is_cuda_family_leaf(pinned) else _driver_cuda_torch_flavor_tag()+ if not driver:+ return env+ _safe_print(+ f" [WARN] the installer handed over a CPU torch expectation, but this venv "+ f"was recorded as {recorded} and an NVIDIA GPU is still present; "+ f"enforcing {driver}."+ )+ return driverreturn envpin = _explicit_torch_index_url()if pin is not None:@@ -4366,7 +4461,7 @@ def _expected_torch_flavor_is_explicit() -> bool:False when only the live hardware probe can answer, which is the one case avisibility mask has any business overruling."""- if os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip():+ if _handover_torch_flavor_tag():return Trueif _explicit_torch_index_url() is not None:return Truediff --git a/tests/studio/install/test_cuda_repair.py b/tests/studio/install/test_cuda_repair.pyindex aa868f2f9e6..05c708aab61 100644--- a/tests/studio/install/test_cuda_repair.py+++ b/tests/studio/install/test_cuda_repair.py@@ -86,6 +86,14 @@ def _make_run(result.returncode = smi_rcif len(cmd) > 1 and str(cmd[1]) == "--query-gpu=compute_cap":out = "".join(f"{cap}\n" for cap in compute_caps)+ elif len(cmd) > 1 and str(cmd[1]) == "-L":+ # The GPU-presence predicate both probes now share. A driver that reports a+ # CUDA version has a GPU to report it for, so this tracks cuda_version.+ out = (+ "GPU 0: NVIDIA GeForce RTX 4090 (UUID: GPU-x)\n"+ if cuda_version+ else "No devices were found\n"+ )else:out = f"CUDA Version: {cuda_version}\n" if cuda_version else "No devices found\n"result.stdout = out if kwargs.get("text") else out.encode()@@ -768,6 +776,14 @@ def _run_flavor_invariant(result.returncode = 0if len(cmd) > 1 and str(cmd[1]) == "--query-gpu=compute_cap":out = "8.6\n"+ elif len(cmd) > 1 and str(cmd[1]) == "-L":+ # The shared GPU-presence predicate; a driver reporting a CUDA version+ # has a GPU to report it for.+ out = (+ "GPU 0: NVIDIA GeForce RTX 4090 (UUID: GPU-x)\n"+ if cuda_version+ else "No devices were found\n"+ )else:out = f"CUDA Version: {cuda_version}\n" if cuda_version else "No devices\n"result.stdout = out if kwargs.get("text") else out.encode()@@ -2047,7 +2063,7 @@ class TestTheWindowsXpuTritonSwapReachesADirectRun:source = inspect.getsource(stack_mod._ensure_xpu_triton)assert "if NO_TORCH or IS_MACOS:" in source, "Windows must no longer be excluded outright"assert (- 'IS_WINDOWS and os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG"' in source+ "IS_WINDOWS and _handover_torch_flavor_tag()" in source), "under setup.ps1 the swap is still that script's job"def test_the_windows_branch_runs_it_after_the_invariant(self):@@ -2194,11 +2210,303 @@ class TestARepairedTorchThatCannotImport:assert any("cannot be imported" in ln for ln in lines), lines+class TestACpuHandoverDoesNotDisarmTheInvariant:+ """setup.ps1 publishes UNSLOTH_EXPECTED_TORCH_TAG=cpu on an empty nvidia-smi probe, and+ the invariant used to pass on that tag without ever reading the installed wheel: a venv+ rebuilt as 2.11.0+cpu was reported as a successful update. The handover is the probe's+ answer, not a stated choice, so it loses to the manifest but still beats a real pin and+ is still honoured on a host whose GPU has genuinely gone."""++ def test_a_cpu_handover_is_overruled_by_a_recorded_cuda_flavor(self):+ # cuda_version has to agree with the record: the enforced family comes from the+ # DRIVER, and a 12.4 driver under a cu128 record is the downgrade case below.+ ok, mock_pip = _run_flavor_invariant(+ expected_env = "cpu",+ recorded = "cu128",+ nvidia = True,+ cuda_version = "12.8",+ repaired = "2.11.0+cu128",+ )+ assert ok is True+ assert mock_pip.call_count == 1+ assert _index_url(mock_pip).endswith("/cu128")++ def test_a_downgraded_driver_is_repaired_onto_its_own_family_end_to_end(self):+ # Record says cu128, driver reports CUDA 12.4: install cu124, not the record.+ ok, mock_pip = _run_flavor_invariant(+ expected_env = "cpu",+ recorded = "cu128",+ nvidia = True,+ cuda_version = "12.4",+ repaired = "2.11.0+cu124",+ )+ assert ok is True+ assert _index_url(mock_pip).endswith("/cu124")++ def test_the_unrepaired_case_fails_the_install_rather_than_reporting_success(self):+ # repaired = None: the reinstall left the venv on +cpu, the state this invariant exists+ # to catch.+ ok, _mock_pip = _run_flavor_invariant(+ expected_env = "cpu",+ recorded = "cu128",+ nvidia = True,+ repaired = None,+ )+ assert ok is False++ def test_a_host_whose_gpu_really_went_away_is_left_alone(self):+ # Repairing here would install a CUDA wheel onto a machine with no NVIDIA GPU.+ ok, mock_pip = _run_flavor_invariant(+ expected_env = "cpu",+ recorded = "cu128",+ nvidia = False,+ )+ assert ok is True+ assert mock_pip.call_count == 0++ def test_a_cpu_handover_with_nothing_recorded_still_skips(self):+ ok, mock_pip = _run_flavor_invariant(+ expected_env = "cpu",+ recorded = None,+ nvidia = True,+ )+ assert ok is True+ assert mock_pip.call_count == 0++ def test_an_explicit_cpu_index_pin_still_wins(self):+ # A stated choice, not a probe result: a cu128 record must not override the CPU index.+ ok, mock_pip = _run_flavor_invariant(+ expected_env = "cpu",+ index_url = f"{stack_mod._PYTORCH_WHL_BASE}/cpu",+ recorded = "cu128",+ nvidia = True,+ installed = "2.11.0+cpu",+ )+ assert ok is True+ assert mock_pip.call_count == 0++ def test_the_enforced_tag_reaches_the_manifest(self, monkeypatch):+ """One resolved tag feeds both the invariant and _recordable_torch_flavor_tag.++ Overriding only the invariant's copy repaired the venv and then wrote "cpu" to the+ manifest, leaving the next update no CUDA record to fire on: a one-shot fix.+ """+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu128")+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack_mod, "_driver_cuda_torch_flavor_tag", lambda: "cu128")++ resolved = stack_mod._expected_torch_flavor_tag()+ assert resolved == "cu128"+ assert stack_mod._recordable_torch_flavor_tag(resolved) == "cu128"++ def test_the_driver_probe_finds_a_windows_nvidia_smi_off_path(self, monkeypatch, tmp_path):+ """Both probes have to look in the same places.++ _has_usable_nvidia_gpu already probed the Windows fixed locations, so a host with+ nvidia-smi.exe off PATH answered "GPU present" while the family probe read PATH+ only, fell back to its cu126 default, and recorded a family the driver never+ reported. On Blackwell that wheel has no supported kernels.+ """+ program_files = tmp_path / "Program Files"+ smi = program_files / "NVIDIA Corporation" / "NVSMI" / "nvidia-smi.exe"+ smi.parent.mkdir(parents = True)+ smi.write_text("")+ monkeypatch.setattr(stack_mod, "IS_WINDOWS", True)+ monkeypatch.setenv("ProgramFiles", str(program_files))+ monkeypatch.setenv("SystemRoot", str(tmp_path / "Windows"))+ monkeypatch.setattr(stack_mod.shutil, "which", lambda _name: None)++ assert str(smi) in stack_mod._nvidia_smi_candidates()+ assert stack_mod._nvidia_smi_path() == str(smi)++ def test_the_candidate_list_is_shared_by_both_probes(self):+ # Two callers that disagree about where nvidia-smi lives disagree about the host.+ assert "_nvidia_smi_candidates()" in inspect.getsource(stack_mod._has_usable_nvidia_gpu)+ assert "_nvidia_smi_candidates()" in inspect.getsource(+ stack_mod._nvidia_smi_usable_candidates+ )+ assert "_nvidia_smi_usable_candidates()" in inspect.getsource(+ stack_mod._detect_cuda_torch_index_url+ )++ def test_a_which_result_is_trusted_without_an_isfile_check(self, monkeypatch):+ """shutil.which already proved its result runnable.++ Gating it on os.path.isfile rejects a bare "nvidia-smi" relative to a CWD it does+ not live in. That is both what a stubbed test double looks like and, on a GPU-free+ runner with nothing at /usr/bin/nvidia-smi, a silent fall back to the cu126 default.+ """+ monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)+ monkeypatch.setattr(stack_mod.shutil, "which", lambda name, *a, **k: "nvidia-smi")+ monkeypatch.setattr(stack_mod.os.path, "isfile", lambda p: "nvidia-smi" not in str(p))+ assert stack_mod._nvidia_smi_path() == "nvidia-smi"++ def test_the_family_probe_walks_past_a_stale_candidate(self, monkeypatch, tmp_path):+ """A stale nvidia-smi on PATH must not decide the family.++ _has_usable_nvidia_gpu already walks past it to the working copy and confirms the+ GPU; stopping at the stale one here read no version and defaulted to cu126, which+ on Blackwell has no kernels.+ """+ working = tmp_path / "nvidia-smi.exe"+ working.write_text("")+ monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_FAMILY", raising = False)+ monkeypatch.setattr(stack_mod.shutil, "which", lambda name, *a, **k: "stale-smi")+ monkeypatch.setattr(+ stack_mod, "_nvidia_smi_candidates", lambda: ["stale-smi", str(working)]+ )+ monkeypatch.setattr(stack_mod, "_cap_cuda_family_for_pre_turing", lambda f, e: f)++ def _run(command, *a, **k):+ if command[0] == "stale-smi":+ return SimpleNamespace(returncode = 9, stdout = "", stderr = "")+ if len(command) > 1 and command[1] == "-L":+ return SimpleNamespace(+ returncode = 0, stdout = "GPU 0: NVIDIA RTX (UUID: GPU-x)", stderr = ""+ )+ return SimpleNamespace(+ returncode = 0, stdout = "| NVIDIA-SMI 591.86 CUDA Version: 12.8 |", stderr = ""+ )++ monkeypatch.setattr(stack_mod.subprocess, "run", _run)+ assert stack_mod._detect_cuda_torch_index_url().endswith("/cu128")++ def test_a_banner_without_a_gpu_does_not_decide_the_family(self, monkeypatch, tmp_path):+ """Exiting 0 with a parseable banner is not enough; -L has to list a GPU.++ A stale copy can print "CUDA Version: 12.6" and enumerate nothing. The presence+ probe rejects it and walks on to the working copy, and setup.ps1 keeps whichever+ executable passes Test-NvidiaSmiHasGpu, so accepting it here reads the family off+ the wrong driver.+ """+ working = tmp_path / "nvidia-smi.exe"+ working.write_text("")+ monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_FAMILY", raising = False)+ monkeypatch.setattr(stack_mod.shutil, "which", lambda name, *a, **k: "stale-smi")+ monkeypatch.setattr(+ stack_mod, "_nvidia_smi_candidates", lambda: ["stale-smi", str(working)]+ )+ monkeypatch.setattr(stack_mod, "_cap_cuda_family_for_pre_turing", lambda f, e: f)++ def _run(command, *a, **k):+ listing = len(command) > 1 and command[1] == "-L"+ if command[0] == "stale-smi":+ # Exits 0 either way, but lists nothing.+ return SimpleNamespace(+ returncode = 0,+ stdout = "" if listing else "| NVIDIA-SMI 550.1 CUDA Version: 12.6 |",+ stderr = "",+ )+ return SimpleNamespace(+ returncode = 0,+ stdout = (+ "GPU 0: NVIDIA RTX (UUID: GPU-x)"+ if listing+ else "| NVIDIA-SMI 591.86 CUDA Version: 13.1 |"+ ),+ stderr = "",+ )++ monkeypatch.setattr(stack_mod.subprocess, "run", _run)+ assert stack_mod._detect_cuda_torch_index_url().endswith("/cu130")++ def test_both_probes_share_the_gpu_presence_predicate(self):+ # Two callers that disagree about what counts as a working nvidia-smi disagree+ # about the host, which is how the family came off the wrong driver twice.+ assert "_nvidia_smi_lists_a_gpu" in inspect.getsource(stack_mod._has_usable_nvidia_gpu)+ assert "_nvidia_smi_lists_a_gpu" in inspect.getsource(+ stack_mod._detect_cuda_torch_index_url+ )++ def test_an_explicit_cuda_pin_outranks_the_driver_probe(self, monkeypatch):+ # The repair helpers install from the pinned URL, so expecting the driver's family+ # instead would flag the venv they just built correctly.+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")+ monkeypatch.setenv("UNSLOTH_TORCH_INDEX_URL", "https://download.pytorch.org/whl/cu128")+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu124")+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack_mod, "_driver_cuda_torch_flavor_tag", lambda: "cu126")++ assert stack_mod._expected_torch_flavor_tag() == "cu128"++ def test_a_driver_too_old_for_any_cuda_wheel_keeps_the_cpu_handover(self, monkeypatch):+ # Get-PytorchCudaTag returns "cpu" for a driver below CUDA 11, so the handover is a+ # STATEMENT here, not an empty probe. Reinstating cu128 would install a wheel this+ # driver cannot load. _detect_cuda_torch_index_url mirrors that and yields "" .+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu128")+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack_mod, "_driver_cuda_torch_flavor_tag", lambda: "")++ resolved = stack_mod._expected_torch_flavor_tag()+ assert resolved == "cpu"+ assert stack_mod._recordable_torch_flavor_tag(resolved) == "cpu"++ def test_a_downgraded_driver_gets_its_own_family_not_the_recorded_one(self, monkeypatch):+ # A driver downgrade, or Get-CudaFamilyCappedForPreTuring lowering the family, leaves+ # the manifest naming a family the host outgrew. The record proves this venv is meant+ # to be CUDA; it does not get to pick which CUDA.+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu128")+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack_mod, "_driver_cuda_torch_flavor_tag", lambda: "cu118")++ resolved = stack_mod._expected_torch_flavor_tag()+ assert resolved == "cu118"+ assert stack_mod._recordable_torch_flavor_tag(resolved) == "cu118"++ def test_the_driver_family_probe_mirrors_the_index_url(self, monkeypatch):+ # The helper reads _detect_cuda_torch_index_url's leaf, so an ancient-driver "cpu"+ # URL has to come back as "" rather than as a family named "cpu".+ monkeypatch.setattr(+ stack_mod,+ "_detect_cuda_torch_index_url",+ lambda: "https://download.pytorch.org/whl/cpu",+ )+ assert stack_mod._driver_cuda_torch_flavor_tag() == ""+ monkeypatch.setattr(+ stack_mod,+ "_detect_cuda_torch_index_url",+ lambda: "https://download.pytorch.org/whl/cu126/",+ )+ assert stack_mod._driver_cuda_torch_flavor_tag() == "cu126"++ def test_a_host_that_lost_its_gpu_records_cpu(self, monkeypatch):+ # Without the GPU the handover stands and the manifest records cpu.+ monkeypatch.setenv("UNSLOTH_EXPECTED_TORCH_TAG", "cpu")+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)+ monkeypatch.setattr(stack_mod, "_RECORDED_TORCH_TAG", "cu128")+ monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: False)++ assert stack_mod._expected_torch_flavor_tag() == "cpu"++ def test_both_handover_readers_go_through_one_helper(self):+ # Three call sites read this tag; an inline os.environ read drifting out of step with+ # the helper is how the skip came back the first time.+ source = inspect.getsource(stack_mod)+ assert source.count('os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG"') == 1+ assert "def _handover_torch_flavor_tag(" in source++# What a localized nvidia-smi writes, which -X utf8 decodes as UTF-8 (#10173)._LOCALIZED_NVIDIA_SMI = ("import sys\n""if sys.argv[1:] == ['--query-gpu=compute_cap', '--format=csv,noheader,nounits']:\n"" sys.stdout.buffer.write(b'8.6\\n')\n"+ # -L is the GPU-presence predicate the family probe now shares with+ # _has_usable_nvidia_gpu. Without an arm for it this double prints the banner, which+ # lists no GPU, and the candidate is skipped: the test then passes only on a host that+ # happens to have a REAL nvidia-smi to fall through to, and fails on a GPU-free runner.+ "elif sys.argv[1:] == ['-L']:\n"+ " sys.stdout.buffer.write(b'GPU 0: NVIDIA GeForce RTX 4090 (UUID: GPU-x)\\n')\n""else:\n"" sys.stdout.buffer.write(b'| NVIDIA-SMI 591.86 CUDA Version: 13.1 |\\n')\n"" sys.stdout.buffer.write('\\u4e02\\u4fdd\\u7559\\u6240\\u6709\\u6743\\u5229\\u3002\\n'.encode('gbk'))\n"diff --git a/tests/studio/test_xpu_triton_swap.py b/tests/studio/test_xpu_triton_swap.pyindex 30c48213b52..51af2a51d36 100644--- a/tests/studio/test_xpu_triton_swap.py+++ b/tests/studio/test_xpu_triton_swap.py@@ -197,6 +197,12 @@ def _load(# pip check only runs when something said it changed the environment."_count_install_action": lambda: counted.append(1),"_red": lambda s: s,+ # _ensure_xpu_triton reads the setup-script handover through this helper rather+ # than inline, so the slice needs it by name or the guard tests NameError at call+ # time. Same semantics as the real one: the env var, lowercased.+ "_handover_torch_flavor_tag": (+ lambda: os.environ.get("UNSLOTH_EXPECTED_TORCH_TAG", "").strip().lower()+ ),# _safe_print, not print: the slice calls it by name, so stubbing "print" would leave _safe_print undefined at# exec time."_safe_print": (