Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio Linux update: torch repair and the source build read the NVIDIA driver library, a CPU wheel recorded as CUDA is repaired, a satisfied pin keeps the GPU prebuilt (#11008)
Daniel Han1ee14f00c3dc
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex c39c153124a..09198075e20 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -3537,16 +3537,17 @@ def _torch_family_for_cuda_version(major: int, minor: int) -> str:return "cpu" # ancient driver: no usable CUDA wheels-def _detect_cuda_torch_index_url() -> str:+def _detect_cuda_torch_index_url(*, known_only: bool = False) -> str | None:"""Return the pytorch.org CUDA wheel index URL for the host's NVIDIA driver.Mirrors install.sh::get_torch_index_url's CUDA ladder so `studio update` repairsto the same wheel family a fresh install would pick. Honours the explicitoverrides first (UNSLOTH_TORCH_INDEX_URL / _FAMILY) so a headless / CI installnever lets the host GPU decide. Otherwise probes nvidia-smi (parsing both "CUDA- Version:" and "CUDA UMD Version:"), defaulting to cu126 when unreadable. The- driver version is only an upper bound, so the GPU architectures can cap the- result at cu126 (see _cap_cuda_family_for_pre_turing).+ Version:" and "CUDA UMD Version:"), then the driver library, defaulting to cu126+ when neither answers, or to None with known_only. The driver version is only an+ upper bound, so the GPU architectures can cap the result at cu126 (see+ _cap_cuda_family_for_pre_turing)."""_override_url = os.environ.get("UNSLOTH_TORCH_INDEX_URL", "").strip()if _override_url:@@ -3554,7 +3555,6 @@ def _detect_cuda_torch_index_url() -> str:_override_family = os.environ.get("UNSLOTH_TORCH_INDEX_FAMILY", "").strip()if _override_family:return f"{_PYTORCH_WHL_BASE}/{_override_family.strip('/')}"- tag = "cu126" # default when the driver CUDA version cannot be read# Every candidate until one ANSWERS, the way _has_usable_nvidia_gpu does. Taking the# first that merely exists loses to a stale nvidia-smi on PATH: the presence probe# walks past it to the working Program Files copy and confirms the GPU, while this one@@ -3588,20 +3588,37 @@ def _detect_cuda_torch_index_url() -> str:return f"{_PYTORCH_WHL_BASE}/{_cap_cuda_family_for_pre_turing(tag, exe)}"# No nvidia-smi: the driver libraries carry the same version and the SMs the pre-Turing# cap needs. Defaulting to cu126 gave Blackwell a kernel-less wheel; without SMs the- # default stays, since cu128+ has none for Maxwell, Pascal or Volta either.+ # caller's default stays, since cu128+ has none for Maxwell, Pascal or Volta either.inventory = _nvidia_library_inventory()if inventory is not None and inventory.cuda_driver_version:- sms = []- for row in inventory.devices:- m = re.fullmatch(r"(\d+)\.(\d+)", row.get("compute_cap", ""))- if m is None:- sms = []- break- sms.append(int(m.group(1)) * 10 + int(m.group(2)))+ sms = _inventory_compute_sms(inventory)if sms:family = _torch_family_for_cuda_version(*inventory.cuda_driver_version)return f"{_PYTORCH_WHL_BASE}/{_cap_cuda_family_for_pre_turing(family, None, sms)}"- return f"{_PYTORCH_WHL_BASE}/{tag}"+ return None if known_only else f"{_PYTORCH_WHL_BASE}/cu126"+++def _inventory_compute_sms(inventory) -> "list[int]":+ """Every sm_NN the driver library lists; empty when one row is unreadable, like+ _nvidia_compute_sms."""+ sms: list[int] = []+ for row in inventory.devices:+ m = re.fullmatch(r"(\d+)\.(\d+)", row.get("compute_cap", ""))+ if m is None:+ return []+ sms.append(int(m.group(1)) * 10 + int(m.group(2)))+ return sms+++def _host_compute_sms() -> "list[int] | None":+ """The host's sm_NN list from nvidia-smi, else from the driver library; None when+ neither can say."""+ smi = _nvidia_smi_path()+ sms = _nvidia_compute_sms(smi) if smi else None+ if sms:+ return sms+ inventory = _nvidia_library_inventory()+ return (_inventory_compute_sms(inventory) or None) if inventory is not None else Nonedef _driver_cuda_torch_flavor_tag() -> str:@@ -3802,6 +3819,13 @@ def _explicit_unknown_family_torch_index_url() -> "str | None":return url+def _deliberate_cpu_torch() -> bool:+ """Someone chose CPU torch: an explicit CPU index pin, or a manifest that recorded cpu+ as NAMED rather than selected and nothing in this run names a GPU family instead.+ An unproven cpu record is not a choice."""+ return _explicit_cpu_torch_index_pin() or _expected_torch_flavor_was_pinned("cpu")++def _ensure_cuda_torch() -> None:"""Repair a venv whose torch is a ROCm build on an NVIDIA host.@@ -3903,8 +3927,7 @@ def _ensure_cuda_torch() -> None:_span = _cuda_family_sm_range(_family, _installed_release)if _span is None:return # untagged or unrecognised build: not this check's business- _smi = _nvidia_smi_path()- _sms = _nvidia_compute_sms(_smi) if _smi else None+ _sms = _host_compute_sms()if not _sms or _span_covers(_span, _sms):return # healthy CUDA torch this host can use# Never trade one partial family for another, or reinstall the same one forever.@@ -3917,6 +3940,23 @@ def _ensure_cuda_torch() -> None:f"torch is {_family} but this host has GPUs outside its "f"sm_{_span[0]}-{_span[1]} range")+ elif (+ _marker == "cpu"+ and not _deliberate_cpu_torch()+ and _is_cuda_family_leaf(+ _torch_index_leaf(_detect_cuda_torch_index_url(known_only = True) or "")+ )+ ):+ # A CPU wheel nobody asked for on an NVIDIA host whose driver is known to run a CUDA+ # wheel (the selector's cu126 default for an unreadable driver is not evidence): a+ # dependency step resolved torch from PyPI, or the GPU was not detected at install+ # time. The Windows flavour invariant catches this; Linux only recorded it.+ _recorded = _RECORDED_TORCH_TAG or ""+ _why = "torch is a CPU build on an NVIDIA host" + (+ f" although this install recorded {_recorded}"+ if _is_cuda_family_leaf(_recorded)+ else ""+ )else:return # healthy CUDA torch matching the pin, or a deliberate CPU wheeldiff --git a/studio/setup.sh b/studio/setup.shindex 9851f9932f2..54d56bd5849 100755--- a/studio/setup.sh+++ b/studio/setup.sh@@ -258,6 +258,19 @@ _nvcc_meets_llama_minimum() {# UNSLOTH_LLAMA_CUDA_ARCHS) wins verbatim; else parse+dedupe compute_cap text# ($1). Empty means "no arch detected", so the caller builds CPU instead of a# PTX-only binary that fails on an old driver (#5854).+# Every GPU's capability from nvidia_probe.py, one per line; nothing when the probe is off or+# absent, or when one listed GPU has no readable capability (a partial list would build kernels+# for part of the machine).+_probe_compute_caps() {+ [ -f "$SCRIPT_DIR/nvidia_probe.py" ] && command -v python3 >/dev/null 2>&1 || return 0+ _setup_run_smi python3 -I "$SCRIPT_DIR/nvidia_probe.py" 2>/dev/null | awk '+ /^GPU [0-9]+:/ {+ if (match($0, /\(compute [0-9]+\.[0-9]+\)$/)) caps = caps substr($0, RSTART + 9, RLENGTH - 10) "\n"+ else bad = 1+ }+ END { if (!bad) printf "%s", caps }' || true+}+_resolve_cuda_archs() {local _raw_caps=$1local _arch_override=$2@@ -3230,6 +3243,33 @@ _has_local_llama_server() {# The backend the installed prebuilt's marker records (cuda/rocm/vulkan/cpu), or nothing.# `backend` arrived with #8520; older markers name it in llama_backend or only in the asset.+# Whether the install marker's $2 names the pinned ref $3: exact, or ($4 = refs) the installer's+# own alias and commit-prefix matching (a short commit pin matches the recorded full one). A+# published release tag is a name, not a ref, so it compares exactly.+_installed_prebuilt_ref_matches() {+ [ -f "$1/UNSLOTH_PREBUILT_INFO.json" ] || return 1+ python - "$SCRIPT_DIR/install_llama_prebuilt.py" "$1/UNSLOTH_PREBUILT_INFO.json" "$2" "$3" "${4:-exact}" <<'PY' 2>/dev/null+import importlib.util+import json+import sys++spec = importlib.util.spec_from_file_location("installer", sys.argv[1])+installer = importlib.util.module_from_spec(spec)+sys.modules["installer"] = installer # the module's dataclasses resolve through sys.modules+spec.loader.exec_module(installer)+try:+ marker = json.load(open(sys.argv[2], encoding="utf-8"))+except Exception:+ marker = {}+values = [marker.get(f) for f in sys.argv[3].split(",")] if isinstance(marker, dict) else []+values = [v.strip() for v in values if isinstance(v, str) and v.strip()]+same = any(+ v == sys.argv[4] or (sys.argv[5] == "refs" and installer.refs_match(v, sys.argv[4])) for v in values+)+sys.exit(0 if same else 1)+PY+}+_installed_prebuilt_backend() {[ -f "$1/UNSLOTH_PREBUILT_INFO.json" ] || return 0python - "$1/UNSLOTH_PREBUILT_INFO.json" <<'PY' 2>/dev/null || true@@ -3292,9 +3332,17 @@ _gpu_prebuilt_to_keep_over_cpu_build() {local install_dir=$1 backend[ "$_LLAMA_FORCE_COMPILE" != "1" ] || return 1[ -z "$_LLAMA_PR" ] || return 1- # An explicit version pin asked for that version, which the old install is not.- [ -z "${UNSLOTH_LLAMA_RELEASE_TAG:-}" ] || return 1- case "${UNSLOTH_LLAMA_TAG:-}" in ""|latest|master) ;; *) return 1 ;; esac+ # An explicit version pin asked for that version; the old install satisfies it only+ # when its marker records the same one.+ if [ -n "${UNSLOTH_LLAMA_RELEASE_TAG:-}" ]; then+ _installed_prebuilt_ref_matches "$install_dir" release_tag "$UNSLOTH_LLAMA_RELEASE_TAG" || return 1+ fi+ case "${UNSLOTH_LLAMA_TAG:-}" in+ ""|latest|master) ;;+ # A commit pin is recorded beside the build tag, in the source ref fields.+ *) _installed_prebuilt_ref_matches "$install_dir" \+ tag,requested_source_ref,resolved_source_ref,source_commit "$UNSLOTH_LLAMA_TAG" refs || return 1 ;;+ esac_has_local_llama_server "$install_dir" || return 1backend="$(_installed_prebuilt_backend "$install_dir")"case "$backend" in@@ -3974,6 +4022,10 @@ else_raw_caps=$(_setup_run_smi "$_smi_bin" --query-gpu=compute_cap --format=csv,noheader 2>/dev/null || true)fiCUDA_ARCHS="$(_resolve_cuda_archs "$_raw_caps" "${UNSLOTH_LLAMA_CUDA_ARCHS:-}")"+ # nvidia-smi absent, stale or answering N/A: the driver library lists the capabilities.+ if [ -z "$CUDA_ARCHS" ] && [ "${UNSLOTH_NVIDIA_LIBRARY_PROBE:-1}" != "0" ]; then+ CUDA_ARCHS="$(_resolve_cuda_archs "$(_probe_compute_caps)" "")"+ fiif [ -n "$CUDA_ARCHS" ]; thenCMAKE_ARGS="$CMAKE_ARGS -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHS}"diff --git a/tests/studio/install/test_cuda_repair.py b/tests/studio/install/test_cuda_repair.pyindex 05c708aab61..c20a3ee6675 100644--- a/tests/studio/install/test_cuda_repair.py+++ b/tests/studio/install/test_cuda_repair.py@@ -269,9 +269,24 @@ class TestCudaRepairSkips:mock_pip.assert_not_called()def test_deliberate_cpu_wheel_no_repair(self):- mock_pip = _run_cuda_repair(torch_state = "cpu")+ with (+ patch.object(stack_mod, "_RECORDED_TORCH_TAG", "cpu"),+ patch.object(stack_mod, "_RECORDED_TORCH_TAG_PINNED", True),+ ):+ mock_pip = _run_cuda_repair(torch_state = "cpu")mock_pip.assert_not_called()+ def test_a_cuda_request_this_run_outranks_the_pinned_cpu_record(self, monkeypatch):+ """UNSLOTH_TORCH_BACKEND=cuda names a GPU family, so the old pinned cpu record no longer+ speaks for this run and a CPU wheel a later step left behind is repaired."""+ monkeypatch.delenv("UNSLOTH_TORCH_BACKEND_SOURCE", raising = False)+ with (+ patch.object(stack_mod, "_RECORDED_TORCH_TAG", "cpu"),+ patch.object(stack_mod, "_RECORDED_TORCH_TAG_PINNED", True),+ ):+ mock_pip = _run_cuda_repair(backend = "cuda", torch_state = "cpu")+ assert mock_pip.call_count == 1+def test_backend_rocm_skips(self):mock_pip = _run_cuda_repair(backend = "rocm", torch_state = "hip")mock_pip.assert_not_called()@@ -699,6 +714,94 @@ class TestPreTuringWheelFamily:# The updater runs setup.ps1 -> install_python_stack.py, never install.ps1, which held the only flavor repair.+class TestTheRepairReadsTheDriverLibrary:+ """nvidia-smi absent or stale left the pre-Turing repair blind; the driver library+ lists the same capabilities (#10985 taught the index selector, not the repair)."""++ def test_the_inventory_supplies_the_sms_when_nvidia_smi_cannot(self):+ inventory = SimpleNamespace(cuda_driver_version = (13, 0), devices = [{"compute_cap": "7.0"}])+ with patch.object(stack_mod, "_nvidia_library_inventory", return_value = inventory):+ mock_pip = _run_cuda_repair(+ torch_state = "cuda|cu130|2.11.0", cuda_version = "13.0", smi_rc = 1+ )+ assert mock_pip.call_count == 1+ assert "cu126" in _index_url(mock_pip)+ with patch.object(stack_mod, "_nvidia_library_inventory", return_value = None):+ blind = _run_cuda_repair(torch_state = "cuda|cu130|2.11.0", cuda_version = "13.0", smi_rc = 1)+ blind.assert_not_called()++ def test_an_unreadable_inventory_row_is_not_evidence(self):+ inventory = SimpleNamespace(+ cuda_driver_version = (13, 0), devices = [{"compute_cap": "7.0"}, {"compute_cap": ""}]+ )+ assert stack_mod._inventory_compute_sms(inventory) == []+ with patch.object(stack_mod, "_nvidia_library_inventory", return_value = inventory):+ mock_pip = _run_cuda_repair(+ torch_state = "cuda|cu130|2.11.0", cuda_version = "13.0", smi_rc = 1+ )+ mock_pip.assert_not_called()++ def test_nvidia_smi_still_wins_when_it_answers(self):+ inventory = SimpleNamespace(cuda_driver_version = (13, 0), devices = [{"compute_cap": "7.0"}])+ with patch.object(stack_mod, "_nvidia_library_inventory", return_value = inventory):+ mock_pip = _run_cuda_repair(+ torch_state = "cuda|cu130|2.11.0", cuda_version = "13.0", compute_caps = ("8.9",)+ )+ mock_pip.assert_not_called()+++class TestACpuWheelTheInstallRecordedAsCudaIsRepaired:+ """Linux only recorded the expected flavour; a dependency step that resolved torch from+ PyPI left a CPU wheel on a GPU host until the next fresh install."""++ def test_the_recorded_cuda_family_brings_cuda_torch_back(self):+ with patch.object(stack_mod, "_RECORDED_TORCH_TAG", "cu128"):+ mock_pip = _run_cuda_repair(torch_state = "cpu", cuda_version = "12.8")+ assert mock_pip.call_count == 1+ assert "cu128" in _index_url(mock_pip)++ def test_a_cpu_wheel_nobody_chose_is_repaired_on_an_nvidia_host(self):+ # No record (an older manifest) and an automatic cpu record (the GPU was not detected+ # at install time) are both accidents once an NVIDIA GPU is visible.+ for recorded, pinned in ((None, False), ("", False), ("cpu", False)):+ with (+ patch.object(stack_mod, "_RECORDED_TORCH_TAG", recorded),+ patch.object(stack_mod, "_RECORDED_TORCH_TAG_PINNED", pinned),+ ):+ mock_pip = _run_cuda_repair(torch_state = "cpu", cuda_version = "13.0")+ assert mock_pip.call_count == 1, (recorded, pinned)+ assert "cu130" in _index_url(mock_pip)++ def test_a_driver_that_runs_no_cuda_wheel_keeps_its_cpu_wheel(self):+ # CUDA 10.2: the index selector answers cpu, so the CPU wheel is what this host gets.+ with patch.object(stack_mod, "_RECORDED_TORCH_TAG", "cu128"):+ _run_cuda_repair(torch_state = "cpu", cuda_version = "10.2").assert_not_called()++ def test_an_unreadable_driver_is_not_evidence_for_a_cuda_wheel(self):+ # Neither nvidia-smi nor the library names the driver's CUDA version: the selector's+ # cu126 default must not replace a CPU wheel a driver older than that could not run.+ with (+ patch.object(stack_mod, "_RECORDED_TORCH_TAG", "cu128"),+ patch.object(stack_mod, "_nvidia_library_inventory", return_value = None),+ ):+ _run_cuda_repair(torch_state = "cpu", cuda_version = "").assert_not_called()+ with patch.object(stack_mod, "_nvidia_smi_usable_candidates", return_value = []):+ assert stack_mod._detect_cuda_torch_index_url(known_only = True) is None+ assert stack_mod._detect_cuda_torch_index_url().endswith("/cu126")++ def test_a_named_cpu_choice_or_an_explicit_cpu_pin_is_respected(self):+ with (+ patch.object(stack_mod, "_RECORDED_TORCH_TAG", "cpu"),+ patch.object(stack_mod, "_RECORDED_TORCH_TAG_PINNED", True),+ ):+ _run_cuda_repair(torch_state = "cpu").assert_not_called()+ with patch.object(stack_mod, "_RECORDED_TORCH_TAG", "cu128"):+ _run_cuda_repair(torch_state = "cpu", index_family = "cpu").assert_not_called()+ _run_cuda_repair(torch_state = "cpu", backend = "cpu").assert_not_called()+ _run_cuda_repair(torch_state = "cpu", nvidia = False).assert_not_called()+ _run_cuda_repair(torch_state = "cpu", cvd = "").assert_not_called()++_ensure_expected_torch_flavor = stack_mod._ensure_expected_torch_flavor_UNSET = object()diff --git a/tests/studio/install/test_setup_keeps_gpu_prebuilt_over_cpu_source_fallback.py b/tests/studio/install/test_setup_keeps_gpu_prebuilt_over_cpu_source_fallback.pyindex da10ccb88a4..fb860f34108 100644--- a/tests/studio/install/test_setup_keeps_gpu_prebuilt_over_cpu_source_fallback.py+++ b/tests/studio/install/test_setup_keeps_gpu_prebuilt_over_cpu_source_fallback.py@@ -31,6 +31,7 @@ requires_bash = pytest.mark.skipif(BASH is None, reason = "a working bash is req_FUNCTIONS = ("_has_local_llama_server() {",+ "_installed_prebuilt_ref_matches() {","_installed_prebuilt_backend() {","_gpu_prebuilt_to_keep_over_cpu_build() {",)@@ -90,6 +91,8 @@ def _decide(tmp_path, install_dir, **env):"PATH": f"{stub_bin}{os.pathsep}{os.environ.get('PATH', '')}","_LLAMA_FORCE_COMPILE": "0","_LLAMA_PR": "",+ # The pin comparison imports install_llama_prebuilt.py from beside setup.sh.+ "SCRIPT_DIR": str(SETUP_SH.parent),"_setup_nvidia_physical": "false","_setup_amd_detected": "false",}@@ -152,6 +155,65 @@ class TestTheKeepDecision:)assert _decide(tmp_path, install_dir, UNSLOTH_LLAMA_TAG = "latest", **nvidia) == "KEEP cuda"+ def test_a_version_pin_the_old_install_already_satisfies_keeps_it(self, tmp_path):+ install_dir = _install(+ tmp_path, {"backend": "cuda", "tag": "b8508", "release_tag": "b8508-mix"}+ )+ nvidia = {"_setup_nvidia_physical": "true"}+ assert _decide(tmp_path, install_dir, UNSLOTH_LLAMA_TAG = "b8508", **nvidia) == "KEEP cuda"+ assert _decide(tmp_path, install_dir, UNSLOTH_LLAMA_TAG = "b8509", **nvidia) == "REPLACE"+ # The installer's own matching: a short commit pin names the recorded full commit.+ install_dir = _install(+ tmp_path,+ {+ "backend": "cuda",+ "tag": "0123456789abcdef0123456789abcdef01234567",+ "release_tag": "b8508-mix",+ },+ )+ assert (+ _decide(tmp_path, install_dir, UNSLOTH_LLAMA_TAG = "0123456789ab", **nvidia)+ == "KEEP cuda"+ )+ assert (+ _decide(tmp_path, install_dir, UNSLOTH_LLAMA_TAG = "fedcba987654", **nvidia) == "REPLACE"+ )+ # The marker writer records a commit pin in source_commit beside the upstream build tag.+ install_dir = _install(+ tmp_path,+ {+ "backend": "cuda",+ "tag": "b8508",+ "release_tag": "b8508-mix",+ "requested_source_ref": "0123456789abcdef0123456789abcdef01234567",+ "resolved_source_ref": "0123456789abcdef0123456789abcdef01234567",+ "source_commit": "0123456789abcdef0123456789abcdef01234567",+ },+ )+ assert (+ _decide(tmp_path, install_dir, UNSLOTH_LLAMA_TAG = "0123456789ab", **nvidia)+ == "KEEP cuda"+ )+ assert (+ _decide(tmp_path, install_dir, UNSLOTH_LLAMA_TAG = "fedcba987654", **nvidia) == "REPLACE"+ )+ assert (+ _decide(tmp_path, install_dir, UNSLOTH_LLAMA_RELEASE_TAG = "b8508-mix", **nvidia)+ == "KEEP cuda"+ )+ assert (+ _decide(tmp_path, install_dir, UNSLOTH_LLAMA_RELEASE_TAG = "b8509-mix", **nvidia)+ == "REPLACE"+ )+ # A published release tag is a name: a hex-looking prefix of the recorded one is not it.+ install_dir = _install(+ tmp_path, {"backend": "cuda", "release_tag": "0123456789abcdef0123456789abcdef01234567"}+ )+ assert (+ _decide(tmp_path, install_dir, UNSLOTH_LLAMA_RELEASE_TAG = "0123456789ab", **nvidia)+ == "REPLACE"+ )+def test_a_cpu_prebuilt_is_not_worth_keeping(self, tmp_path):install_dir = _install(tmp_path, {"backend": "cpu"})assert _decide(tmp_path, install_dir, _setup_nvidia_physical = "true") == "REPLACE"@@ -457,3 +519,49 @@ def test_every_footer_names_the_outcome():def test_the_flags_are_initialised_for_set_u():assert '_LLAMA_KEPT_GPU_PREBUILT=""' in SETUP_TEXTassert "_LLAMA_CPU_ONLY_ON_GPU_HOST=false" in SETUP_TEXT+++def test_the_source_build_reads_capabilities_from_the_driver_library_too():+ """setup.sh's CUDA source build turned CUDA off without nvidia-smi (#5854); the probe+ module beside it lists the same capabilities."""+ start = SETUP_TEXT.index(+ 'CUDA_ARCHS="$(_resolve_cuda_archs "$_raw_caps" "${UNSLOTH_LLAMA_CUDA_ARCHS:-}")"'+ )+ end = SETUP_TEXT.index('if [ -n "$CUDA_ARCHS" ]; then', start)+ between = SETUP_TEXT[start:end]+ # After the first resolution, so an nvidia-smi answering N/A falls back as an absent one does.+ assert "_probe_compute_caps" in between and '[ -z "$CUDA_ARCHS" ]' in between+ assert "UNSLOTH_NVIDIA_LIBRARY_PROBE" in between+++@requires_bash+ "listing, expected",+ [+ ("GPU 0: A (compute 8.9)\nGPU 1: B (compute 12.0)\n", "8.9\n12.0\n"),+ # One GPU without a readable capability voids the list, as the Python side does.+ ("GPU 0: A (compute 8.9)\nGPU 1: B (compute )\n", ""),+ ("", ""),+ ],+)+def test_the_probe_capabilities_are_all_or_nothing(tmp_path, listing, expected):+ start = SETUP_TEXT.index("_probe_compute_caps() {")+ body = SETUP_TEXT[start : SETUP_TEXT.index("\n}\n", start) + 3]+ (tmp_path / "python3").write_text(f"#!/bin/sh\nprintf '%b' {listing!r}\n", encoding = "utf-8")+ (tmp_path / "python3").chmod(0o755)+ (tmp_path / "nvidia_probe.py").write_text("", encoding = "utf-8")+ script = '_setup_run_smi() { "$@"; }\n' + body + "\n_probe_compute_caps\n"+ result = subprocess.run(+ [BASH, "-c", script],+ env = {+ **os.environ,+ "PATH": f"{tmp_path}{os.pathsep}{os.environ.get('PATH', '')}",+ "SCRIPT_DIR": str(tmp_path),+ },+ stdout = subprocess.PIPE,+ stderr = subprocess.PIPE,+ text = True,+ timeout = 60,+ )+ assert result.returncode == 0, result.stderr+ assert result.stdout == expected