Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: recover the GPU capacity nvidia-smi will not report on a DGX Spark (#10703)
Daniel Han0392cb2c65d3
diff --git a/studio/backend/tests/test_dgx_spark_gpu_inventory.py b/studio/backend/tests/test_dgx_spark_gpu_inventory.pynew file mode 100644index 00000000000..aed3d47eba8--- /dev/null+++ b/studio/backend/tests/test_dgx_spark_gpu_inventory.py@@ -0,0 +1,516 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""An integrated CUDA SoC (Jetson, DGX Spark) publishes no capacity through nvidia-smi.++``nvidia-smi --query-gpu=memory.total`` answers ``[N/A]`` on a DGX Spark, which NVIDIA+documents as a known issue. The row is kept so the card stays in the inventory, but with+no capacity the frontend maps it to zero, the GGUF fit classifier returns ``ram``, and+the picker warns "No GPU detected. Runs on system RAM and CPU" on a 121 GiB Blackwell,+while the live monitor names the GB10 and prints "Unknown / 0.00 GiB" beside it (#10691).++Hermetic: torch, nvidia-smi and host memory are stubbed, so these run anywhere.+"""++from __future__ import annotations++import sys+import types++import psutil+import utils.hardware.hardware as hw+from utils.hardware import nvidia++GIB = 1 << 30+MIB = 1 << 20+# What a DGX Spark actually reports.+SPARK_TOTAL_BYTES = 124609 * MIB+SPARK_TOTAL_GB = round(SPARK_TOTAL_BYTES / GIB, 2)+++class _SparkProps:+ """cudaDeviceProp as torch surfaces it for a GB10."""++ name = "NVIDIA GB10"+ total_memory = SPARK_TOTAL_BYTES+ is_integrated = 1+ gcnArchName = ""+++class _DiscreteProps:+ name = "NVIDIA GB200"+ total_memory = 183 * GIB+ is_integrated = 0+ gcnArchName = ""+++def _torch_module(props) -> types.SimpleNamespace:+ return types.SimpleNamespace(get_device_properties = lambda ordinal: props)+++def _not_hip(monkeypatch) -> None:+ """Stub the torch the CUDA classifier asks for HIP.++ Without this the suite reads the HOST's torch, so every assertion about an integrated+ CUDA part inverts on a ROCm machine, where ``torch.version.hip`` is set and the+ classifier correctly declines. Caught on a real gfx1151 runner, not by reading it.+ """+ monkeypatch.setitem(+ sys.modules, "torch", types.SimpleNamespace(version = types.SimpleNamespace(hip = None))+ )+++def _cuda_host(monkeypatch, props) -> None:+ _not_hip(monkeypatch)+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0])+ monkeypatch.setattr(+ hw,+ "_get_parent_visible_gpu_spec",+ lambda: {"raw": "0", "numeric_ids": [0], "supports_explicit_gpu_ids": True},+ )+ monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (_torch_module(props), "cuda"))+++def _smi_rows(monkeypatch, memory_total_gb) -> None:+ """Stand in for nvidia.py's parsed inventory, whose [N/A] total is already a None."""+ monkeypatch.setattr(+ nvidia,+ "_query_gpu_inventory",+ lambda caller: [{"index": 0, "name": "NVIDIA GB10", "memory_total_gb": memory_total_gb}],+ )+++def test_spark_recovers_the_capacity_nvidia_smi_will_not_report(monkeypatch):+ _cuda_host(monkeypatch, _SparkProps())+ _smi_rows(monkeypatch, None)++ result = hw.get_backend_visible_gpu_info()+ device = result["devices"][0]++ assert result["available"] is True+ assert device["name"] == "NVIDIA GB10"+ # The regression: this was None, which the frontend reads as a 0 GiB card.+ assert device["memory_total_gb"] == SPARK_TOTAL_GB+++def test_spark_is_reported_as_one_pool_not_two(monkeypatch):+ """The recovered total IS system memory, so it must not be added to it."""+ _cuda_host(monkeypatch, _SparkProps())+ _smi_rows(monkeypatch, None)++ device = hw.get_backend_visible_gpu_info()["devices"][0]++ assert device["unified_memory"] is True+ assert device["shared_memory_host_backed_gb"] == SPARK_TOTAL_GB+++def test_readable_smi_capacity_is_kept_as_is(monkeypatch):+ """A discrete card answers memory.total, and nothing here may second-guess it."""+ _cuda_host(monkeypatch, _DiscreteProps())+ _smi_rows(monkeypatch, 23.99)++ device = hw.get_backend_visible_gpu_info()["devices"][0]++ assert device["memory_total_gb"] == 23.99+ assert device.get("unified_memory") is not True+++def test_torch_fallback_also_flags_the_integrated_pool(monkeypatch):+ """With nvidia-smi missing entirely the inventory comes from torch; same verdict."""+ _cuda_host(monkeypatch, _SparkProps())+ monkeypatch.setattr(nvidia, "_query_gpu_inventory", lambda caller: None)++ device = hw.get_backend_visible_gpu_info()["devices"][0]++ assert device["memory_total_gb"] == SPARK_TOTAL_GB+ assert device["unified_memory"] is True+ assert device["shared_memory_host_backed_gb"] == SPARK_TOTAL_GB+++def test_rocm_is_not_classified_by_the_cuda_integrated_flag(monkeypatch):+ """HIP left that field unassigned before 6.2, so reading it there calls a card unified."""+ props = _SparkProps()+ monkeypatch.setattr(hw, "IS_ROCM", True)++ assert hw._cuda_props_are_integrated(props) is False+++# ── nothing above may reach a host that is not one of these parts ────────────+++def test_a_readable_smi_host_never_consults_torch(monkeypatch):+ """The repair is scoped to hosts with a missing capacity.++ /api/system is polled every 5s by the floating monitor and every 3s from Settings.+ A discrete card answers memory.total, so it must leave that poll exactly as it was:+ no torch import, no property query, no new failure mode on a machine that was fine.+ """+ _cuda_host(monkeypatch, _DiscreteProps())+ _smi_rows(monkeypatch, 23.99)++ def _forbidden(*args, **kwargs):+ raise AssertionError("torch inventory consulted on a host with nothing to repair")++ monkeypatch.setattr(hw, "_torch_get_device_inventory", _forbidden)++ assert hw.get_backend_visible_gpu_info()["devices"][0]["memory_total_gb"] == 23.99+++def test_multi_gpu_smi_host_is_untouched(monkeypatch):+ """Two discrete cards, both readable: same rows out as in."""+ _cuda_host(monkeypatch, _DiscreteProps())+ monkeypatch.setattr(+ hw,+ "_get_parent_visible_gpu_spec",+ lambda: {"raw": "0,1", "numeric_ids": [0, 1], "supports_explicit_gpu_ids": True},+ )+ monkeypatch.setattr(+ nvidia,+ "_query_gpu_inventory",+ lambda caller: [+ {"index": 0, "name": "NVIDIA H100", "memory_total_gb": 79.65},+ {"index": 1, "name": "NVIDIA H100", "memory_total_gb": 79.65},+ ],+ )++ devices = hw.get_backend_visible_gpu_info()["devices"]++ assert [d["memory_total_gb"] for d in devices] == [79.65, 79.65]+ assert not any(d.get("unified_memory") for d in devices)+ assert not any(d.get("shared_memory_host_backed_gb") for d in devices)+++def test_xpu_is_not_classified_by_the_cuda_integrated_flag(monkeypatch):+ """A same-named field on a future Intel wheel must not rewrite an iGPU's capacity."""+ _not_hip(monkeypatch)+ monkeypatch.setattr(hw, "IS_ROCM", False)+ assert hw._cuda_props_are_integrated(_SparkProps(), "xpu") is False+ assert hw._cuda_props_are_integrated(_SparkProps(), None) is False+ assert hw._cuda_props_are_integrated(_SparkProps(), "cuda") is True+++def test_an_unsizeable_card_is_still_reported_when_torch_cannot_answer(monkeypatch):+ """Both sources blank: keep the row nvidia-smi found rather than claim no GPU."""+ _cuda_host(monkeypatch, _SparkProps())+ _smi_rows(monkeypatch, None)+ monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (None, None))++ result = hw.get_backend_visible_gpu_info()++ assert result["available"] is True+ assert result["devices"][0]["name"] == "NVIDIA GB10"+ assert result["devices"][0]["memory_total_gb"] is None+++# ── the live monitor ─────────────────────────────────────────────────────────+++def _smi_utilization(+ monkeypatch,+ vram_total_gb,+ vram_used_gb = None,+) -> dict:+ """nvidia-smi's utilization rows, whose [N/A] memory columns are already None."""+ payload = {+ "available": True,+ "devices": [+ {+ "index": 0,+ "index_kind": "physical",+ "visible_ordinal": 0,+ "gpu_utilization_pct": 0.0,+ "temperature_c": 46.0,+ "vram_used_gb": vram_used_gb,+ "vram_total_gb": vram_total_gb,+ "vram_utilization_pct": None,+ "power_draw_w": 12.09,+ "power_limit_w": None,+ "power_utilization_pct": None,+ }+ ],+ "backend_cuda_visible_devices": "0",+ "parent_visible_gpu_ids": [0],+ "index_kind": "physical",+ }+ monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: payload)+ return payload+++def test_monitor_sizes_the_spark_instead_of_showing_unknown(monkeypatch):+ _cuda_host(monkeypatch, _SparkProps())+ _smi_utilization(monkeypatch, vram_total_gb = None)+ import psutil++ monkeypatch.setattr(+ psutil,+ "virtual_memory",+ lambda: types.SimpleNamespace(total = 121 * GIB, available = 100 * GIB),+ )++ device = hw.get_gpu_utilization()["devices"][0]++ assert device["vram_total_gb"] == SPARK_TOTAL_GB+ assert device["vram_used_gb"] == 21.0+ assert device["vram_utilization_pct"] == round(21.0 / SPARK_TOTAL_GB * 100, 1)+ # Columns nvidia-smi DID answer are the CLI's, untouched.+ assert device["gpu_utilization_pct"] == 0.0+ assert device["temperature_c"] == 46.0+ assert device["power_draw_w"] == 12.09+++def test_monitor_leaves_a_readable_card_alone(monkeypatch):+ """A discrete card answers both memory columns, so nothing here may run."""+ _cuda_host(monkeypatch, _DiscreteProps())+ _smi_utilization(monkeypatch, vram_total_gb = 79.65, vram_used_gb = 12.0)++ def _forbidden(*args, **kwargs):+ raise AssertionError("torch inventory consulted for a card nvidia-smi could size")++ monkeypatch.setattr(hw, "_torch_get_device_inventory", _forbidden)++ device = hw.get_gpu_utilization()["devices"][0]++ assert (device["vram_total_gb"], device["vram_used_gb"]) == (79.65, 12.0)+++def test_monitor_does_not_size_a_discrete_card_smi_could_not_read(monkeypatch):+ """An unreadable DISCRETE card keeps its unknown: host RAM is not its VRAM."""+ _cuda_host(monkeypatch, _DiscreteProps())+ _smi_utilization(monkeypatch, vram_total_gb = None)++ device = hw.get_gpu_utilization()["devices"][0]++ assert device["vram_total_gb"] is None+ assert device["vram_used_gb"] is None+++def test_monitor_reconciliation_creates_no_driver_context(monkeypatch):+ """The poll runs every 3-5s; mem_get_info would pin ~612 MiB for the process life."""+ _cuda_host(monkeypatch, _SparkProps())+ _smi_utilization(monkeypatch, vram_total_gb = None)++ def _forbidden(*args, **kwargs):+ raise AssertionError("the monitor poll reached the occupancy probe")++ monkeypatch.setattr(hw, "_torch_get_per_device_info", _forbidden)++ assert hw.get_gpu_utilization()["devices"][0]["vram_total_gb"] == SPARK_TOTAL_GB+++def test_a_hip_torch_is_never_read_with_the_cuda_rule(monkeypatch):+ """HIP reuses this namespace and a real APU sets the same flag.++ IS_ROCM is a global that detection publishes, so the classifier also asks torch+ directly: this covers the window before detection has settled, and the ROCm CI runner+ where the previous revision of these tests inverted.+ """+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setitem(+ sys.modules, "torch", types.SimpleNamespace(version = types.SimpleNamespace(hip = "6.2.0"))+ )++ assert hw._cuda_props_are_integrated(_SparkProps(), "cuda") is False+++def _smi_visible_utilization(monkeypatch, rows) -> dict:+ payload = {+ "available": True,+ "devices": rows,+ "backend_cuda_visible_devices": "0",+ "parent_visible_gpu_ids": [0],+ "index_kind": "physical",+ }+ monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: payload)+ return payload+++def _unsized_row(+ index = 0,+ ordinal = 0,+ index_kind = "physical",+) -> dict:+ return {+ "index": index,+ "index_kind": index_kind,+ "visible_ordinal": ordinal,+ "gpu_utilization_pct": 0.0,+ "temperature_c": 46.0,+ "vram_used_gb": None,+ "vram_total_gb": None,+ "vram_utilization_pct": None,+ "power_draw_w": 12.09,+ "power_limit_w": None,+ "power_utilization_pct": None,+ }+++def test_the_system_poll_function_is_the_one_that_gets_repaired(monkeypatch):+ """/api/system reads get_visible_gpu_utilization, NOT get_gpu_utilization.++ main.py::_get_cached_system_gpu_info calls the former, so repairing only the latter+ left the floating monitor showing Unknown / 0.00 GiB, which is the screen #10691 is+ actually about.+ """+ _cuda_host(monkeypatch, _SparkProps())+ _smi_visible_utilization(monkeypatch, [_unsized_row()])+ monkeypatch.setattr(+ psutil,+ "virtual_memory",+ lambda: types.SimpleNamespace(total = 121 * GIB, available = 100 * GIB),+ )++ device = hw.get_visible_gpu_utilization()["devices"][0]++ assert device["vram_total_gb"] == SPARK_TOTAL_GB+ assert device["vram_used_gb"] == 21.0+++def test_a_uuid_mask_still_reaches_the_reconciliation(monkeypatch):+ """A UUID or MIG mask resolves to numeric_ids=None, and nvidia.py answers anyway.++ Its rows are relative-indexed and ordered by the mask, which is the order torch+ enumerates too, so the join is on visible_ordinal rather than a physical id.+ """+ _cuda_host(monkeypatch, _SparkProps())+ monkeypatch.setattr(+ hw,+ "_get_parent_visible_gpu_spec",+ lambda: {"raw": "GPU-9254a6cb", "numeric_ids": None, "supports_explicit_gpu_ids": False},+ )+ monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1)+ _smi_visible_utilization(monkeypatch, [_unsized_row(index = 0, index_kind = "relative")])++ device = hw.get_visible_gpu_utilization()["devices"][0]++ assert device["vram_total_gb"] == SPARK_TOTAL_GB+++def test_a_mismatched_device_order_refuses_the_join(monkeypatch):+ """FASTEST_FIRST puts torch ordinals in a different space from nvidia-smi rows.++ Joining them anyway attaches another card's capacity to a row. The module already+ rejects this exact cross-source mapping for its SMI VRAM query; the repair uses the+ same gate rather than guessing.+ """+ _cuda_host(monkeypatch, _DiscreteProps())+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: False)+ _smi_rows(monkeypatch, None)++ # The resolver refuses outright rather than handing back a mapping to guess with.+ assert hw._integrated_cuda_inventory([0]) == ({}, "index")++ # ...and the endpoint keeps the nvidia-smi rows rather than reaching the torch+ # fallback. That fallback labels its rows with physical ids taken from the mask, so+ # here it would publish one card's name and capacity under another card's index,+ # which is the same wrong join by another route and is what GPU selection reads.+ # An SMI row missing a total is the lesser answer, and the one this host had before+ # the repair existed.+ device = hw.get_backend_visible_gpu_info()["devices"][0]+ assert device["name"] == "NVIDIA GB10"+ assert device["memory_total_gb"] is None+ assert device.get("unified_memory") is not True+++def test_a_partial_torch_inventory_never_drops_an_smi_card(monkeypatch):+ """_torch_get_device_inventory skips a device whose probe failed, so its list can+ be SHORT rather than empty. A short list must not replace the cards nvidia-smi saw.+ """+ _cuda_host(monkeypatch, _SparkProps())+ monkeypatch.setattr(+ hw,+ "_get_parent_visible_gpu_spec",+ lambda: {"raw": "0,1", "numeric_ids": [0, 1], "supports_explicit_gpu_ids": True},+ )+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0, 1])+ monkeypatch.setattr(+ nvidia,+ "_query_gpu_inventory",+ lambda caller: [+ {"index": 0, "name": "NVIDIA GB10", "memory_total_gb": None},+ {"index": 1, "name": "NVIDIA GB10", "memory_total_gb": None},+ ],+ )+ # Only ordinal 0 answers, the shape a fallen-off-the-bus card produces.+ monkeypatch.setattr(+ hw,+ "_torch_get_device_inventory",+ lambda indices: [+ {+ "index": 0,+ "visible_ordinal": 0,+ "name": "NVIDIA GB10",+ "total_gb": SPARK_TOTAL_GB,+ "used_gb": None,+ "shared_memory": False,+ "shared_memory_host_backed_gb": None,+ "_rocm_known_unified": False,+ "_rocm_gfx": "",+ "_cuda_integrated": True,+ }+ ],+ )++ result = hw.get_backend_visible_gpu_info()++ assert len(result["devices"]) == 2+ assert [d["name"] for d in result["devices"]] == ["NVIDIA GB10", "NVIDIA GB10"]+++def test_a_repaired_spark_row_is_marked_shared(monkeypatch):+ """gpu-vram.ts splits the dedicated and shared pools on `shared_memory` alone.++ The host-backed figure is read only after that split, so a repaired row that carries+ the figure without the flag lands in the dedicated total and is then counted a+ second time beside the same system RAM, which is the double count this repair+ exists to prevent.+ """+ _cuda_host(monkeypatch, _SparkProps())+ _smi_rows(monkeypatch, None)++ device = hw.get_backend_visible_gpu_info()["devices"][0]++ assert device["shared_memory"] is True+ assert device["unified_memory"] is True+ assert device["shared_memory_host_backed_gb"] == SPARK_TOTAL_GB+++def test_a_discrete_card_is_never_marked_shared(monkeypatch):+ """The flag follows the integrated property, not the repair."""+ _cuda_host(monkeypatch, _DiscreteProps())+ _smi_rows(monkeypatch, None)++ device = hw.get_backend_visible_gpu_info()["devices"][0]++ assert device.get("shared_memory") is not True+ assert device.get("unified_memory") is not True+++def test_a_known_usage_still_gets_its_percentage(monkeypatch):+ """memory.used can be readable on a row whose memory.total is [N/A].++ Filling the total is what makes the percentage computable, so a device that already+ carries a usage must not be skipped: it was the one case where both operands existed+ and the monitor still showed an unknown.+ """+ _cuda_host(monkeypatch, _SparkProps())+ utilization = {+ "devices": [+ {+ "index": 0,+ "visible_ordinal": 0,+ "vram_total_gb": None,+ "vram_used_gb": 30.0,+ "vram_utilization_pct": None,+ }+ ]+ }++ hw._reconcile_cuda_integrated_memory(utilization, [0])+ device = utilization["devices"][0]++ assert device["vram_total_gb"] == SPARK_TOTAL_GB+ # The CLI's own figure, not the host counter that stands in when it is absent.+ assert device["vram_used_gb"] == 30.0+ assert device["vram_utilization_pct"] == round((30.0 / SPARK_TOTAL_GB) * 100, 1)diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex e45e9552f44..4633021235a 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -2880,9 +2880,17 @@ def get_package_versions() -> Dict[str, Optional[str]]:def _torch_get_device_module():- """Return the appropriate torch device module (cuda or xpu) and its name."""+ """Return the appropriate torch device module (cuda or xpu) and its name.++ No torch at all answers ``(None, None)`` like an unsupported device: raising took the+ exception out through /api/system on a host whose vendor CLI HAD found GPUs.+ """device = get_device()- import torch+ try:+ import torch+ except Exception as e: # noqa: BLE001 - no torch is an answer, not a fault+ logger.debug("torch is not importable: %s", e)+ return None, Noneif device == DeviceType.CUDA:return torch.cuda, "cuda"@@ -3084,6 +3092,31 @@ def _rocm_props_total_is_carve_out(props: Any) -> bool:)+def _cuda_props_are_integrated(props: Any, backend: Optional[str] = "cuda") -> bool:+ """Whether ``props`` describes an integrated CUDA part whose VRAM is system RAM.++ Jetson and DGX Spark class parts set ``cudaDeviceProp::integrated`` (torch's+ ``is_integrated``, ``integrated`` on older wheels).++ ROCm and XPU are excluded BY NAME, not by trusting the field to be absent: HIP reuses+ this namespace and left that field unassigned before 6.2, so reading it would call a+ discrete card integrated on exactly the runtimes ``_HIP_INTEGRATED_FLAG_MIN``+ distrusts (``_rocm_props_unified_status`` is the classifier for that hardware), and a+ same-named field on a future Intel wheel must not start rewriting an iGPU's capacity.+ ``torch.version.hip`` as well as ``IS_ROCM`` because that global is published by+ detection, and this inventory is reachable from inside detection.+ """+ if IS_ROCM or backend != "cuda":+ return False+ try:+ import torch+ if getattr(torch.version, "hip", None):+ return False+ except Exception as e: # noqa: BLE001 - no torch to ask: IS_ROCM alone then+ logger.debug("HIP probe failed while classifying an integrated device: %s", e)+ return bool(getattr(props, "is_integrated", False) or getattr(props, "integrated", False))++def _torch_get_device_inventory(device_indices: list[int]) -> list[Dict[str, Any]]:"""Per-GPU name and total VRAM only, without creating a driver context.@@ -3099,7 +3132,7 @@ def _torch_get_device_inventory(device_indices: list[int]) -> list[Dict[str, Anytotals are unchanged. ``used_gb`` is always None, the value this module alreadyuses for "telemetry unavailable"."""- mod, _ = _torch_get_device_module()+ mod, backend = _torch_get_device_module()if mod is None:return []@@ -3153,6 +3186,7 @@ def _torch_get_device_inventory(device_indices: list[int]) -> list[Dict[str, Any),"_rocm_known_unified": known_unified,"_rocm_gfx": rocm_gfx,+ "_cuda_integrated": _cuda_props_are_integrated(props, backend),})except Exception as e:@@ -4856,6 +4890,10 @@ def get_gpu_utilization() -> Dict[str, Any]:numeric_ids = parent_visible_spec.get("numeric_ids")if IS_ROCM and numeric_ids is not None:_reconcile_rocm_unified_memory(result, numeric_ids)+ elif not IS_ROCM:+ # numeric_ids is None under a UUID/MIG mask, which nvidia.py resolves+ # itself and answers with relative-indexed rows: still repairable.+ _reconcile_cuda_integrated_memory(result, numeric_ids)return _gpu_utilization_payload(device,@@ -5077,6 +5115,108 @@ def _reconcile_rocm_unified_memory(utilization: Dict[str, Any], device_indices:_apply_unified_memory_correction(dev, td)+def _cuda_join_is_unsafe(device_indices: Optional[list[int]]) -> bool:+ """Whether a torch row cannot be attached to an nvidia-smi row on this host.++ CUDA enumerates FASTEST_FIRST by default while nvidia-smi reports PCI order, and+ equal-sized cards defeat every other check, so with a NUMERIC mask the join is only+ safe behind ``_cuda_order_matches_smi``. A UUID or MIG mask has no physical ids to+ disagree about: nvidia.py resolves the mask itself and torch enumerates that same+ mask in the same order, so ``visible_ordinal`` joins them whatever the order says.+ """+ if not device_indices:+ return False+ return not _cuda_order_matches_smi()+++def _integrated_cuda_inventory(+ device_indices: Optional[list[int]],+) -> tuple[Dict[Any, Dict[str, Any]], str]:+ """torch's context-free inventory, keyed the way the SMI rows are indexed.++ Returns ``({key: row}, key_field)``, empty when the two sources cannot be joined.++ Two index spaces, and picking the wrong one attaches another card's capacity to a+ row. With a NUMERIC mask the SMI rows carry physical ids, so the join is on+ ``index`` -- but only behind ``_cuda_order_matches_smi``, because CUDA enumerates+ FASTEST_FIRST by default while nvidia-smi reports PCI order, and equal-sized cards+ defeat every other check. Same gate the SMI VRAM query already applies. With a UUID+ or MIG mask there are no physical ids: nvidia.py resolves the mask itself and+ returns rows ordered by it, and torch enumerates that same mask in that same order,+ so ``visible_ordinal`` joins them whatever CUDA_DEVICE_ORDER says.+ """+ if device_indices is None:+ ordinals = list(range(_torch_get_physical_gpu_count() or 0))+ if not ordinals:+ return {}, "visible_ordinal"+ return {+ td["visible_ordinal"]: td for td in _torch_get_device_inventory(ordinals)+ }, "visible_ordinal"+ if _cuda_join_is_unsafe(device_indices):+ # Refusing beats guessing: the caller keeps whatever the CLI reported.+ return {}, "index"+ inventory = _torch_get_device_inventory(+ device_indices if device_indices else list(range(_torch_get_physical_gpu_count() or 0))+ )+ return {td["index"]: td for td in inventory}, "index"+++def _reconcile_cuda_integrated_memory(+ utilization: Dict[str, Any], device_indices: Optional[list[int]]+) -> None:+ """Fill the VRAM columns nvidia-smi leaves at ``[N/A]`` on an integrated CUDA SoC.++ The monitor names a Spark's GB10 and prints "Unknown / 0.00 GiB" beside it (#10691).++ NOT _torch_get_per_device_info, which the ROCm twin above can afford and this cannot:+ this is the /api/system poll, and mem_get_info pins a ~612 MiB primary context for+ the life of the process (test_system_poll_no_cuda_context.py). Both figures here are+ context-free. Host counters are not an approximation of the used half on one shared+ pool, they are the same measurement.++ Writes only what the CLI could not answer, only on a confirmed integrated device.+ """+ missing = [dev for dev in utilization.get("devices", []) if dev.get("vram_total_gb") is None]+ if not missing:+ return+ try:+ inventory, key_field = _integrated_cuda_inventory(device_indices)+ except Exception as e: # noqa: BLE001 - reached on a host that HAS nvidia-smi+ logger.debug("torch inventory unavailable while sizing an integrated GPU: %s", e)+ return+ integrated = {key: td for key, td in inventory.items() if td.get("_cuda_integrated")}+ if not integrated:+ return+ used_gb = None+ try:+ import psutil+ vm = psutil.virtual_memory()+ used_gb = round((int(vm.total) - int(vm.available)) / (1024**3), 2)+ except Exception as e: # noqa: BLE001 - a total alone still beats Unknown / 0.00+ logger.debug("host memory probe failed while sizing an integrated GPU: %s", e)+ for dev in missing:+ td = integrated.get(dev.get(key_field))+ if td is None:+ continue+ total_gb = td["total_gb"]+ dev["vram_total_gb"] = total_gb+ # The CLI's own used figure wins where it has one: memory.used can be readable+ # on a row whose memory.total is [N/A], and filling the total is exactly what+ # makes the percentage computable. Host counters stand in only where it is not.+ pool_used_gb = dev.get("vram_used_gb")+ if pool_used_gb is None:+ if used_gb is None:+ continue+ pool_used_gb = min(used_gb, total_gb)+ dev["vram_used_gb"] = pool_used_gb+ if dev.get("vram_utilization_pct") is None:+ dev["vram_utilization_pct"] = (+ round((min(pool_used_gb, total_gb) / total_gb) * 100, 1)+ if total_gb > 0+ else None+ )++def _reconcile_primary_rocm_unified_memory(utilization: Dict[str, Any], parent_visible_spec: Dict[str, Any]) -> None:@@ -5268,6 +5408,11 @@ def get_visible_gpu_utilization() -> Dict[str, Any]:if IS_ROCM and numeric_ids is not None:# Fix unified-memory VRAM on AMD iGPUs (Strix Halo etc.)._reconcile_rocm_unified_memory(result, numeric_ids)+ elif not IS_ROCM:+ # THIS is what /api/system reads (main.py::_get_cached_system_gpu_info),+ # so the floating monitor's Unknown / 0.00 GiB is repaired here; the twin+ # in get_gpu_utilization serves the training-side callers.+ _reconcile_cuda_integrated_memory(result, numeric_ids)return result# Windows AMD/ROCm (issue #7072): the System tab's VRAM source. The torch@@ -6826,11 +6971,59 @@ def get_vulkan_inference_gpu_info() -> Optional[Dict[str, Any]]:return result+def _repair_smi_visible_devices(+ devices: list[Dict[str, Any]], parent_visible_ids: Optional[list[int]]+) -> bool:+ """Fill in what nvidia-smi could not answer for, from torch's context-free inventory.++ Returns whether every device now carries a capacity.++ nvidia-smi answers ``[N/A]`` for memory.total on a DGX Spark, which NVIDIA documents.+ Keeping that row is right, but a missing total is indistinguishable from no GPU+ downstream: the frontend maps it to zero, the fit classifier returns ``ram``, and the+ picker warns "No GPU detected" on a 121 GiB Blackwell (#10691). ``props.total_memory``+ answers on the same host for no driver context.++ The integrated flag rides along because the reason the capacity is unreadable is that+ this part has no memory of its own, and a total published without it is counted twice.++ A host whose nvidia-smi CAN answer returns above without touching torch, so the 3-5s+ poll is unchanged and a readable part is never reclassified as unified.+ """+ if not devices:+ return False+ if all(dev.get("memory_total_gb") is not None for dev in devices):+ return True+ try:+ inventory, key_field = _integrated_cuda_inventory(parent_visible_ids)+ except Exception as e: # noqa: BLE001 - the caller keeps the rows nvidia-smi found+ logger.debug("torch inventory unavailable while repairing a GPU capacity: %s", e)+ return False+ for dev in devices:+ td = inventory.get(dev.get(key_field))+ if td is None:+ continue+ if dev.get("memory_total_gb") is None:+ dev["memory_total_gb"] = td["total_gb"]+ if td.get("_cuda_integrated"):+ dev["unified_memory"] = True+ # `shared_memory` as well, not just the host-backed figure: gpu-vram.ts+ # splits the pools on THAT flag alone and only then reads the figure, so a+ # row carrying one without the other is added to the dedicated total and+ # counted a second time beside the same system RAM.+ dev["shared_memory"] = True+ dev["shared_memory_host_backed_gb"] = dev["memory_total_gb"]+ return all(dev.get("memory_total_gb") is not None for dev in devices)++def get_backend_visible_gpu_info() -> Dict[str, Any]:device = get_device()if device in (DeviceType.CUDA, DeviceType.XPU):parent_visible_ids = get_parent_visible_gpu_ids()+ # Held back in case torch cannot size them either: an unknown total is a poor+ # answer, but losing the card outright is worse than the pre-repair behaviour.+ unrepaired_smi_result: Optional[Dict[str, Any]] = None# Try native SMI first (nvidia-smi; skipped for ROCm).if device == DeviceType.CUDA and not IS_ROCM:try:@@ -6842,8 +7035,28 @@ def get_backend_visible_gpu_info() -> Dict[str, Any]:parent_visible_spec["raw"],)if result.get("available"):- result["backend"] = _backend_label(device)- return result+ if _repair_smi_visible_devices(+ result.get("devices") or [], parent_visible_spec["numeric_ids"]+ ):+ result["backend"] = _backend_label(device)+ return result+ # Falls through to the torch inventory rather than publishing an+ # unknown the frontend reads as zero -- unless the reason the repair+ # failed is that the two sources cannot be joined at all. The torch+ # fallback labels its rows with physical ids taken from the mask, so+ # under FASTEST_FIRST it would publish one card's name and capacity+ # under another card's index and send GPU selection at the wrong+ # hardware. The SMI rows are coherent; one of them is missing a+ # total, which is the lesser answer and the one this host had before+ # the repair existed.+ if _cuda_join_is_unsafe(parent_visible_spec["numeric_ids"]):+ logger.debug(+ "Keeping the nvidia-smi rows: CUDA_DEVICE_ORDER does not "+ "match nvidia-smi, so no torch row can be trusted here."+ )+ result["backend"] = _backend_label(device)+ return result+ unrepaired_smi_result = resultexcept Exception as e:logger.warning("Backend GPU visibility query failed: %s", e)@@ -6878,18 +7091,35 @@ def get_backend_visible_gpu_info() -> Dict[str, Any]:"visible_ordinal": td["visible_ordinal"],"name": td["name"],"memory_total_gb": td["total_gb"],- "shared_memory": bool(td.get("shared_memory")),- "shared_memory_host_backed_gb": td.get("shared_memory_host_backed_gb"),+ "shared_memory": bool(td.get("shared_memory") or td.get("_cuda_integrated")),+ # An integrated part's total IS host memory; publishing it here is+ # what stops a consumer adding the two pools together.+ "shared_memory_host_backed_gb": (+ td["total_gb"]+ if td.get("_cuda_integrated")+ else td.get("shared_memory_host_backed_gb")+ ),# Surfaced from the inventory's own `_rocm_known_unified`# rather than re-derived: a ROCm APU's total is the GTT pool,# which moves with host usage and is not a VRAM ceiling a fit# verdict can be measured against. Distinct from# `shared_memory`, which is that flag AND Windows, so a Linux# APU reads as not-shared while still having no such ceiling.- "unified_memory": bool(td.get("_rocm_known_unified")),+ "unified_memory": bool(+ td.get("_rocm_known_unified") or td.get("_cuda_integrated")+ ),}for td in torch_devices]+ # A per-device probe failure leaves this list SHORT rather than empty, and+ # a shorter list drops cards nvidia-smi could see. The repair's guarantee is+ # that it never reports fewer devices than before it existed.+ if unrepaired_smi_result is not None and len(devices) < len(+ unrepaired_smi_result.get("devices") or []+ ):+ unrepaired_smi_result["backend"] = _backend_label(device)+ return unrepaired_smi_result+return {"available": True,"backend": _backend_label(device),@@ -6899,6 +7129,12 @@ def get_backend_visible_gpu_info() -> Dict[str, Any]:"index_kind": index_kind,}+ if unrepaired_smi_result is not None:+ # Neither source could size them, but nvidia-smi found them: reporting no GPU+ # for a host that has one is worse than an unknown capacity.+ unrepaired_smi_result["backend"] = _backend_label(device)+ return unrepaired_smi_result+return {"available": False,"backend": _backend_label(device),