Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: recover the GPU capacity nvidia-smi will not report on a DGX Spark (#10703)

Daniel Han0392cb2c65d3
diff --git a/studio/backend/tests/test_dgx_spark_gpu_inventory.py b/studio/backend/tests/test_dgx_spark_gpu_inventory.py
new file mode 100644
index 00000000000..aed3d47eba8
--- /dev/null
+++ b/studio/backend/tests/test_dgx_spark_gpu_inventory.py
@@ -0,0 +1,516 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An integrated CUDA SoC (Jetson, DGX Spark) publishes no capacity through nvidia-smi.
+
+``nvidia-smi --query-gpu=memory.total`` answers ``[N/A]`` on a DGX Spark, which NVIDIA
+documents as a known issue. The row is kept so the card stays in the inventory, but with
+no capacity the frontend maps it to zero, the GGUF fit classifier returns ``ram``, and
+the picker warns "No GPU detected. Runs on system RAM and CPU" on a 121 GiB Blackwell,
+while the live monitor names the GB10 and prints "Unknown / 0.00 GiB" beside it (#10691).
+
+Hermetic: torch, nvidia-smi and host memory are stubbed, so these run anywhere.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import psutil
+import utils.hardware.hardware as hw
+from utils.hardware import nvidia
+
+GIB = 1 << 30
+MIB = 1 << 20
+# What a DGX Spark actually reports.
+SPARK_TOTAL_BYTES = 124609 * MIB
+SPARK_TOTAL_GB = round(SPARK_TOTAL_BYTES / GIB, 2)
+
+
+class _SparkProps:
+ """cudaDeviceProp as torch surfaces it for a GB10."""
+
+ name = "NVIDIA GB10"
+ total_memory = SPARK_TOTAL_BYTES
+ is_integrated = 1
+ gcnArchName = ""
+
+
+class _DiscreteProps:
+ name = "NVIDIA GB200"
+ total_memory = 183 * GIB
+ is_integrated = 0
+ gcnArchName = ""
+
+
+def _torch_module(props) -> types.SimpleNamespace:
+ return types.SimpleNamespace(get_device_properties = lambda ordinal: props)
+
+
+def _not_hip(monkeypatch) -> None:
+ """Stub the torch the CUDA classifier asks for HIP.
+
+ Without this the suite reads the HOST's torch, so every assertion about an integrated
+ CUDA part inverts on a ROCm machine, where ``torch.version.hip`` is set and the
+ classifier correctly declines. Caught on a real gfx1151 runner, not by reading it.
+ """
+ monkeypatch.setitem(
+ sys.modules, "torch", types.SimpleNamespace(version = types.SimpleNamespace(hip = None))
+ )
+
+
+def _cuda_host(monkeypatch, props) -> None:
+ _not_hip(monkeypatch)
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0])
+ monkeypatch.setattr(
+ hw,
+ "_get_parent_visible_gpu_spec",
+ lambda: {"raw": "0", "numeric_ids": [0], "supports_explicit_gpu_ids": True},
+ )
+ monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (_torch_module(props), "cuda"))
+
+
+def _smi_rows(monkeypatch, memory_total_gb) -> None:
+ """Stand in for nvidia.py's parsed inventory, whose [N/A] total is already a None."""
+ monkeypatch.setattr(
+ nvidia,
+ "_query_gpu_inventory",
+ lambda caller: [{"index": 0, "name": "NVIDIA GB10", "memory_total_gb": memory_total_gb}],
+ )
+
+
+def test_spark_recovers_the_capacity_nvidia_smi_will_not_report(monkeypatch):
+ _cuda_host(monkeypatch, _SparkProps())
+ _smi_rows(monkeypatch, None)
+
+ result = hw.get_backend_visible_gpu_info()
+ device = result["devices"][0]
+
+ assert result["available"] is True
+ assert device["name"] == "NVIDIA GB10"
+ # The regression: this was None, which the frontend reads as a 0 GiB card.
+ assert device["memory_total_gb"] == SPARK_TOTAL_GB
+
+
+def test_spark_is_reported_as_one_pool_not_two(monkeypatch):
+ """The recovered total IS system memory, so it must not be added to it."""
+ _cuda_host(monkeypatch, _SparkProps())
+ _smi_rows(monkeypatch, None)
+
+ device = hw.get_backend_visible_gpu_info()["devices"][0]
+
+ assert device["unified_memory"] is True
+ assert device["shared_memory_host_backed_gb"] == SPARK_TOTAL_GB
+
+
+def test_readable_smi_capacity_is_kept_as_is(monkeypatch):
+ """A discrete card answers memory.total, and nothing here may second-guess it."""
+ _cuda_host(monkeypatch, _DiscreteProps())
+ _smi_rows(monkeypatch, 23.99)
+
+ device = hw.get_backend_visible_gpu_info()["devices"][0]
+
+ assert device["memory_total_gb"] == 23.99
+ assert device.get("unified_memory") is not True
+
+
+def test_torch_fallback_also_flags_the_integrated_pool(monkeypatch):
+ """With nvidia-smi missing entirely the inventory comes from torch; same verdict."""
+ _cuda_host(monkeypatch, _SparkProps())
+ monkeypatch.setattr(nvidia, "_query_gpu_inventory", lambda caller: None)
+
+ device = hw.get_backend_visible_gpu_info()["devices"][0]
+
+ assert device["memory_total_gb"] == SPARK_TOTAL_GB
+ assert device["unified_memory"] is True
+ assert device["shared_memory_host_backed_gb"] == SPARK_TOTAL_GB
+
+
+def test_rocm_is_not_classified_by_the_cuda_integrated_flag(monkeypatch):
+ """HIP left that field unassigned before 6.2, so reading it there calls a card unified."""
+ props = _SparkProps()
+ monkeypatch.setattr(hw, "IS_ROCM", True)
+
+ assert hw._cuda_props_are_integrated(props) is False
+
+
+# ── nothing above may reach a host that is not one of these parts ────────────
+
+
+def test_a_readable_smi_host_never_consults_torch(monkeypatch):
+ """The repair is scoped to hosts with a missing capacity.
+
+ /api/system is polled every 5s by the floating monitor and every 3s from Settings.
+ A discrete card answers memory.total, so it must leave that poll exactly as it was:
+ no torch import, no property query, no new failure mode on a machine that was fine.
+ """
+ _cuda_host(monkeypatch, _DiscreteProps())
+ _smi_rows(monkeypatch, 23.99)
+
+ def _forbidden(*args, **kwargs):
+ raise AssertionError("torch inventory consulted on a host with nothing to repair")
+
+ monkeypatch.setattr(hw, "_torch_get_device_inventory", _forbidden)
+
+ assert hw.get_backend_visible_gpu_info()["devices"][0]["memory_total_gb"] == 23.99
+
+
+def test_multi_gpu_smi_host_is_untouched(monkeypatch):
+ """Two discrete cards, both readable: same rows out as in."""
+ _cuda_host(monkeypatch, _DiscreteProps())
+ monkeypatch.setattr(
+ hw,
+ "_get_parent_visible_gpu_spec",
+ lambda: {"raw": "0,1", "numeric_ids": [0, 1], "supports_explicit_gpu_ids": True},
+ )
+ monkeypatch.setattr(
+ nvidia,
+ "_query_gpu_inventory",
+ lambda caller: [
+ {"index": 0, "name": "NVIDIA H100", "memory_total_gb": 79.65},
+ {"index": 1, "name": "NVIDIA H100", "memory_total_gb": 79.65},
+ ],
+ )
+
+ devices = hw.get_backend_visible_gpu_info()["devices"]
+
+ assert [d["memory_total_gb"] for d in devices] == [79.65, 79.65]
+ assert not any(d.get("unified_memory") for d in devices)
+ assert not any(d.get("shared_memory_host_backed_gb") for d in devices)
+
+
+def test_xpu_is_not_classified_by_the_cuda_integrated_flag(monkeypatch):
+ """A same-named field on a future Intel wheel must not rewrite an iGPU's capacity."""
+ _not_hip(monkeypatch)
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ assert hw._cuda_props_are_integrated(_SparkProps(), "xpu") is False
+ assert hw._cuda_props_are_integrated(_SparkProps(), None) is False
+ assert hw._cuda_props_are_integrated(_SparkProps(), "cuda") is True
+
+
+def test_an_unsizeable_card_is_still_reported_when_torch_cannot_answer(monkeypatch):
+ """Both sources blank: keep the row nvidia-smi found rather than claim no GPU."""
+ _cuda_host(monkeypatch, _SparkProps())
+ _smi_rows(monkeypatch, None)
+ monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (None, None))
+
+ result = hw.get_backend_visible_gpu_info()
+
+ assert result["available"] is True
+ assert result["devices"][0]["name"] == "NVIDIA GB10"
+ assert result["devices"][0]["memory_total_gb"] is None
+
+
+# ── the live monitor ─────────────────────────────────────────────────────────
+
+
+def _smi_utilization(
+ monkeypatch,
+ vram_total_gb,
+ vram_used_gb = None,
+) -> dict:
+ """nvidia-smi's utilization rows, whose [N/A] memory columns are already None."""
+ payload = {
+ "available": True,
+ "devices": [
+ {
+ "index": 0,
+ "index_kind": "physical",
+ "visible_ordinal": 0,
+ "gpu_utilization_pct": 0.0,
+ "temperature_c": 46.0,
+ "vram_used_gb": vram_used_gb,
+ "vram_total_gb": vram_total_gb,
+ "vram_utilization_pct": None,
+ "power_draw_w": 12.09,
+ "power_limit_w": None,
+ "power_utilization_pct": None,
+ }
+ ],
+ "backend_cuda_visible_devices": "0",
+ "parent_visible_gpu_ids": [0],
+ "index_kind": "physical",
+ }
+ monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: payload)
+ return payload
+
+
+def test_monitor_sizes_the_spark_instead_of_showing_unknown(monkeypatch):
+ _cuda_host(monkeypatch, _SparkProps())
+ _smi_utilization(monkeypatch, vram_total_gb = None)
+ import psutil
+
+ monkeypatch.setattr(
+ psutil,
+ "virtual_memory",
+ lambda: types.SimpleNamespace(total = 121 * GIB, available = 100 * GIB),
+ )
+
+ device = hw.get_gpu_utilization()["devices"][0]
+
+ assert device["vram_total_gb"] == SPARK_TOTAL_GB
+ assert device["vram_used_gb"] == 21.0
+ assert device["vram_utilization_pct"] == round(21.0 / SPARK_TOTAL_GB * 100, 1)
+ # Columns nvidia-smi DID answer are the CLI's, untouched.
+ assert device["gpu_utilization_pct"] == 0.0
+ assert device["temperature_c"] == 46.0
+ assert device["power_draw_w"] == 12.09
+
+
+def test_monitor_leaves_a_readable_card_alone(monkeypatch):
+ """A discrete card answers both memory columns, so nothing here may run."""
+ _cuda_host(monkeypatch, _DiscreteProps())
+ _smi_utilization(monkeypatch, vram_total_gb = 79.65, vram_used_gb = 12.0)
+
+ def _forbidden(*args, **kwargs):
+ raise AssertionError("torch inventory consulted for a card nvidia-smi could size")
+
+ monkeypatch.setattr(hw, "_torch_get_device_inventory", _forbidden)
+
+ device = hw.get_gpu_utilization()["devices"][0]
+
+ assert (device["vram_total_gb"], device["vram_used_gb"]) == (79.65, 12.0)
+
+
+def test_monitor_does_not_size_a_discrete_card_smi_could_not_read(monkeypatch):
+ """An unreadable DISCRETE card keeps its unknown: host RAM is not its VRAM."""
+ _cuda_host(monkeypatch, _DiscreteProps())
+ _smi_utilization(monkeypatch, vram_total_gb = None)
+
+ device = hw.get_gpu_utilization()["devices"][0]
+
+ assert device["vram_total_gb"] is None
+ assert device["vram_used_gb"] is None
+
+
+def test_monitor_reconciliation_creates_no_driver_context(monkeypatch):
+ """The poll runs every 3-5s; mem_get_info would pin ~612 MiB for the process life."""
+ _cuda_host(monkeypatch, _SparkProps())
+ _smi_utilization(monkeypatch, vram_total_gb = None)
+
+ def _forbidden(*args, **kwargs):
+ raise AssertionError("the monitor poll reached the occupancy probe")
+
+ monkeypatch.setattr(hw, "_torch_get_per_device_info", _forbidden)
+
+ assert hw.get_gpu_utilization()["devices"][0]["vram_total_gb"] == SPARK_TOTAL_GB
+
+
+def test_a_hip_torch_is_never_read_with_the_cuda_rule(monkeypatch):
+ """HIP reuses this namespace and a real APU sets the same flag.
+
+ IS_ROCM is a global that detection publishes, so the classifier also asks torch
+ directly: this covers the window before detection has settled, and the ROCm CI runner
+ where the previous revision of these tests inverted.
+ """
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setitem(
+ sys.modules, "torch", types.SimpleNamespace(version = types.SimpleNamespace(hip = "6.2.0"))
+ )
+
+ assert hw._cuda_props_are_integrated(_SparkProps(), "cuda") is False
+
+
+def _smi_visible_utilization(monkeypatch, rows) -> dict:
+ payload = {
+ "available": True,
+ "devices": rows,
+ "backend_cuda_visible_devices": "0",
+ "parent_visible_gpu_ids": [0],
+ "index_kind": "physical",
+ }
+ monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: payload)
+ return payload
+
+
+def _unsized_row(
+ index = 0,
+ ordinal = 0,
+ index_kind = "physical",
+) -> dict:
+ return {
+ "index": index,
+ "index_kind": index_kind,
+ "visible_ordinal": ordinal,
+ "gpu_utilization_pct": 0.0,
+ "temperature_c": 46.0,
+ "vram_used_gb": None,
+ "vram_total_gb": None,
+ "vram_utilization_pct": None,
+ "power_draw_w": 12.09,
+ "power_limit_w": None,
+ "power_utilization_pct": None,
+ }
+
+
+def test_the_system_poll_function_is_the_one_that_gets_repaired(monkeypatch):
+ """/api/system reads get_visible_gpu_utilization, NOT get_gpu_utilization.
+
+ main.py::_get_cached_system_gpu_info calls the former, so repairing only the latter
+ left the floating monitor showing Unknown / 0.00 GiB, which is the screen #10691 is
+ actually about.
+ """
+ _cuda_host(monkeypatch, _SparkProps())
+ _smi_visible_utilization(monkeypatch, [_unsized_row()])
+ monkeypatch.setattr(
+ psutil,
+ "virtual_memory",
+ lambda: types.SimpleNamespace(total = 121 * GIB, available = 100 * GIB),
+ )
+
+ device = hw.get_visible_gpu_utilization()["devices"][0]
+
+ assert device["vram_total_gb"] == SPARK_TOTAL_GB
+ assert device["vram_used_gb"] == 21.0
+
+
+def test_a_uuid_mask_still_reaches_the_reconciliation(monkeypatch):
+ """A UUID or MIG mask resolves to numeric_ids=None, and nvidia.py answers anyway.
+
+ Its rows are relative-indexed and ordered by the mask, which is the order torch
+ enumerates too, so the join is on visible_ordinal rather than a physical id.
+ """
+ _cuda_host(monkeypatch, _SparkProps())
+ monkeypatch.setattr(
+ hw,
+ "_get_parent_visible_gpu_spec",
+ lambda: {"raw": "GPU-9254a6cb", "numeric_ids": None, "supports_explicit_gpu_ids": False},
+ )
+ monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1)
+ _smi_visible_utilization(monkeypatch, [_unsized_row(index = 0, index_kind = "relative")])
+
+ device = hw.get_visible_gpu_utilization()["devices"][0]
+
+ assert device["vram_total_gb"] == SPARK_TOTAL_GB
+
+
+def test_a_mismatched_device_order_refuses_the_join(monkeypatch):
+ """FASTEST_FIRST puts torch ordinals in a different space from nvidia-smi rows.
+
+ Joining them anyway attaches another card's capacity to a row. The module already
+ rejects this exact cross-source mapping for its SMI VRAM query; the repair uses the
+ same gate rather than guessing.
+ """
+ _cuda_host(monkeypatch, _DiscreteProps())
+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: False)
+ _smi_rows(monkeypatch, None)
+
+ # The resolver refuses outright rather than handing back a mapping to guess with.
+ assert hw._integrated_cuda_inventory([0]) == ({}, "index")
+
+ # ...and the endpoint keeps the nvidia-smi rows rather than reaching the torch
+ # fallback. That fallback labels its rows with physical ids taken from the mask, so
+ # here it would publish one card's name and capacity under another card's index,
+ # which is the same wrong join by another route and is what GPU selection reads.
+ # An SMI row missing a total is the lesser answer, and the one this host had before
+ # the repair existed.
+ device = hw.get_backend_visible_gpu_info()["devices"][0]
+ assert device["name"] == "NVIDIA GB10"
+ assert device["memory_total_gb"] is None
+ assert device.get("unified_memory") is not True
+
+
+def test_a_partial_torch_inventory_never_drops_an_smi_card(monkeypatch):
+ """_torch_get_device_inventory skips a device whose probe failed, so its list can
+ be SHORT rather than empty. A short list must not replace the cards nvidia-smi saw.
+ """
+ _cuda_host(monkeypatch, _SparkProps())
+ monkeypatch.setattr(
+ hw,
+ "_get_parent_visible_gpu_spec",
+ lambda: {"raw": "0,1", "numeric_ids": [0, 1], "supports_explicit_gpu_ids": True},
+ )
+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0, 1])
+ monkeypatch.setattr(
+ nvidia,
+ "_query_gpu_inventory",
+ lambda caller: [
+ {"index": 0, "name": "NVIDIA GB10", "memory_total_gb": None},
+ {"index": 1, "name": "NVIDIA GB10", "memory_total_gb": None},
+ ],
+ )
+ # Only ordinal 0 answers, the shape a fallen-off-the-bus card produces.
+ monkeypatch.setattr(
+ hw,
+ "_torch_get_device_inventory",
+ lambda indices: [
+ {
+ "index": 0,
+ "visible_ordinal": 0,
+ "name": "NVIDIA GB10",
+ "total_gb": SPARK_TOTAL_GB,
+ "used_gb": None,
+ "shared_memory": False,
+ "shared_memory_host_backed_gb": None,
+ "_rocm_known_unified": False,
+ "_rocm_gfx": "",
+ "_cuda_integrated": True,
+ }
+ ],
+ )
+
+ result = hw.get_backend_visible_gpu_info()
+
+ assert len(result["devices"]) == 2
+ assert [d["name"] for d in result["devices"]] == ["NVIDIA GB10", "NVIDIA GB10"]
+
+
+def test_a_repaired_spark_row_is_marked_shared(monkeypatch):
+ """gpu-vram.ts splits the dedicated and shared pools on `shared_memory` alone.
+
+ The host-backed figure is read only after that split, so a repaired row that carries
+ the figure without the flag lands in the dedicated total and is then counted a
+ second time beside the same system RAM, which is the double count this repair
+ exists to prevent.
+ """
+ _cuda_host(monkeypatch, _SparkProps())
+ _smi_rows(monkeypatch, None)
+
+ device = hw.get_backend_visible_gpu_info()["devices"][0]
+
+ assert device["shared_memory"] is True
+ assert device["unified_memory"] is True
+ assert device["shared_memory_host_backed_gb"] == SPARK_TOTAL_GB
+
+
+def test_a_discrete_card_is_never_marked_shared(monkeypatch):
+ """The flag follows the integrated property, not the repair."""
+ _cuda_host(monkeypatch, _DiscreteProps())
+ _smi_rows(monkeypatch, None)
+
+ device = hw.get_backend_visible_gpu_info()["devices"][0]
+
+ assert device.get("shared_memory") is not True
+ assert device.get("unified_memory") is not True
+
+
+def test_a_known_usage_still_gets_its_percentage(monkeypatch):
+ """memory.used can be readable on a row whose memory.total is [N/A].
+
+ Filling the total is what makes the percentage computable, so a device that already
+ carries a usage must not be skipped: it was the one case where both operands existed
+ and the monitor still showed an unknown.
+ """
+ _cuda_host(monkeypatch, _SparkProps())
+ utilization = {
+ "devices": [
+ {
+ "index": 0,
+ "visible_ordinal": 0,
+ "vram_total_gb": None,
+ "vram_used_gb": 30.0,
+ "vram_utilization_pct": None,
+ }
+ ]
+ }
+
+ hw._reconcile_cuda_integrated_memory(utilization, [0])
+ device = utilization["devices"][0]
+
+ assert device["vram_total_gb"] == SPARK_TOTAL_GB
+ # The CLI's own figure, not the host counter that stands in when it is absent.
+ assert device["vram_used_gb"] == 30.0
+ assert device["vram_utilization_pct"] == round((30.0 / SPARK_TOTAL_GB) * 100, 1)
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index e45e9552f44..4633021235a 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -2880,9 +2880,17 @@ def get_package_versions() -> Dict[str, Optional[str]]:
def _torch_get_device_module():
- """Return the appropriate torch device module (cuda or xpu) and its name."""
+ """Return the appropriate torch device module (cuda or xpu) and its name.
+
+ No torch at all answers ``(None, None)`` like an unsupported device: raising took the
+ exception out through /api/system on a host whose vendor CLI HAD found GPUs.
+ """
device = get_device()
- import torch
+ try:
+ import torch
+ except Exception as e: # noqa: BLE001 - no torch is an answer, not a fault
+ logger.debug("torch is not importable: %s", e)
+ return None, None
if device == DeviceType.CUDA:
return torch.cuda, "cuda"
@@ -3084,6 +3092,31 @@ def _rocm_props_total_is_carve_out(props: Any) -> bool:
)
+def _cuda_props_are_integrated(props: Any, backend: Optional[str] = "cuda") -> bool:
+ """Whether ``props`` describes an integrated CUDA part whose VRAM is system RAM.
+
+ Jetson and DGX Spark class parts set ``cudaDeviceProp::integrated`` (torch's
+ ``is_integrated``, ``integrated`` on older wheels).
+
+ ROCm and XPU are excluded BY NAME, not by trusting the field to be absent: HIP reuses
+ this namespace and left that field unassigned before 6.2, so reading it would call a
+ discrete card integrated on exactly the runtimes ``_HIP_INTEGRATED_FLAG_MIN``
+ distrusts (``_rocm_props_unified_status`` is the classifier for that hardware), and a
+ same-named field on a future Intel wheel must not start rewriting an iGPU's capacity.
+ ``torch.version.hip`` as well as ``IS_ROCM`` because that global is published by
+ detection, and this inventory is reachable from inside detection.
+ """
+ if IS_ROCM or backend != "cuda":
+ return False
+ try:
+ import torch
+ if getattr(torch.version, "hip", None):
+ return False
+ except Exception as e: # noqa: BLE001 - no torch to ask: IS_ROCM alone then
+ logger.debug("HIP probe failed while classifying an integrated device: %s", e)
+ return bool(getattr(props, "is_integrated", False) or getattr(props, "integrated", False))
+
+
def _torch_get_device_inventory(device_indices: list[int]) -> list[Dict[str, Any]]:
"""Per-GPU name and total VRAM only, without creating a driver context.
@@ -3099,7 +3132,7 @@ def _torch_get_device_inventory(device_indices: list[int]) -> list[Dict[str, Any
totals are unchanged. ``used_gb`` is always None, the value this module already
uses for "telemetry unavailable".
"""
- mod, _ = _torch_get_device_module()
+ mod, backend = _torch_get_device_module()
if mod is None:
return []
@@ -3153,6 +3186,7 @@ def _torch_get_device_inventory(device_indices: list[int]) -> list[Dict[str, Any
),
"_rocm_known_unified": known_unified,
"_rocm_gfx": rocm_gfx,
+ "_cuda_integrated": _cuda_props_are_integrated(props, backend),
}
)
except Exception as e:
@@ -4856,6 +4890,10 @@ def get_gpu_utilization() -> Dict[str, Any]:
numeric_ids = parent_visible_spec.get("numeric_ids")
if IS_ROCM and numeric_ids is not None:
_reconcile_rocm_unified_memory(result, numeric_ids)
+ elif not IS_ROCM:
+ # numeric_ids is None under a UUID/MIG mask, which nvidia.py resolves
+ # itself and answers with relative-indexed rows: still repairable.
+ _reconcile_cuda_integrated_memory(result, numeric_ids)
return _gpu_utilization_payload(
device,
@@ -5077,6 +5115,108 @@ def _reconcile_rocm_unified_memory(utilization: Dict[str, Any], device_indices:
_apply_unified_memory_correction(dev, td)
+def _cuda_join_is_unsafe(device_indices: Optional[list[int]]) -> bool:
+ """Whether a torch row cannot be attached to an nvidia-smi row on this host.
+
+ CUDA enumerates FASTEST_FIRST by default while nvidia-smi reports PCI order, and
+ equal-sized cards defeat every other check, so with a NUMERIC mask the join is only
+ safe behind ``_cuda_order_matches_smi``. A UUID or MIG mask has no physical ids to
+ disagree about: nvidia.py resolves the mask itself and torch enumerates that same
+ mask in the same order, so ``visible_ordinal`` joins them whatever the order says.
+ """
+ if not device_indices:
+ return False
+ return not _cuda_order_matches_smi()
+
+
+def _integrated_cuda_inventory(
+ device_indices: Optional[list[int]],
+) -> tuple[Dict[Any, Dict[str, Any]], str]:
+ """torch's context-free inventory, keyed the way the SMI rows are indexed.
+
+ Returns ``({key: row}, key_field)``, empty when the two sources cannot be joined.
+
+ Two index spaces, and picking the wrong one attaches another card's capacity to a
+ row. With a NUMERIC mask the SMI rows carry physical ids, so the join is on
+ ``index`` -- but only behind ``_cuda_order_matches_smi``, because CUDA enumerates
+ FASTEST_FIRST by default while nvidia-smi reports PCI order, and equal-sized cards
+ defeat every other check. Same gate the SMI VRAM query already applies. With a UUID
+ or MIG mask there are no physical ids: nvidia.py resolves the mask itself and
+ returns rows ordered by it, and torch enumerates that same mask in that same order,
+ so ``visible_ordinal`` joins them whatever CUDA_DEVICE_ORDER says.
+ """
+ if device_indices is None:
+ ordinals = list(range(_torch_get_physical_gpu_count() or 0))
+ if not ordinals:
+ return {}, "visible_ordinal"
+ return {
+ td["visible_ordinal"]: td for td in _torch_get_device_inventory(ordinals)
+ }, "visible_ordinal"
+ if _cuda_join_is_unsafe(device_indices):
+ # Refusing beats guessing: the caller keeps whatever the CLI reported.
+ return {}, "index"
+ inventory = _torch_get_device_inventory(
+ device_indices if device_indices else list(range(_torch_get_physical_gpu_count() or 0))
+ )
+ return {td["index"]: td for td in inventory}, "index"
+
+
+def _reconcile_cuda_integrated_memory(
+ utilization: Dict[str, Any], device_indices: Optional[list[int]]
+) -> None:
+ """Fill the VRAM columns nvidia-smi leaves at ``[N/A]`` on an integrated CUDA SoC.
+
+ The monitor names a Spark's GB10 and prints "Unknown / 0.00 GiB" beside it (#10691).
+
+ NOT _torch_get_per_device_info, which the ROCm twin above can afford and this cannot:
+ this is the /api/system poll, and mem_get_info pins a ~612 MiB primary context for
+ the life of the process (test_system_poll_no_cuda_context.py). Both figures here are
+ context-free. Host counters are not an approximation of the used half on one shared
+ pool, they are the same measurement.
+
+ Writes only what the CLI could not answer, only on a confirmed integrated device.
+ """
+ missing = [dev for dev in utilization.get("devices", []) if dev.get("vram_total_gb") is None]
+ if not missing:
+ return
+ try:
+ inventory, key_field = _integrated_cuda_inventory(device_indices)
+ except Exception as e: # noqa: BLE001 - reached on a host that HAS nvidia-smi
+ logger.debug("torch inventory unavailable while sizing an integrated GPU: %s", e)
+ return
+ integrated = {key: td for key, td in inventory.items() if td.get("_cuda_integrated")}
+ if not integrated:
+ return
+ used_gb = None
+ try:
+ import psutil
+ vm = psutil.virtual_memory()
+ used_gb = round((int(vm.total) - int(vm.available)) / (1024**3), 2)
+ except Exception as e: # noqa: BLE001 - a total alone still beats Unknown / 0.00
+ logger.debug("host memory probe failed while sizing an integrated GPU: %s", e)
+ for dev in missing:
+ td = integrated.get(dev.get(key_field))
+ if td is None:
+ continue
+ total_gb = td["total_gb"]
+ dev["vram_total_gb"] = total_gb
+ # The CLI's own used figure wins where it has one: memory.used can be readable
+ # on a row whose memory.total is [N/A], and filling the total is exactly what
+ # makes the percentage computable. Host counters stand in only where it is not.
+ pool_used_gb = dev.get("vram_used_gb")
+ if pool_used_gb is None:
+ if used_gb is None:
+ continue
+ pool_used_gb = min(used_gb, total_gb)
+ dev["vram_used_gb"] = pool_used_gb
+ if dev.get("vram_utilization_pct") is None:
+ dev["vram_utilization_pct"] = (
+ round((min(pool_used_gb, total_gb) / total_gb) * 100, 1)
+ if total_gb > 0
+ else None
+ )
+
+
def _reconcile_primary_rocm_unified_memory(
utilization: Dict[str, Any], parent_visible_spec: Dict[str, Any]
) -> None:
@@ -5268,6 +5408,11 @@ def get_visible_gpu_utilization() -> Dict[str, Any]:
if IS_ROCM and numeric_ids is not None:
# Fix unified-memory VRAM on AMD iGPUs (Strix Halo etc.).
_reconcile_rocm_unified_memory(result, numeric_ids)
+ elif not IS_ROCM:
+ # THIS is what /api/system reads (main.py::_get_cached_system_gpu_info),
+ # so the floating monitor's Unknown / 0.00 GiB is repaired here; the twin
+ # in get_gpu_utilization serves the training-side callers.
+ _reconcile_cuda_integrated_memory(result, numeric_ids)
return result
# Windows AMD/ROCm (issue #7072): the System tab's VRAM source. The torch
@@ -6826,11 +6971,59 @@ def get_vulkan_inference_gpu_info() -> Optional[Dict[str, Any]]:
return result
+def _repair_smi_visible_devices(
+ devices: list[Dict[str, Any]], parent_visible_ids: Optional[list[int]]
+) -> bool:
+ """Fill in what nvidia-smi could not answer for, from torch's context-free inventory.
+
+ Returns whether every device now carries a capacity.
+
+ nvidia-smi answers ``[N/A]`` for memory.total on a DGX Spark, which NVIDIA documents.
+ Keeping that row is right, but a missing total is indistinguishable from no GPU
+ downstream: the frontend maps it to zero, the fit classifier returns ``ram``, and the
+ picker warns "No GPU detected" on a 121 GiB Blackwell (#10691). ``props.total_memory``
+ answers on the same host for no driver context.
+
+ The integrated flag rides along because the reason the capacity is unreadable is that
+ this part has no memory of its own, and a total published without it is counted twice.
+
+ A host whose nvidia-smi CAN answer returns above without touching torch, so the 3-5s
+ poll is unchanged and a readable part is never reclassified as unified.
+ """
+ if not devices:
+ return False
+ if all(dev.get("memory_total_gb") is not None for dev in devices):
+ return True
+ try:
+ inventory, key_field = _integrated_cuda_inventory(parent_visible_ids)
+ except Exception as e: # noqa: BLE001 - the caller keeps the rows nvidia-smi found
+ logger.debug("torch inventory unavailable while repairing a GPU capacity: %s", e)
+ return False
+ for dev in devices:
+ td = inventory.get(dev.get(key_field))
+ if td is None:
+ continue
+ if dev.get("memory_total_gb") is None:
+ dev["memory_total_gb"] = td["total_gb"]
+ if td.get("_cuda_integrated"):
+ dev["unified_memory"] = True
+ # `shared_memory` as well, not just the host-backed figure: gpu-vram.ts
+ # splits the pools on THAT flag alone and only then reads the figure, so a
+ # row carrying one without the other is added to the dedicated total and
+ # counted a second time beside the same system RAM.
+ dev["shared_memory"] = True
+ dev["shared_memory_host_backed_gb"] = dev["memory_total_gb"]
+ return all(dev.get("memory_total_gb") is not None for dev in devices)
+
+
def get_backend_visible_gpu_info() -> Dict[str, Any]:
device = get_device()
if device in (DeviceType.CUDA, DeviceType.XPU):
parent_visible_ids = get_parent_visible_gpu_ids()
+ # Held back in case torch cannot size them either: an unknown total is a poor
+ # answer, but losing the card outright is worse than the pre-repair behaviour.
+ unrepaired_smi_result: Optional[Dict[str, Any]] = None
# Try native SMI first (nvidia-smi; skipped for ROCm).
if device == DeviceType.CUDA and not IS_ROCM:
try:
@@ -6842,8 +7035,28 @@ def get_backend_visible_gpu_info() -> Dict[str, Any]:
parent_visible_spec["raw"],
)
if result.get("available"):
- result["backend"] = _backend_label(device)
- return result
+ if _repair_smi_visible_devices(
+ result.get("devices") or [], parent_visible_spec["numeric_ids"]
+ ):
+ result["backend"] = _backend_label(device)
+ return result
+ # Falls through to the torch inventory rather than publishing an
+ # unknown the frontend reads as zero -- unless the reason the repair
+ # failed is that the two sources cannot be joined at all. The torch
+ # fallback labels its rows with physical ids taken from the mask, so
+ # under FASTEST_FIRST it would publish one card's name and capacity
+ # under another card's index and send GPU selection at the wrong
+ # hardware. The SMI rows are coherent; one of them is missing a
+ # total, which is the lesser answer and the one this host had before
+ # the repair existed.
+ if _cuda_join_is_unsafe(parent_visible_spec["numeric_ids"]):
+ logger.debug(
+ "Keeping the nvidia-smi rows: CUDA_DEVICE_ORDER does not "
+ "match nvidia-smi, so no torch row can be trusted here."
+ )
+ result["backend"] = _backend_label(device)
+ return result
+ unrepaired_smi_result = result
except Exception as e:
logger.warning("Backend GPU visibility query failed: %s", e)
@@ -6878,18 +7091,35 @@ def get_backend_visible_gpu_info() -> Dict[str, Any]:
"visible_ordinal": td["visible_ordinal"],
"name": td["name"],
"memory_total_gb": td["total_gb"],
- "shared_memory": bool(td.get("shared_memory")),
- "shared_memory_host_backed_gb": td.get("shared_memory_host_backed_gb"),
+ "shared_memory": bool(td.get("shared_memory") or td.get("_cuda_integrated")),
+ # An integrated part's total IS host memory; publishing it here is
+ # what stops a consumer adding the two pools together.
+ "shared_memory_host_backed_gb": (
+ td["total_gb"]
+ if td.get("_cuda_integrated")
+ else td.get("shared_memory_host_backed_gb")
+ ),
# Surfaced from the inventory's own `_rocm_known_unified`
# rather than re-derived: a ROCm APU's total is the GTT pool,
# which moves with host usage and is not a VRAM ceiling a fit
# verdict can be measured against. Distinct from
# `shared_memory`, which is that flag AND Windows, so a Linux
# APU reads as not-shared while still having no such ceiling.
- "unified_memory": bool(td.get("_rocm_known_unified")),
+ "unified_memory": bool(
+ td.get("_rocm_known_unified") or td.get("_cuda_integrated")
+ ),
}
for td in torch_devices
]
+ # A per-device probe failure leaves this list SHORT rather than empty, and
+ # a shorter list drops cards nvidia-smi could see. The repair's guarantee is
+ # that it never reports fewer devices than before it existed.
+ if unrepaired_smi_result is not None and len(devices) < len(
+ unrepaired_smi_result.get("devices") or []
+ ):
+ unrepaired_smi_result["backend"] = _backend_label(device)
+ return unrepaired_smi_result
+
return {
"available": True,
"backend": _backend_label(device),
@@ -6899,6 +7129,12 @@ def get_backend_visible_gpu_info() -> Dict[str, Any]:
"index_kind": index_kind,
}
+ if unrepaired_smi_result is not None:
+ # Neither source could size them, but nvidia-smi found them: reporting no GPU
+ # for a host that has one is worse than an unknown capacity.
+ unrepaired_smi_result["backend"] = _backend_label(device)
+ return unrepaired_smi_result
+
return {
"available": False,
"backend": _backend_label(device),