Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""The /api/system poll must not pin a CUDA/HIP primary context, which holds ~612 MiB until exit."""
from __future__ import annotations
import importlib
import os
import subprocess
import sys
import textwrap
import types
from pathlib import Path
def _shared_setup_1(monkeypatch):
monkeypatch.setattr(
hw,
"_get_parent_visible_gpu_spec",
lambda: {"raw": None, "numeric_ids": [0, 1], "supports_explicit_gpu_ids": True},
)
def _shared_setup_2(monkeypatch):
mod = _apu_mod(gtt_total = _APU_GTT_TOTAL, carve_out = _APU_GTT_TOTAL)
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0])
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
def _shared_setup_3(monkeypatch):
gib = 1 << 30
torch_stub, _props = _summary_torch(
properties_total = 8 * gib,
driver_free = 98 * gib,
driver_total = 100 * gib,
)
monkeypatch.setitem(sys.modules, "torch", torch_stub)
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "_rocm_props_total_is_carve_out", lambda _props: True)
return _props, gib
def _shared_setup_4(monkeypatch):
monkeypatch.setattr(
hw,
"_get_parent_visible_gpu_spec",
lambda: {"raw": None, "numeric_ids": [0], "supports_explicit_gpu_ids": True},
)
def _shared_setup_5(monkeypatch):
for name in (
"GPU_DEVICE_ORDINAL",
"ROCR_VISIBLE_DEVICES",
"HIP_VISIBLE_DEVICES",
"CUDA_VISIBLE_DEVICES",
):
monkeypatch.delenv(name, raising = False)
def _shared_setup_6(mod, monkeypatch):
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "_hip_runtime_version", lambda: (6, 4))
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
inventory = hw._torch_get_device_inventory([0])[0]
return inventory
def _shared_setup_7(monkeypatch):
from utils.hardware import amd
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw.platform, "system", lambda: "Darwin")
return amd
_BACKEND_DIR = Path(__file__).resolve().parent.parent
if str(_BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(_BACKEND_DIR))
def _maybe_stub(name: str, builder):
# Stub only if the real module is missing, so we never shadow it for later tests.
try:
importlib.import_module(name)
except ImportError:
sys.modules[name] = builder()
def _build_loggers_stub():
m = types.ModuleType("loggers")
m.get_logger = lambda name: __import__("logging").getLogger(name)
return m
def _build_structlog_stub():
m = types.ModuleType("structlog")
m.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
return m
_maybe_stub("loggers", _build_loggers_stub)
_maybe_stub("structlog", _build_structlog_stub)
import pytest
import utils.hardware.hardware as hw # noqa: E402
def _has_cuda() -> bool:
try:
import torch
except Exception:
return False
try:
return bool(torch.cuda.is_available() and torch.cuda.device_count() > 0)
except Exception:
return False
needs_cuda = pytest.mark.skipif(
not _has_cuda(), reason = "needs a real CUDA/HIP device to observe context creation"
)
# memory_reserved(i) reads allocator stats and needs no context, so it is a safe probe.
_CHILD = textwrap.dedent(
"""
import sys
sys.path.insert(0, {backend!r})
import torch
assert sum(torch.cuda.memory_reserved(i) for i in range(torch.cuda.device_count())) == 0
import utils.hardware.hardware as hw
{call}
leaked = [i for i in range(torch.cuda.device_count()) if torch.cuda.memory_reserved(i)]
print("LEAKED:" + repr(leaked))
print("CONTEXT:" + repr(_probe_context()))
"""
)
# Per-PID nvidia-smi is the ground truth: a primary context shows as held memory.
_PROBE = textwrap.dedent(
"""
import os, subprocess
def _probe_context():
try:
out = subprocess.run(
["nvidia-smi", "--query-compute-apps=pid,used_memory",
"--format=csv,noheader,nounits"],
capture_output = True, text = True, timeout = 60,
)
except Exception:
return None
if out.returncode != 0:
return None
me = str(os.getpid())
for line in out.stdout.splitlines():
parts = [p.strip() for p in line.split(",")]
if len(parts) >= 2 and parts[0] == me:
return int(parts[1])
return 0
"""
)
def _run_child(call: str):
"""Run ``call`` in a fresh interpreter; return (leaked_devices, mib_or_None)."""
src = _PROBE + _CHILD.format(backend = str(_BACKEND_DIR), call = call)
proc = subprocess.run([sys.executable, "-c", src], capture_output = True, text = True, timeout = 600)
assert proc.returncode == 0, f"child failed:\n{proc.stdout}\n{proc.stderr}"
leaked, mib = None, None
for line in proc.stdout.splitlines():
if line.startswith("LEAKED:"):
leaked = eval(line[len("LEAKED:") :])
elif line.startswith("CONTEXT:"):
mib = eval(line[len("CONTEXT:") :])
assert leaked is not None, proc.stdout
return leaked, mib
@needs_cuda
def test_backend_visible_gpu_info_creates_no_context():
leaked, mib = _run_child("hw.get_backend_visible_gpu_info()")
assert leaked == []
if mib is not None:
assert mib == 0, f"the visibility poll pinned {mib} MiB of primary context"
@needs_cuda
def test_clear_gpu_cache_creates_no_context_when_nothing_reserved():
leaked, mib = _run_child("hw.clear_gpu_cache()")
assert leaked == []
if mib is not None:
assert mib == 0, f"clear_gpu_cache pinned {mib} MiB of primary context"
@needs_cuda
def test_the_probe_would_catch_a_regression():
# Prove mem_get_info really pins a context here, so the tests above are not vacuous.
leaked, mib = _run_child("__import__('torch').cuda.mem_get_info(0)")
if mib is None:
pytest.skip("no per-PID nvidia-smi accounting available to observe the context")
assert mib > 0, "mem_get_info created no context here, so the assertions above prove nothing"
def _cuda_device_count() -> int:
try:
import torch
return int(torch.cuda.device_count()) if torch.cuda.is_available() else 0
except Exception:
return 0
def _is_hip() -> bool:
try:
import torch
return bool(getattr(torch.version, "hip", None))
except Exception:
return False
needs_two_nvidia = pytest.mark.skipif(
_cuda_device_count() < 2 or _is_hip(),
reason = "needs two CUDA devices on NVIDIA to observe a per-card primary context",
)
_PROBE_BY_GPU = textwrap.dedent(
"""
import os, subprocess
def _held_by_gpu():
try:
out = subprocess.run(
["nvidia-smi", "--query-compute-apps=gpu_uuid,pid,used_memory",
"--format=csv,noheader,nounits"],
capture_output = True, text = True, timeout = 60,
)
except Exception:
return None
if out.returncode != 0:
return None
me = str(os.getpid())
held = {}
for line in out.stdout.splitlines():
parts = [p.strip() for p in line.split(",")]
if len(parts) >= 3 and parts[1] == me:
try:
held[parts[0]] = held.get(parts[0], 0) + int(parts[2])
except ValueError:
return None
return held
def _primary_contexts():
import torch
has = getattr(torch._C, "_cuda_hasPrimaryContext", None)
if has is None:
return None
return [i for i in range(torch.cuda.device_count()) if has(i)]
"""
)
_BY_GPU_CHILD = textwrap.dedent(
"""
import sys
sys.path.insert(0, {backend!r})
import torch
assert sum(torch.cuda.memory_reserved(i) for i in range(torch.cuda.device_count())) == 0
visited = None
{call}
print("VISITED:" + repr(visited))
print("HELD:" + repr(_held_by_gpu()))
print("PRIMARY:" + repr(_primary_contexts()))
"""
)
# Probe bodies exec'd from main.py source: importing main builds the app.
_DENSE_QUANT_PROBES = textwrap.dedent(
"""
import ast
from typing import Optional
src = open({main!r}, encoding = "utf-8").read()
tree = ast.parse(src)
ns = {{"Optional": Optional}}
for name in ("_probe_dense_quant_supported", "_probe_dense_quant_schemes"):
node = next(n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == name)
exec(ast.get_source_segment(src, node), ns)
from core.inference import diffusion_transformer_quant as tq
visited = {{"bit": [], "ladder": []}}
_capable, _ladder = tq.dense_quant_host_capable, tq.auto_scheme_candidates_cached
def _bit(t):
visited["bit"].append(getattr(t, "ordinal", None))
return _capable(t)
def _schemes(t):
visited["ladder"].append(getattr(t, "ordinal", None))
return _ladder(t)
tq.dense_quant_host_capable = _bit
tq.auto_scheme_candidates_cached = _schemes
ns["_probe_dense_quant_supported"]()
ns["_probe_dense_quant_schemes"]()
"""
).format(main = str(_BACKEND_DIR / "main.py"))
def _run_child_by_gpu(call: str):
"""Run ``call`` in a fresh interpreter; return (held_by_gpu | None, primary | None, visited)."""
src = _PROBE_BY_GPU + _BY_GPU_CHILD.format(backend = str(_BACKEND_DIR), call = call)
proc = subprocess.run([sys.executable, "-c", src], capture_output = True, text = True, timeout = 600)
assert proc.returncode == 0, f"child failed:\n{proc.stdout}\n{proc.stderr}"
out: dict = {}
for line in proc.stdout.splitlines():
for key in ("VISITED", "HELD", "PRIMARY"):
if line.startswith(key + ":"):
out[key] = eval(line[len(key) + 1 :])
assert set(out) == {"VISITED", "HELD", "PRIMARY"}, proc.stdout
return out["HELD"], out["PRIMARY"], out["VISITED"]
@needs_two_nvidia
def test_the_dense_quant_probes_pin_no_context_on_any_card():
held, primary, visited = _run_child_by_gpu(_DENSE_QUANT_PROBES)
# A swallowed import error also answers "incapable"; prove every card was visited.
assert visited["ladder"] == list(range(_cuda_device_count())), visited
assert visited["bit"][:1] == [0], visited
if held is None and primary is None:
pytest.skip("no per-PID nvidia-smi accounting and no torch primary-context flag")
if held is not None:
assert not any(held.values()), f"the dense-quant probes pinned a context: {held}"
if primary is not None:
assert primary == [], f"the dense-quant probes pinned a primary context on {primary}"
@needs_two_nvidia
def test_the_multi_gpu_probe_would_catch_a_regression():
held, primary, _ = _run_child_by_gpu("torch.cuda.device(1).__enter__()")
if held is None and primary is None:
pytest.skip("no per-PID nvidia-smi accounting and no torch primary-context flag")
pinned = bool(held and any(held.values())) or bool(primary and 1 in primary)
assert pinned, "entering torch.cuda.device(1) pinned nothing, so the test above proves nothing"
def _summary_torch(*, properties_total, driver_free, driver_total):
props = types.SimpleNamespace(total_memory = properties_total, name = "Test GPU")
def _mem_get_info(_ordinal):
if driver_free is None:
raise AssertionError("the native probe answered; mem_get_info would pin a context")
return driver_free, driver_total
cuda = types.SimpleNamespace(
current_device = lambda: 0,
get_device_properties = lambda _ordinal: props,
memory_allocated = lambda _ordinal: 1 << 30,
memory_reserved = lambda _ordinal: 2 << 30,
mem_get_info = _mem_get_info,
)
return types.SimpleNamespace(cuda = cuda), props
def test_gpu_summary_prefers_context_free_driver_memory(monkeypatch):
gib = 1 << 30
torch_stub, _props = _summary_torch(
properties_total = 16 * gib,
driver_free = None,
driver_total = 16 * gib,
)
monkeypatch.setitem(sys.modules, "torch", torch_stub)
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "IS_ROCM", False)
_shared_setup_4(monkeypatch)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: {
"available": True,
"devices": [{"visible_ordinal": 0, "vram_used_gb": 7.0, "vram_total_gb": 16.0}],
},
)
summary = hw.get_gpu_summary()
assert summary == {"gpu_name": "Test GPU", "vram_total_gb": 16.0, "vram_free_gb": 9.0}
def test_gpu_summary_pairs_rocm_apu_free_with_driver_total(monkeypatch):
_props, gib = _shared_setup_3(monkeypatch)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: pytest.fail("an APU needs hipMemGetInfo's matching GTT total"),
)
monkeypatch.setattr(hw, "_rocm_windows_unified_used_bytes", lambda: None)
summary = hw.get_gpu_summary()
assert summary == {"gpu_name": "Test GPU", "vram_total_gb": 100.0, "vram_free_gb": 98.0}
def test_gpu_summary_apu_prefers_wddm_counters_over_process_local_hip(monkeypatch):
# WDDM hipMemGetInfo is process-local: APU total comes from HIP, used from the counters.
_props, gib = _shared_setup_3(monkeypatch)
monkeypatch.setattr(hw, "_rocm_props_are_positively_unified", lambda _props: True)
monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: pytest.fail("an APU must not take its total from the vendor CLI"),
)
monkeypatch.setattr(hw, "_rocm_windows_unified_used_bytes", lambda: 40.0 * gib)
summary = hw.get_gpu_summary()
assert summary == {"gpu_name": "Test GPU", "vram_total_gb": 100.0, "vram_free_gb": 60.0}
def test_context_free_rocm_smi_translates_hip_ordinals(monkeypatch):
from utils.hardware import amd
gib = 1 << 30
monkeypatch.setattr(hw, "IS_ROCM", True)
# Visibility vars are read from the real environment, so isolate them.
_shared_setup_5(monkeypatch)
_shared_setup_1(monkeypatch)
monkeypatch.setattr(amd, "get_hip_id_by_gpu_index", lambda: {0: 1, 1: 0})
monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 2)
# amd-smi GPU 1 is HIP 0 and owns the 96 GiB pool.
readings = {0: (15.0, 16.0), 1: (2.0, 96.0)}
def _smi(_name, visible_ids, **_kwargs):
return {
"available": True,
"devices": [
{
"index": smi_id,
"visible_ordinal": ordinal,
"vram_used_gb": readings[smi_id][0],
"vram_total_gb": readings[smi_id][1],
}
for ordinal, smi_id in enumerate(visible_ids)
],
}
monkeypatch.setattr(hw, "_smi_query", _smi)
assert hw._context_free_cuda_memory_info(0, 96 * gib) == 94 * gib
def test_context_free_rocm_smi_declines_without_an_id_mapping(monkeypatch):
amd = _shared_setup_7(monkeypatch)
_shared_setup_1(monkeypatch)
monkeypatch.setattr(amd, "get_hip_id_by_gpu_index", lambda: None)
monkeypatch.setattr(amd, "get_physical_gpu_count", lambda: 2)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: pytest.fail("an unproven amd-smi/HIP mapping must be declined"),
)
assert hw._context_free_cuda_memory_info(0, 16 << 30) is None
def test_context_free_rocm_smi_declines_stacked_visibility_masks(monkeypatch):
from utils.hardware import amd
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw.sys, "platform", "linux")
monkeypatch.setattr(hw.platform, "system", lambda: "Darwin")
monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "1,2")
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
monkeypatch.setattr(
hw,
"_get_parent_visible_gpu_spec",
lambda: {"raw": "0", "numeric_ids": [0], "supports_explicit_gpu_ids": True},
)
monkeypatch.setattr(
amd,
"get_hip_id_by_gpu_index",
lambda: pytest.fail("stacked masks have no global HIP mapping"),
)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: pytest.fail("stacked masks must decline amd-smi telemetry"),
)
assert hw._context_free_cuda_memory_info(0, 24 << 30) is None
def test_context_free_rocm_smi_declines_gpu_device_ordinal(monkeypatch):
amd = _shared_setup_7(monkeypatch)
monkeypatch.setenv("GPU_DEVICE_ORDINAL", "1")
monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
_shared_setup_1(monkeypatch)
monkeypatch.setattr(
amd,
"get_hip_id_by_gpu_index",
lambda: pytest.fail("GPU_DEVICE_ORDINAL makes the global mapping unusable"),
)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: pytest.fail("an external GPU ordinal filter must decline amd-smi"),
)
assert hw._context_free_cuda_memory_info(0, 24 << 30) is None
def test_context_free_rocm_smi_declines_inventory_count_mismatch(monkeypatch):
amd = _shared_setup_7(monkeypatch)
_shared_setup_5(monkeypatch)
_shared_setup_1(monkeypatch)
monkeypatch.setattr(amd, "get_hip_id_by_gpu_index", lambda: {0: 0, 1: 1})
monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: pytest.fail("amd-smi and HIP inventory counts must agree"),
)
assert hw._context_free_cuda_memory_info(0, 24 << 30) is None
def test_context_free_nvidia_smi_resolves_a_uuid_mask(monkeypatch):
from utils.hardware import nvidia
gib = 1 << 30
monkeypatch.setattr(hw, "IS_ROCM", False)
monkeypatch.setattr(
hw,
"_get_parent_visible_gpu_spec",
lambda: {
"raw": "GPU-bbbbbbbb",
"numeric_ids": None,
"supports_explicit_gpu_ids": False,
},
)
def _run(command, **_kwargs):
assert "--query-gpu=index,uuid,utilization.gpu,temperature.gpu," in command[1]
return types.SimpleNamespace(
returncode = 0,
stdout = "\n".join(
[
"0, GPU-aaaaaaaa-1111-2222-3333-444444444444, 5, 30, 15360, 16384, 100, 300",
"1, GPU-bbbbbbbb-1111-2222-3333-444444444444, 7, 31, 2048, 98304, 120, 350",
]
),
)
monkeypatch.setattr(nvidia.subprocess, "run", _run)
assert hw._context_free_cuda_memory_info(0, 96 * gib) == 94 * gib
def test_nvidia_uuid_mask_keeps_public_indices_relative(monkeypatch):
from utils.hardware import nvidia
def _run(_command, **_kwargs):
return types.SimpleNamespace(
returncode = 0,
stdout = "\n".join(
[
"0, GPU-aaaaaaaa-1111-2222-3333-444444444444, 5, 30, 15360, 16384, 100, 300",
"1, GPU-bbbbbbbb-1111-2222-3333-444444444444, 7, 31, 2048, 98304, 120, 350",
]
),
)
monkeypatch.setattr(nvidia.subprocess, "run", _run)
result = nvidia.get_visible_gpu_utilization(
None,
"GPU-bbbbbbbb,GPU-aaaaaaaa",
)
assert result["index_kind"] == "relative"
assert result["parent_visible_gpu_ids"] == []
assert {device["visible_ordinal"]: device["index"] for device in result["devices"]} == {
0: 0,
1: 1,
}
# The mask asks for the b-card first; nvidia-smi emitted the rows the other way round.
assert [device["vram_used_gb"] for device in result["devices"]] == [2.0, 15.0]
assert [device["visible_ordinal"] for device in result["devices"]] == [0, 1]
def _nvidia_smi_reading(monkeypatch, used_gb, driver_total_gb):
monkeypatch.setattr(hw, "IS_ROCM", False)
monkeypatch.setattr(
hw,
"_get_parent_visible_gpu_spec",
lambda: {"raw": None, "numeric_ids": [0], "supports_explicit_gpu_ids": True},
)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: {
"available": True,
"devices": [
{"visible_ordinal": 0, "vram_used_gb": used_gb, "vram_total_gb": driver_total_gb}
],
},
)
def test_context_free_free_excludes_the_reserved_framebuffer(monkeypatch):
# torch's total excludes the reservation; subtracting from the driver total adds it back.
gib = 1 << 30
_nvidia_smi_reading(monkeypatch, used_gb = 2.0, driver_total_gb = 100.0)
assert hw._context_free_cuda_memory_info(0, round(99.5 * gib)) == round(97.5 * gib)
def test_context_free_free_is_zero_on_a_full_card(monkeypatch):
gib = 1 << 30
_nvidia_smi_reading(monkeypatch, used_gb = 99.5, driver_total_gb = 100.0)
assert hw._context_free_cuda_memory_info(0, round(99.5 * gib)) == 0
def test_context_free_free_never_goes_negative(monkeypatch):
gib = 1 << 30
_nvidia_smi_reading(monkeypatch, used_gb = 120.0, driver_total_gb = 100.0)
assert hw._context_free_cuda_memory_info(0, round(99.5 * gib)) == 0
def test_context_free_declines_when_cuda_order_is_not_pci_bus_id(monkeypatch):
# CUDA enumerates FASTEST_FIRST while nvidia-smi reports PCI order.
monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
_nvidia_smi_reading(monkeypatch, used_gb = 2.0, driver_total_gb = 24.0)
monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: pytest.fail("an unshared index space must decline SMI telemetry"),
)
assert hw._context_free_cuda_memory_info(0, 24 << 30) is None
def test_context_free_keeps_smi_on_a_single_gpu_under_fastest_first(monkeypatch):
gib = 1 << 30
monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
_nvidia_smi_reading(monkeypatch, used_gb = 2.0, driver_total_gb = 24.0)
monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 1)
monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", True)
assert hw._context_free_cuda_memory_info(0, 24 * gib) == 22 * gib
def test_context_free_declines_when_the_gpu_count_is_not_smi_confirmed(monkeypatch):
# Without nvidia-smi -L, a masked torch count would be cached, so decline the exemption.
gib = 1 << 30
monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
_nvidia_smi_reading(monkeypatch, used_gb = 2.0, driver_total_gb = 24.0)
monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 1)
monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)
assert hw._context_free_cuda_memory_info(0, 24 * gib) is None
def test_physical_gpu_count_records_whether_the_smi_answered(monkeypatch):
from utils.hardware import nvidia
monkeypatch.setattr(hw, "_physical_gpu_count", None)
monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "IS_ROCM", False)
monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda gpu_rows_only = False: 4)
assert hw.get_physical_gpu_count() == 4
assert hw._physical_gpu_count_from_smi is True
monkeypatch.setattr(hw, "_physical_gpu_count", None)
monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)
monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda gpu_rows_only = False: None)
monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1)
assert hw.get_physical_gpu_count() == 1
assert hw._physical_gpu_count_from_smi is False
def test_free_in_torch_scope_never_exceeds_the_total(monkeypatch):
# SMI parsers accept signed values; a negative used counter must not inflate free.
gib = 1 << 30
assert hw._free_in_torch_scope(24 * gib, -1.0) == 24 * gib
assert hw._free_in_torch_scope(24 * gib, 0.0) == 24 * gib
assert hw._free_in_torch_scope(24 * gib, 2.0) == 22 * gib
assert hw._free_in_torch_scope(24 * gib, 99.0) == 0
def test_context_free_uuid_mask_skips_the_cuda_order_gate(monkeypatch):
from utils.hardware import nvidia
# A UUID mask is absolute and ordered, so FASTEST_FIRST changes nothing.
gib = 1 << 30
monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
monkeypatch.setattr(hw, "IS_ROCM", False)
monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)
monkeypatch.setattr(
hw,
"_get_parent_visible_gpu_spec",
lambda: {
"raw": "GPU-bbbbbbbb",
"numeric_ids": None,
"supports_explicit_gpu_ids": False,
},
)
def _run(_command, **_kwargs):
return types.SimpleNamespace(
returncode = 0,
stdout = "\n".join(
[
"0, GPU-aaaaaaaa-1111-2222-3333-444444444444, 5, 30, 15360, 16384, 100, 300",
"1, GPU-bbbbbbbb-1111-2222-3333-444444444444, 7, 31, 2048, 98304, 120, 350",
]
),
)
monkeypatch.setattr(nvidia.subprocess, "run", _run)
assert hw._context_free_cuda_memory_info(0, 96 * gib) == 94 * gib
def test_parent_visible_spec_ignores_a_rocr_mask_on_windows(monkeypatch):
# Windows HIP has no ROCr layer: a stray ROCR var masks nothing and must not map ordinals.
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)
monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "1,0")
monkeypatch.setattr(hw.sys, "platform", "win32")
assert hw._get_parent_visible_gpu_spec()["numeric_ids"] == [0, 1]
monkeypatch.setattr(hw.sys, "platform", "linux")
assert hw._get_parent_visible_gpu_spec()["numeric_ids"] == [1, 0]
def test_context_free_rocm_windows_declines_gpu_device_ordinal(monkeypatch):
# GPU_DEVICE_ORDINAL is a ROCclr var _get_parent_visible_gpu_spec never reads.
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
monkeypatch.setenv("GPU_DEVICE_ORDINAL", "1")
_shared_setup_1(monkeypatch)
monkeypatch.setattr(
hw,
"_rocm_windows_per_device_vram",
lambda ids, adapters = None: pytest.fail(
"an external GPU ordinal filter must decline the adapter path"
),
)
assert hw._context_free_cuda_memory_info(0, 24 << 30) is None
def test_context_free_nvidia_smi_declines_whole_gpu_metrics_for_mig(monkeypatch):
gib = 1 << 30
monkeypatch.setattr(hw, "IS_ROCM", False)
_shared_setup_4(monkeypatch)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: {
"available": True,
"devices": [{"visible_ordinal": 0, "vram_used_gb": 70.0, "vram_total_gb": 80.0}],
},
)
# torch reports the 10 GiB MIG slice, --query-gpu its 80 GiB parent.
assert hw._context_free_cuda_memory_info(0, 10 * gib) is None
def test_context_free_never_spawns_amd_smi_on_windows_without_a_hip_sdk(monkeypatch):
"""amd-smi raises a UAC prompt on Windows without HIP, so the amd.py guard must hold."""
from utils.hardware import amd
spawned = []
def _spy(command, *args, **kwargs):
spawned.append(command)
raise FileNotFoundError("no amd-smi should be spawned here")
monkeypatch.setattr(amd, "platform", types.SimpleNamespace(system = lambda: "Windows"))
monkeypatch.setattr(amd.shutil, "which", lambda _n: r"C:\rocm\bin\amd-smi.exe")
monkeypatch.setattr(amd.subprocess, "run", _spy)
monkeypatch.delenv("UNSLOTH_ENABLE_AMD_SMI", raising = False)
monkeypatch.setattr(amd, "_amd_smi_disabled", False)
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
_shared_setup_1(monkeypatch)
monkeypatch.setattr(
hw,
"_rocm_windows_perf_counter_vram_by_adapter",
lambda counter = "Dedicated Usage": [("a", 8.0 * (1 << 30)), ("b", 0.0)],
)
monkeypatch.setattr(
hw,
"_rocm_windows_per_device_vram",
lambda ids, adapters = None: (
[{"index": 0, "visible_ordinal": 0, "name": "RX", "used_gb": 8.0, "total_gb": 24.0}],
8.0,
),
)
assert hw._context_free_cuda_memory_info(0, 24 << 30) == 16 << 30
assert spawned == []
@pytest.mark.skipif(sys.platform != "win32", reason = "the guard is Windows-only")
def test_amd_smi_is_refused_on_a_real_windows_host_unless_opted_in():
# Runs the same guard on the real OS, PATH and environment, not a faked platform.
from utils.hardware import amd
if os.environ.get("UNSLOTH_ENABLE_AMD_SMI", "").strip().lower() in ("1", "true", "yes", "on"):
pytest.skip(
reason = "UNSLOTH_ENABLE_AMD_SMI opts this host in, so amd-smi is allowed by design"
)
assert amd._amd_smi_allowed() is False
def test_context_free_rocm_smi_declines_whole_gpu_metrics_for_a_partition(monkeypatch):
from utils.hardware import amd
gib = 1 << 30
monkeypatch.setattr(hw, "IS_ROCM", True)
_shared_setup_5(monkeypatch)
_shared_setup_4(monkeypatch)
monkeypatch.setattr(amd, "get_hip_id_by_gpu_index", lambda: {0: 0})
monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1)
monkeypatch.setattr(
hw,
"_smi_query",
lambda *a, **k: {
"available": True,
"devices": [{"visible_ordinal": 0, "vram_used_gb": 70.0, "vram_total_gb": 80.0}],
},
)
# Pin the whole path including sysfs, else this is vacuous on hosts with or without sysfs.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
monkeypatch.setattr(
hw,
"_torch_get_device_module",
lambda: (
types.SimpleNamespace(
get_device_properties = lambda _o: _FakeProps("MI300X partition", 10 * gib)
),
"cuda",
),
)
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
monkeypatch.setattr(hw, "_rocm_visibility_mask_active", lambda: False)
monkeypatch.setattr(
hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {"0000:03:00.0": (70.0, 80.0)}
)
assert hw._context_free_cuda_memory_info(0, 10 * gib) is None
class _FakeProps:
def __init__(self, name, total):
self.name = name
self.total_memory = total
def _fake_mod(
totals,
names = None,
*,
with_mem_get_info = True,
):
mod = types.SimpleNamespace()
names = names or [f"GPU{i}" for i in range(len(totals))]
mod.get_device_properties = lambda o: _FakeProps(names[o], totals[o])
if with_mem_get_info:
mod.mem_get_info = lambda o: (totals[o] - (1 << 30), totals[o])
mod.memory_allocated = lambda o: 1 << 30
return mod
@pytest.mark.parametrize("totals", [[191505498112], [25757220864, 8589934592]])
def test_inventory_totals_match_the_occupancy_path(monkeypatch, totals):
mod = _fake_mod(totals)
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "rocm_windows_free_is_untrusted", lambda: False)
indices = list(range(len(totals)))
inventory = hw._torch_get_device_inventory(indices)
occupancy = hw._torch_get_per_device_info(indices)
assert [d["total_gb"] for d in inventory] == [d["total_gb"] for d in occupancy]
assert [d["name"] for d in inventory] == [d["name"] for d in occupancy]
assert [d["index"] for d in inventory] == [d["index"] for d in occupancy]
assert [d["visible_ordinal"] for d in inventory] == [d["visible_ordinal"] for d in occupancy]
assert all(d["used_gb"] is None for d in inventory)
assert all(d["shared_memory"] is False for d in inventory)
def test_inventory_never_touches_mem_get_info(monkeypatch):
def _boom(*a, **k):
raise AssertionError("mem_get_info would create a primary context")
mod = _fake_mod([8589934592])
mod.mem_get_info = _boom
mod.memory_allocated = _boom
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
assert hw._torch_get_device_inventory([0])[0]["total_gb"] == 8.0
def test_inventory_skips_a_device_that_fails_to_enumerate(monkeypatch):
def _props(o):
if o == 1:
raise RuntimeError("device lost")
return _FakeProps("GPU0", 8589934592)
mod = types.SimpleNamespace(get_device_properties = _props)
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
assert [d["index"] for d in hw._torch_get_device_inventory([0, 1])] == [0]
def test_inventory_is_empty_without_torch(monkeypatch):
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (None, None))
assert hw._torch_get_device_inventory([0]) == []
def test_visibility_endpoint_uses_inventory_not_occupancy(monkeypatch):
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0, 1])
monkeypatch.setattr(
hw,
"_torch_get_per_device_info",
lambda ids, adapters = None: pytest.fail(
"the visibility poll must not ask torch for occupancy"
),
)
monkeypatch.setattr(
hw,
"_torch_get_device_inventory",
lambda ids, adapters = None: [
{
"index": 0,
"visible_ordinal": 0,
"name": "MI300X",
"total_gb": 192.0,
"used_gb": None,
"shared_memory": False,
},
{
"index": 1,
"visible_ordinal": 1,
"name": "MI300X",
"total_gb": 192.0,
"used_gb": None,
"shared_memory": True,
},
],
)
result = hw.get_backend_visible_gpu_info()
assert result["available"] is True
assert [d["memory_total_gb"] for d in result["devices"]] == [192.0, 192.0]
assert [d["name"] for d in result["devices"]] == ["MI300X", "MI300X"]
assert [d["shared_memory"] for d in result["devices"]] == [False, True]
assert result["index_kind"] == "physical"
# On a unified-memory APU props reports the carve-out and hipMemGetInfo the larger GTT pool;
# the inventory must use GTT (as _apply_unified_memory_correction does).
_APU_CARVE_OUT = 8 * 1024**3
_APU_GTT_TOTAL = 100 * 1024**3
class _FakeApuProps(_FakeProps):
def __init__(self, total = _APU_CARVE_OUT):
super().__init__("AMD Radeon 8060S Graphics", total)
self.gcnArchName = "gfx1151" # Strix Halo
self.is_integrated = 1
def _apu_mod(gtt_total = _APU_GTT_TOTAL, *, carve_out = _APU_CARVE_OUT):
mod = types.SimpleNamespace()
mod.get_device_properties = lambda o: _FakeApuProps(carve_out)
mod.mem_get_info = lambda o: (gtt_total - (2 << 30), gtt_total)
mod.memory_allocated = lambda o: 1 << 30
return mod
def test_rocm_apu_inventory_keeps_the_gtt_total(monkeypatch):
mod = _apu_mod()
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "rocm_windows_free_is_untrusted", lambda: False)
inventory = hw._torch_get_device_inventory([0])
occupancy = hw._torch_get_per_device_info([0])
assert inventory[0]["total_gb"] == occupancy[0]["total_gb"] == 100.0
assert inventory[0]["total_gb"] != round(_APU_CARVE_OUT / (1024**3), 2)
assert inventory[0]["shared_memory"] is True
assert inventory[0]["shared_memory_host_backed_gb"] == 92.0
class _FakeRocmProps(_FakeProps):
"""Discrete card as a current ROCm runtime describes it: the flag is filled in."""
def __init__(
self,
name = "MI300X",
total = 191505498112,
arch = "gfx942",
integrated = 0,
):
super().__init__(name, total)
self.gcnArchName = arch
self.is_integrated = integrated
def _rocm_mod(props, gtt_total = _APU_GTT_TOTAL):
mod = types.SimpleNamespace()
mod.get_device_properties = lambda o: props
mod.mem_get_info = lambda o: (gtt_total - (2 << 30), gtt_total)
return mod
def test_rocm_discrete_inventory_stays_context_free(monkeypatch):
mod = _rocm_mod(_FakeRocmProps())
mod.mem_get_info = lambda o: pytest.fail("mem_get_info would create a primary context")
inventory = _shared_setup_6(mod, monkeypatch)
assert inventory["total_gb"] == 178.35
assert inventory["shared_memory"] is False
def test_an_apu_shaped_name_on_cuda_stays_context_free(monkeypatch):
mod = _apu_mod()
mod.mem_get_info = lambda o: pytest.fail("no ROCm here, so no reason to pay for a context")
monkeypatch.setattr(hw, "IS_ROCM", False)
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
assert hw._torch_get_device_inventory([0])[0]["total_gb"] == 8.0
def test_rocm_apu_keeps_the_carve_out_when_the_driver_total_fails(monkeypatch):
mod = _apu_mod()
def _fail(o):
raise RuntimeError("hipMemGetInfo unavailable")
mod.mem_get_info = _fail
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
inventory = hw._torch_get_device_inventory([0])[0]
assert inventory["total_gb"] == 8.0
assert inventory["shared_memory"] is False
def test_windows_rocm_apu_keeps_shared_scope_when_driver_total_fails(monkeypatch):
mod = _apu_mod(gtt_total = _APU_GTT_TOTAL, carve_out = _APU_GTT_TOTAL)
def _fail(o):
raise RuntimeError("hipMemGetInfo unavailable")
mod.mem_get_info = _fail
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
inventory = hw._torch_get_device_inventory([0])[0]
assert inventory["total_gb"] == 100.0
assert inventory["shared_memory"] is True
assert inventory["shared_memory_host_backed_gb"] is None
@pytest.mark.parametrize(("system", "expected_shared"), [("Linux", False), ("Windows", True)])
def test_rocm_apu_equal_driver_total_scope_is_platform_specific(
monkeypatch, system, expected_shared
):
mod = _apu_mod(gtt_total = _APU_GTT_TOTAL, carve_out = _APU_GTT_TOTAL)
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw.platform, "system", lambda: system)
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
inventory = hw._torch_get_device_inventory([0])[0]
assert inventory["total_gb"] == 100.0
assert inventory["shared_memory"] is expected_shared
def test_windows_rocm_apu_reports_only_the_host_backed_pool_overlap(monkeypatch):
_shared_setup_2(monkeypatch)
monkeypatch.setattr(
hw,
"_windows_amd_adapter_records_by_luid",
lambda: {
0x14CF5: {
"name": "AMD Radeon(TM) 8060S Graphics",
"dedicated_memory_bytes": 32 * 1024**3,
}
},
)
result = hw.get_backend_visible_gpu_info()
assert result["devices"][0]["shared_memory"] is True
assert result["devices"][0]["memory_total_gb"] == 100.0
assert result["devices"][0]["shared_memory_host_backed_gb"] == 68.0
def test_windows_rocm_apu_uses_arch_when_registry_name_differs(monkeypatch):
_shared_setup_2(monkeypatch)
monkeypatch.setattr(
hw,
"_windows_amd_adapter_records_by_luid",
lambda: {
0x14CF5: {
"name": "AMD Radeon(TM) Graphics",
"gfx": "gfx1151",
"dedicated_memory_bytes": 32 * 1024**3,
},
0x24CF5: {
"name": "AMD Radeon 8060S Graphics",
"gfx": "gfx1103",
"dedicated_memory_bytes": 2 * 1024**3,
},
},
)
result = hw.get_backend_visible_gpu_info()
assert result["devices"][0]["shared_memory_host_backed_gb"] == 68.0
def test_windows_rocm_apu_declines_an_ambiguous_arch_fallback(monkeypatch):
_shared_setup_2(monkeypatch)
monkeypatch.setattr(
hw,
"_windows_amd_adapter_records_by_luid",
lambda: {
0x14CF5: {
"name": "AMD Radeon(TM) Graphics",
"gfx": "gfx1151",
"dedicated_memory_bytes": 32 * 1024**3,
},
0x24CF5: {
"name": "AMD Radeon(TM) Graphics 2",
"gfx": "gfx1151",
"dedicated_memory_bytes": 16 * 1024**3,
},
},
)
result = hw.get_backend_visible_gpu_info()
assert result["devices"][0]["shared_memory_host_backed_gb"] is None
def test_windows_rocm_apu_accepts_equal_duplicate_carve_outs(monkeypatch):
_shared_setup_2(monkeypatch)
monkeypatch.setattr(
hw,
"_windows_amd_adapter_records_by_luid",
lambda: {
0x14CF5: {
"name": "AMD Radeon 8060S Graphics",
"gfx": "gfx1151",
"dedicated_memory_bytes": 8 * 1024**3,
},
0x24CF5: {
"name": "AMD Radeon 8060S Graphics",
"gfx": "gfx1151",
"dedicated_memory_bytes": 8 * 1024**3,
},
},
)
result = hw.get_backend_visible_gpu_info()
assert result["devices"][0]["shared_memory_host_backed_gb"] == 92.0
def test_windows_rocm_apu_declines_a_conflicting_arch_candidate(monkeypatch):
_shared_setup_2(monkeypatch)
monkeypatch.setattr(
hw,
"_windows_amd_adapter_records_by_luid",
lambda: {
0x14CF5: {
"name": "AMD Radeon 8060S Graphics",
"gfx": "gfx1151",
"dedicated_memory_bytes": 8 * 1024**3,
},
0x24CF5: {
"name": "AMD Radeon 8060S Graphics",
"gfx": "gfx1151",
"dedicated_memory_bytes": 8 * 1024**3,
},
0x34CF5: {
"name": "AMD Radeon(TM) Graphics",
"gfx": "gfx1151",
"dedicated_memory_bytes": 32 * 1024**3,
},
},
)
result = hw.get_backend_visible_gpu_info()
assert result["devices"][0]["shared_memory_host_backed_gb"] is None
def test_windows_rocm_shared_pool_requires_a_complete_record_matching(monkeypatch):
monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
monkeypatch.setattr(
hw,
"_windows_amd_adapter_records_by_luid",
lambda: {
0x01: {"name": "AMD Radeon Graphics", "dedicated_memory_bytes": 8 * 1024**3},
0x02: {
"name": "AMD Radeon Graphics",
"gfx": "gfx1151",
"dedicated_memory_bytes": 8 * 1024**3,
},
0x03: {
"name": "AMD Radeon Graphics",
"gfx": "gfx1151",
"dedicated_memory_bytes": 8 * 1024**3,
},
0x04: {
"name": "AMD Radeon 780M",
"gfx": "gfx1103",
"dedicated_memory_bytes": 16 * 1024**3,
},
0x05: {
"name": "AMD Radeon 8060S",
"gfx": "gfx1150",
"dedicated_memory_bytes": 32 * 1024**3,
},
},
)
devices = [
{
"index": index,
"name": "AMD Radeon Graphics",
"total_gb": 100.0,
"shared_memory": True,
"_rocm_gfx": gfx,
}
for index, gfx in enumerate(("gfx1103", "gfx1150", "gfx1151"))
]
assert hw._windows_rocm_shared_pool_host_gb_by_index(devices) == {}
@pytest.mark.parametrize(
("torch_total", "sysfs_total", "expected_shared"),
[(_APU_GTT_TOTAL, _APU_CARVE_OUT, True), (_APU_CARVE_OUT, _APU_CARVE_OUT, False)],
)
@pytest.mark.parametrize(("hip_mask", "cuda_mask"), [(None, None), ("0", None), ("0", "0")])
def test_linux_rocm_apu_uses_sysfs_to_confirm_equal_total_scope(
monkeypatch, torch_total, sysfs_total, expected_shared, hip_mask, cuda_mask
):
mod = _apu_mod(gtt_total = torch_total, carve_out = torch_total)
for var in (
"HIP_VISIBLE_DEVICES",
"ROCR_VISIBLE_DEVICES",
"CUDA_VISIBLE_DEVICES",
"GPU_DEVICE_ORDINAL",
):
monkeypatch.delenv(var, raising = False)
if hip_mask is not None:
monkeypatch.setenv("HIP_VISIBLE_DEVICES", hip_mask)
if cuda_mask is not None:
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", cuda_mask)
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0])
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
monkeypatch.setattr(
hw,
"_rocm_linux_sysfs_vram_by_pci_gb",
lambda: {"0000:03:00.0": (1.0, sysfs_total / (1024**3))},
)
result = hw.get_backend_visible_gpu_info()
assert result["devices"][0]["memory_total_gb"] == torch_total / (1024**3)
assert result["devices"][0]["shared_memory"] is expected_shared
# With sysfs answering, host-backed is a measured zero; omitted renders as all shared.
assert result["devices"][0]["shared_memory_host_backed_gb"] == (
round((torch_total - sysfs_total) / (1024**3), 2) if expected_shared else 0.0
)
@pytest.mark.parametrize("ambiguous_var", ["ROCR_VISIBLE_DEVICES", "GPU_DEVICE_ORDINAL"])
def test_linux_rocm_shared_pool_declines_ambiguous_masks(monkeypatch, ambiguous_var):
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")
monkeypatch.setenv(ambiguous_var, "0")
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
monkeypatch.setattr(
hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {"0000:03:00.0": (1.0, 8.0)}
)
devices = [{"index": 0, "total_gb": 100.0, "_rocm_known_unified": True}]
assert hw._rocm_linux_shared_pool_host_gb_by_index(devices) == {}
def test_linux_rocm_shared_pool_declines_a_compacted_numeric_mask(monkeypatch):
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")
monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
monkeypatch.delenv("GPU_DEVICE_ORDINAL", raising = False)
monkeypatch.setattr(
hw,
"_rocm_kfd_gpu_pci_ids",
lambda: {0: "0000:03:00.0", 1: "0000:04:00.0"},
)
monkeypatch.setattr(
hw,
"_rocm_linux_sysfs_vram_by_pci_gb",
lambda: {
"0000:03:00.0": (1.0, 8.0),
"0000:04:00.0": (1.0, 16.0),
},
)
devices = [{"index": 0, "total_gb": 100.0, "_rocm_known_unified": True}]
assert hw._rocm_linux_shared_pool_host_gb_by_index(devices) == {}
def test_linux_rocm_shared_pool_maps_lone_uuid_mask(monkeypatch):
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
monkeypatch.delenv("GPU_DEVICE_ORDINAL", raising = False)
monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "GPU-DEADBEEFDEADBEEF")
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
monkeypatch.setattr(
hw,
"_rocm_linux_sysfs_vram_by_pci_gb",
lambda: {"0000:03:00.0": (1.0, 8.0)},
)
devices = [{"index": 0, "total_gb": 100.0, "_rocm_known_unified": True}]
assert hw._rocm_linux_shared_pool_host_gb_by_index(devices) == {0: 92.0}
def test_rocm_apu_sysfs_overlay_still_declines_against_the_gtt_total(monkeypatch):
# Against the GTT total the sysfs-first overlay declines, so the APU is not shrunk.
mod = _apu_mod()
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
monkeypatch.setattr(hw, "platform", types.SimpleNamespace(system = lambda: "Linux"))
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
monkeypatch.setattr(hw, "_rocm_visibility_mask_active", lambda: False)
monkeypatch.setattr(
hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {"0000:03:00.0": (1.0, 8.0)}
)
inventory = hw._torch_get_device_inventory([0])
probe = [{"index": d["index"], "vram_total_gb": d["total_gb"]} for d in inventory]
assert hw._rocm_system_wide_vram_by_index(probe) == {}
# The classifier never says "cannot tell": an iGPU outside gfx1150-1152 without the integrated
# flag (set only from ROCm 6.1.2) reads as not-unified.
class _FakeUnclassifiedApuProps(_FakeProps):
"""gfx1103 Phoenix: a real APU that sits outside the classifier's arch set."""
def __init__(
self,
total = _APU_CARVE_OUT,
*,
integrated = 0,
):
super().__init__("AMD Radeon 780M Graphics", total)
self.gcnArchName = "gfx1103"
if integrated is not None:
self.is_integrated = integrated
def test_rocm_unclassified_apu_keeps_the_driver_total_on_an_older_runtime(monkeypatch):
mod = _rocm_mod(_FakeUnclassifiedApuProps())
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "_hip_runtime_version", lambda: (6, 1))
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
inventory = hw._torch_get_device_inventory([0])[0]
assert inventory["total_gb"] == 100.0
assert inventory["shared_memory"] is True
def test_rocm_unclassified_apu_keeps_the_driver_total_with_a_legacy_flag(monkeypatch):
mod = _rocm_mod(_FakeUnclassifiedApuProps())
inventory = _shared_setup_6(mod, monkeypatch)
assert inventory["total_gb"] == 100.0
assert inventory["shared_memory"] is True
def test_rocm_unclassified_apu_keeps_the_driver_total_without_the_flag(monkeypatch):
mod = _rocm_mod(_FakeUnclassifiedApuProps(integrated = None))
inventory = _shared_setup_6(mod, monkeypatch)
assert inventory["total_gb"] == 100.0
assert inventory["shared_memory"] is True
def test_rocm_props_that_cannot_be_classified_keep_the_driver_total(monkeypatch):
class _HostileProps(_FakeProps):
def __init__(self):
super().__init__("AMD Radeon Graphics", _APU_CARVE_OUT)
@property
def gcnArchName(self):
raise RuntimeError("properties unreadable")
mod = _rocm_mod(_HostileProps())
inventory = _shared_setup_6(mod, monkeypatch)
assert inventory["total_gb"] == 100.0
assert inventory["shared_memory"] is False
def test_hip_runtime_version_reads_both_wheel_spellings(monkeypatch):
import torch
monkeypatch.setattr(torch.version, "hip", "6.4.43483-a187df25c", raising = False)
assert hw._hip_runtime_version() == (6, 4)
# AMD SDK / Radeon wheels leave version.hip unset; the tag is in __version__.
monkeypatch.setattr(torch.version, "hip", None, raising = False)
monkeypatch.setattr(torch, "__version__", "2.9.0+rocm6.2", raising = False)
assert hw._hip_runtime_version() == (6, 2)
monkeypatch.setattr(torch, "__version__", "2.9.0+cu128", raising = False)
assert hw._hip_runtime_version() is None
def _torch_stub(reserved_by_device, calls):
cuda = types.SimpleNamespace(
is_available = lambda: True,
device_count = lambda: len(reserved_by_device),
memory_reserved = lambda i = 0: reserved_by_device[i],
synchronize = lambda: calls.append("synchronize"),
empty_cache = lambda: calls.append("empty_cache"),
ipc_collect = lambda: calls.append("ipc_collect"),
)
return types.SimpleNamespace(cuda = cuda)
def test_clear_gpu_cache_skips_synchronize_with_nothing_reserved(monkeypatch):
calls: list[str] = []
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setitem(sys.modules, "torch", _torch_stub([0, 0], calls))
hw.clear_gpu_cache()
# Only synchronize is skipped; empty_cache/ipc_collect need no context.
assert calls == ["empty_cache", "ipc_collect"]
def test_clear_gpu_cache_still_drains_a_live_allocator(monkeypatch):
calls: list[str] = []
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setitem(sys.modules, "torch", _torch_stub([0, 4 << 30], calls))
hw.clear_gpu_cache()
assert calls == ["synchronize", "empty_cache", "ipc_collect"]
def test_clear_gpu_cache_still_propagates_a_sticky_cuda_fault(monkeypatch):
# Unload paths rely on sticky faults surfacing, so the guard must not swallow one.
calls: list[str] = []
stub = _torch_stub([4 << 30], calls)
def _boom():
raise RuntimeError("CUDA error: an illegal memory access was encountered")
stub.cuda.synchronize = _boom
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setitem(sys.modules, "torch", stub)
with pytest.raises(RuntimeError, match = "illegal memory access"):
hw.clear_gpu_cache()
def test_clear_gpu_cache_no_ops_without_cuda(monkeypatch):
calls: list[str] = []
stub = _torch_stub([0], calls)
stub.cuda.is_available = lambda: False
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setitem(sys.modules, "torch", stub)
hw.clear_gpu_cache()
assert calls == ["empty_cache", "ipc_collect"]
def _rocm_linux(monkeypatch, resolved):
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None)
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids, adapters = None: ([], None))
monkeypatch.setattr(
hw,
"_get_parent_visible_gpu_spec",
lambda: {"raw": None, "numeric_ids": [0, 1], "supports_explicit_gpu_ids": True},
)
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0, 1])
monkeypatch.setattr(
hw,
"_torch_get_device_inventory",
lambda ids, adapters = None: [
{"index": i, "visible_ordinal": i, "name": "MI210", "total_gb": 64.0, "used_gb": None}
for i in ids
],
)
monkeypatch.setattr(hw, "_rocm_system_wide_vram_by_index", lambda devs: resolved)
def test_rocm_linux_reads_sysfs_without_asking_torch_for_occupancy(monkeypatch):
_rocm_linux(monkeypatch, {0: (12.0, 64.0), 1: (3.0, 64.0)})
monkeypatch.setattr(
hw,
"_torch_get_per_device_info",
lambda ids, adapters = None: pytest.fail(
"sysfs answered in full; torch occupancy is dead weight"
),
)
result = hw.get_visible_gpu_utilization()
assert result["available"] is True
assert result["index_kind"] == "physical"
assert [(d["vram_used_gb"], d["vram_total_gb"]) for d in result["devices"]] == [
(12.0, 64.0),
(3.0, 64.0),
]
assert [d["vram_utilization_pct"] for d in result["devices"]] == [18.8, 4.7]
def test_rocm_linux_sysfs_first_matches_the_old_torch_then_overlay_result(monkeypatch):
resolved = {0: (12.0, 64.0), 1: (3.0, 64.0)}
_rocm_linux(monkeypatch, resolved)
monkeypatch.setattr(
hw,
"_torch_get_per_device_info",
lambda ids, adapters = None: [
{"index": i, "visible_ordinal": i, "name": "MI210", "total_gb": 64.0, "used_gb": 0.02}
for i in ids
],
)
new = hw.get_visible_gpu_utilization()
old_devices = [
{
"index": i,
"index_kind": "physical",
"visible_ordinal": i,
"gpu_utilization_pct": None,
"temperature_c": None,
"vram_used_gb": 0.02,
"vram_total_gb": 64.0,
"vram_utilization_pct": 0.0,
"power_draw_w": None,
"power_limit_w": None,
"power_utilization_pct": None,
}
for i in (0, 1)
]
hw._apply_system_wide_vram(old_devices, resolved)
assert new["devices"] == old_devices
def test_rocm_linux_falls_back_to_torch_when_sysfs_covers_only_some(monkeypatch):
# One device uncovered: the whole set must go back to torch.
_rocm_linux(monkeypatch, {0: (12.0, 64.0)})
used = []
monkeypatch.setattr(
hw,
"_torch_get_per_device_info",
lambda ids, adapters = None: (
used.append(ids)
or [
{
"index": i,
"visible_ordinal": i,
"name": "MI210",
"total_gb": 64.0,
"used_gb": 1.0,
}
for i in ids
]
),
)
result = hw.get_visible_gpu_utilization()
assert used == [[0, 1]]
assert result["devices"][0]["vram_used_gb"] == 12.0
assert result["devices"][1]["vram_used_gb"] == 1.0
def test_rocm_linux_falls_back_when_sysfs_knows_nothing(monkeypatch):
_rocm_linux(monkeypatch, {})
used = []
monkeypatch.setattr(
hw,
"_torch_get_per_device_info",
lambda ids, adapters = None: (
used.append(ids)
or [
{
"index": i,
"visible_ordinal": i,
"name": "MI210",
"total_gb": 64.0,
"used_gb": 1.0,
}
for i in ids
]
),
)
result = hw.get_visible_gpu_utilization()
assert used == [[0, 1]]
assert [d["vram_used_gb"] for d in result["devices"]] == [1.0, 1.0]
def test_nvidia_torch_fallback_keeps_using_occupancy(monkeypatch):
monkeypatch.setattr(hw, "IS_ROCM", False)
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None)
_shared_setup_4(monkeypatch)
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0])
monkeypatch.setattr(
hw,
"_torch_get_per_device_info",
lambda ids, adapters = None: [
{"index": 0, "visible_ordinal": 0, "name": "L40S", "total_gb": 44.0, "used_gb": 4.0}
],
)
result = hw.get_visible_gpu_utilization()
assert result["devices"][0]["vram_used_gb"] == 4.0
def test_rocm_relative_index_skips_the_sysfs_shortcut(monkeypatch):
_rocm_linux(monkeypatch, {0: (12.0, 64.0), 1: (3.0, 64.0)})
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [])
monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 2)
seen = []
monkeypatch.setattr(
hw,
"_torch_get_per_device_info",
lambda ids, adapters = None: (
seen.append(ids)
or [
{
"index": i,
"visible_ordinal": i,
"name": "MI210",
"total_gb": 64.0,
"used_gb": 1.0,
}
for i in ids
]
),
)
result = hw.get_visible_gpu_utilization()
assert result["index_kind"] == "relative"
assert seen == [[0, 1]]
def test_rocm_windows_does_not_take_the_linux_sysfs_path(monkeypatch):
_rocm_linux(monkeypatch, {0: (12.0, 64.0), 1: (3.0, 64.0)})
monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
monkeypatch.setattr(
hw,
"_rocm_windows_per_device_vram",
lambda ids, adapters = None: (
[
{"index": i, "visible_ordinal": i, "name": "RX", "used_gb": 2.0, "total_gb": 24.0}
for i in ids
],
4.0,
),
)
result = hw.get_visible_gpu_utilization()
assert result["vram_used_gb_aggregate"] == 4.0
assert [d["vram_used_gb"] for d in result["devices"]] == [2.0, 2.0]
def test_overlay_wrapper_still_mutates_in_place(monkeypatch):
devices = [
{"index": 0, "vram_used_gb": 0.02, "vram_total_gb": 64.0, "vram_utilization_pct": 0.0}
]
monkeypatch.setattr(hw, "_rocm_system_wide_vram_by_index", lambda d: {0: (32.0, 64.0)})
assert hw._overlay_system_wide_vram(devices) is None
assert devices[0] == {
"index": 0,
"vram_used_gb": 32.0,
"vram_total_gb": 64.0,
"vram_utilization_pct": 50.0,
}
def test_decision_helper_does_not_mutate(monkeypatch):
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: ["0000:00:02.0"])
monkeypatch.setattr(hw, "_rocm_visibility_mask_active", lambda: False)
monkeypatch.setattr(
hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {"0000:00:02.0": (32.0, 64.0)}
)
devices = [{"index": 0, "vram_total_gb": 64.0}]
snapshot = dict(devices[0])
assert hw._rocm_system_wide_vram_by_index(devices) == {0: (32.0, 64.0)}
assert devices[0] == snapshot
def test_context_free_windows_rocm_declines_when_counters_outnumber_devices(monkeypatch):
# A hidden adapter enlarges the counter set and can map its reading onto a visible card,
# overstating free_gb, which feeds training-method selection. Decline instead.
gib = 1 << 30
monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw, "_rocm_device_ordinal_active", lambda: False)
monkeypatch.setattr(
hw,
"_get_parent_visible_gpu_spec",
lambda: {"raw": "0,1", "numeric_ids": [0, 1], "supports_explicit_gpu_ids": True},
)
monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: {"available": False, "devices": []})
monkeypatch.setattr(
hw,
"_rocm_windows_per_device_vram",
lambda _ids: (_ for _ in ()).throw(
AssertionError("must decline before consulting the capacity mapping")
),
)
monkeypatch.setattr(
hw,
"_rocm_windows_perf_counter_vram_by_adapter",
lambda counter = "Dedicated Usage": [("a", 0.0), ("b", 1.0 * gib), ("hidden", 20.0 * gib)],
)
assert hw._context_free_cuda_memory_info(0, 48 * gib) is None
def test_gpu_summary_apu_counters_need_positively_identified_uma(monkeypatch):
# Shared Usage needs positively identified UMA: carve-out=True also covers uncertain GPUs.
_props, gib = _shared_setup_3(monkeypatch)
monkeypatch.setattr(hw, "_rocm_props_are_positively_unified", lambda _props: False)
monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
monkeypatch.setattr(
hw,
"_rocm_windows_unified_used_bytes",
lambda: (_ for _ in ()).throw(AssertionError("uncertain is not positively unified")),
)
summary = hw.get_gpu_summary()
assert summary == {"gpu_name": "Test GPU", "vram_total_gb": 100.0, "vram_free_gb": 98.0}
# On measured gfx1151 (Strix Halo) the sysfs vram carve-out is 64 GiB, so VRAM is not zero.
_MEASURED_STRIX_HALO_TOTAL_BYTES = 68719476736
def test_the_measured_strix_halo_is_not_reported_as_having_no_vram(monkeypatch):
"""An absent shared_memory_host_backed_gb renders as all-host memory, so report zero explicitly."""
total = _MEASURED_STRIX_HALO_TOTAL_BYTES
mod = _apu_mod(gtt_total = total, carve_out = total)
for var in (
"HIP_VISIBLE_DEVICES",
"ROCR_VISIBLE_DEVICES",
"CUDA_VISIBLE_DEVICES",
"GPU_DEVICE_ORDINAL",
):
monkeypatch.delenv(var, raising = False)
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0])
monkeypatch.setattr(hw, "_torch_get_device_module", lambda: (mod, "cuda"))
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
monkeypatch.setattr(
hw,
"_rocm_linux_sysfs_vram_by_pci_gb",
lambda: {"0000:03:00.0": (9.06, total / (1024**3))},
)
device = hw.get_backend_visible_gpu_info()["devices"][0]
assert device["memory_total_gb"] == 64.0
assert (
device["shared_memory_host_backed_gb"] == 0.0
), "a readable sysfs total that torch does not exceed is a measured zero, not an unknown"
assert device["unified_memory"] is True
# `shared_memory` also means "rows are one pool"; a zero host-backed part must not set it.
assert device["shared_memory"] is False
def test_an_unreadable_sysfs_total_still_leaves_the_split_unknown(monkeypatch):
"""With no DRM sysfs (WSL and the like), answer None: a zero would be an unmeasured claim."""
# The sysfs reader returns {} off Linux, so pin the platform to test the logic itself.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
devices = [{"index": 0, "total_gb": 100.0, "_rocm_known_unified": True}]
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
monkeypatch.setattr(
hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {"0000:03:00.0": (0.0, 0.0)}
)
assert hw._rocm_linux_shared_pool_host_gb_by_index(devices) == {}
def test_a_real_gtt_backed_excess_is_still_reported_as_host_backed(monkeypatch):
"""The case #9314 and #11366 exist for must not regress: a torch total well
above the sysfs heap is a genuine host-backed window and keeps its figure."""
# See above: pin Linux so the platform gate does not answer.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
devices = [{"index": 0, "total_gb": 100.0, "_rocm_known_unified": True}]
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
monkeypatch.setattr(
hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {"0000:03:00.0": (1.0, 8.0)}
)
assert hw._rocm_linux_shared_pool_host_gb_by_index(devices) == {0: 92.0}
def test_a_discrete_rocm_card_is_never_given_a_host_backed_figure(monkeypatch):
"""Gated on `_rocm_known_unified`, so a dGPU keeps its discrete rendering."""
# See above: pin Linux so the platform gate does not answer.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
devices = [{"index": 0, "total_gb": 24.0}]
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
monkeypatch.setattr(
hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {"0000:03:00.0": (1.0, 24.0)}
)
assert hw._rocm_linux_shared_pool_host_gb_by_index(devices) == {}
def test_a_partitioned_device_keeps_its_split_unknown(monkeypatch):
"""A partitioned device keeps its split unknown; sysfs showing the whole card is a scope change."""
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
devices = [{"index": 0, "total_gb": 24.0, "_rocm_known_unified": True}]
monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: {0: "0000:03:00.0"})
# sysfs reports the whole 192 GiB card, torch reports a 24 GiB partition.
monkeypatch.setattr(
hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {"0000:03:00.0": (1.0, 192.0)}
)
assert hw._rocm_linux_shared_pool_host_gb_by_index(devices) == {}