Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""ROCm arch gate: selection must not pick a device the wheel lacks kernels for (#7669)."""
from __future__ import annotations
import os
import sys
import types
from unittest.mock import patch
import pytest
import utils.hardware.hardware as _hw_module
from utils.hardware.hardware import (
DeviceType,
apply_gpu_ids,
auto_select_gpu_ids,
gpu_ids_with_torch_kernels,
rocm_gpu_ids_without_torch_kernels,
)
# gfx1036 (the Raphael iGPU in #7669) is deliberately absent: that is the bug.
GFX110X = ["gfx1100", "gfx1101", "gfx1102", "gfx1103"]
# Cleared wholesale so a developer's own ROCm env cannot decide what these assert.
_MASK_VARS = (
"CUDA_VISIBLE_DEVICES",
"HIP_VISIBLE_DEVICES",
"ROCR_VISIBLE_DEVICES",
"GPU_DEVICE_ORDINAL",
"ZE_AFFINITY_MASK",
)
def _props(
arch = "",
*,
attr = "gcnArchName",
name = "AMD Radeon RX 7700 XT",
):
p = types.SimpleNamespace(name = name)
if arch:
setattr(p, attr, arch)
return p
def _fake_torch(
devices,
*,
arch_list = GFX110X,
vendor = "amd",
available = True,
device_count = None,
):
torch = types.ModuleType("torch")
if vendor == "amd":
torch.version = types.SimpleNamespace(hip = "7.13.99004", cuda = None)
torch.__version__ = "2.11.0+rocm7.13.0"
elif vendor == "amd_sdk":
torch.version = types.SimpleNamespace()
torch.__version__ = "2.6.0+rocm6.4"
else:
torch.version = types.SimpleNamespace(hip = None, cuda = "12.4")
torch.__version__ = "2.6.0+cu124"
def _get_device_properties(ordinal):
entry = devices[ordinal]
if isinstance(entry, Exception):
raise entry
return entry
def _get_arch_list():
if isinstance(arch_list, Exception):
raise arch_list
return list(arch_list)
torch.cuda = types.SimpleNamespace(
is_available = lambda: available,
device_count = lambda: len(devices) if device_count is None else device_count,
get_arch_list = _get_arch_list,
get_device_properties = _get_device_properties,
)
return torch
@pytest.fixture
def no_mask(monkeypatch):
for var in _MASK_VARS:
monkeypatch.delenv(var, raising = False)
monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 2)
@pytest.fixture(autouse = True)
def _no_device_ordinal(monkeypatch):
monkeypatch.delenv("GPU_DEVICE_ORDINAL", raising = False)
@pytest.fixture(autouse = True)
def _detection_is_declared_not_detected(monkeypatch):
"""Keeps detect_hardware() from latching IS_ROCM off the fake AMD torch session-wide."""
monkeypatch.setattr(_hw_module, "DEVICE", DeviceType.CUDA)
monkeypatch.setattr(_hw_module, "IS_ROCM", _hw_module.IS_ROCM)
def _install(monkeypatch, torch):
monkeypatch.setitem(sys.modules, "torch", torch)
class TestTheReportedHost:
"""#7669: RX 7700 XT (gfx1101) + Raphael iGPU (gfx1036), gfx110X wheel."""
def test_the_uncovered_igpu_is_dropped(self, monkeypatch, no_mask):
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
assert rocm_gpu_ids_without_torch_kernels() == {1}
def test_a_fully_covered_host_drops_nothing(self, monkeypatch, no_mask):
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1100")]))
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_the_xnack_suffix_does_not_defeat_the_match(self, monkeypatch, no_mask):
_install(
monkeypatch,
_fake_torch([_props("gfx1101:sramecc-:xnack-"), _props("gfx1036:xnack-")]),
)
assert rocm_gpu_ids_without_torch_kernels() == {1}
class TestArchSpellings:
"""Reading only gcnArchName makes the gate a no-op on AMD SDK wheels."""
@pytest.mark.parametrize("attr", ["gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"])
def test_every_spelling_is_read(self, monkeypatch, no_mask, attr):
_install(
monkeypatch,
_fake_torch([_props("gfx1101", attr = attr), _props("gfx1036", attr = attr)]),
)
assert rocm_gpu_ids_without_torch_kernels() == {1}
class TestFailsOpen:
"""Uncertainty keeps the pre-gate selection: a working machine on CPU is worse."""
def test_a_cuda_wheel_is_inert(self, monkeypatch, no_mask):
# PTX JIT covers archs not listed, so filtering on NVIDIA drops working cards.
_install(
monkeypatch,
_fake_torch([_props("gfx1036")], arch_list = ["sm_80", "sm_90"], vendor = "nvidia"),
)
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_an_amd_sdk_wheel_still_gates(self, monkeypatch, no_mask):
_install(
monkeypatch,
_fake_torch([_props("gfx1101"), _props("gfx1036")], vendor = "amd_sdk"),
)
assert rocm_gpu_ids_without_torch_kernels() == {1}
def test_every_device_uncovered_keeps_them_all(self, monkeypatch, no_mask):
# Dropping all of them hands the caller an empty selection: silent CPU.
_install(monkeypatch, _fake_torch([_props("gfx900"), _props("gfx906")]))
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_an_unreadable_arch_list_is_unknown(self, monkeypatch, no_mask):
_install(
monkeypatch,
_fake_torch([_props("gfx1036")], arch_list = RuntimeError("no arch list")),
)
assert rocm_gpu_ids_without_torch_kernels() == set()
@pytest.mark.parametrize(
"arch_list",
[[], ["gfx11-generic"], ["gfx110X"], ["", " "], ["garbage"], ["sm_90"]],
ids = ["empty", "generic", "family_label", "blank", "garbage", "cuda_tokens"],
)
def test_a_non_concrete_arch_list_is_unknown(self, monkeypatch, no_mask, arch_list):
_install(monkeypatch, _fake_torch([_props("gfx1101")], arch_list = arch_list))
assert rocm_gpu_ids_without_torch_kernels() == set()
@pytest.mark.parametrize(
"arch_list",
[
["gfx1100", "gfx11-generic"],
["gfx11-generic", "gfx1100"],
["gfx900", "gfx110X", "gfx1100"],
],
ids = ["generic_last", "generic_first", "family_label_between"],
)
def test_one_non_concrete_token_disables_the_whole_list(self, monkeypatch, no_mask, arch_list):
# The concrete subset alone marks gfx1101 uncovered and drops it (#7624).
_install(
monkeypatch,
_fake_torch([_props("gfx1100"), _props("gfx1101")], arch_list = arch_list),
)
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_a_device_with_no_readable_arch_is_kept(self, monkeypatch, no_mask):
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("")]))
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_properties_raising_is_not_fatal(self, monkeypatch, no_mask):
_install(
monkeypatch,
_fake_torch([_props("gfx1101"), RuntimeError("cannot describe device")]),
)
assert rocm_gpu_ids_without_torch_kernels() == set()
@pytest.mark.parametrize(
"third",
[_props(""), RuntimeError("cannot describe device")],
ids = ["no_arch_attribute", "properties_raise"],
)
def test_one_unreadable_device_does_not_spare_a_known_uncovered_one(
self, monkeypatch, no_mask, third
):
# Discarding the whole answer would put the uncovered GPU 1 back (#8792).
monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 3)
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036"), third]))
assert rocm_gpu_ids_without_torch_kernels() == {1}
@pytest.mark.parametrize(
"second",
[_props(""), RuntimeError("cannot describe device")],
ids = ["no_arch_attribute", "properties_raise"],
)
def test_an_unreadable_device_is_not_an_all_uncovered_host(self, monkeypatch, no_mask, second):
_install(monkeypatch, _fake_torch([_props("gfx1036"), second]))
assert rocm_gpu_ids_without_torch_kernels() == {0}
def test_no_cuda_runtime_is_inert(self, monkeypatch, no_mask):
_install(monkeypatch, _fake_torch([_props("gfx1036")], available = False))
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_a_uuid_mask_cannot_be_named_back(self, monkeypatch, no_mask):
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "GPU-DEADBEEFDEADBEEF,0")
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_gpu_device_ordinal_renumbers_the_map_away(self, monkeypatch, no_mask):
# No visibility spec reads it, so ordinal 1 is not physical 1.
monkeypatch.setenv("GPU_DEVICE_ORDINAL", "1,0")
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_stacked_rocr_and_cuda_masks_renumber_the_map_away(self, monkeypatch, no_mask):
# Ordinal 1 is phys1, but the spec sees only ROCr and would name it physical 0.
monkeypatch.setattr(sys, "platform", "linux")
monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", True)
monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "2,0,1")
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1,2")
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
assert rocm_gpu_ids_without_torch_kernels() == set()
# torch.cuda.get_arch_list() of AMD's 2.11.0+rocm7.14.0 multi-arch wheel, read from its libtorch_python.
MULTIARCH = (
"gfx1010 gfx1011 gfx1012 gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036 "
"gfx1100 gfx1101 gfx1102 gfx1103 gfx1200 gfx1201 gfx1150 gfx1151 gfx1152 gfx1153 "
"gfx908 gfx90a gfx942 gfx950 gfx1250"
).split()
class TestMultiArchKernelPacks:
"""#8847: Windows multi-arch torch installed with only the dGPU's device pack."""
def _torch(self, tmp_path, devices, packs):
torch = _fake_torch(devices, arch_list = MULTIARCH)
root = tmp_path / "torch"
root.mkdir()
torch.__file__ = str(root / "__init__.py")
if packs is not None:
(root / ".kpack").mkdir()
for name in packs:
(root / ".kpack" / name).write_bytes(b"")
return torch
def test_an_igpu_without_its_pack_is_dropped(self, monkeypatch, no_mask, tmp_path):
torch = self._torch(
tmp_path, [_props("gfx1036"), _props("gfx1101")], ["torch_gfx1101.kpack"]
)
_install(monkeypatch, torch)
assert rocm_gpu_ids_without_torch_kernels() == {0}
def test_both_packs_installed_drops_nothing(self, monkeypatch, no_mask, tmp_path):
packs = ["torch_gfx1036.kpack", "torch_gfx1101.kpack"]
_install(monkeypatch, self._torch(tmp_path, [_props("gfx1036"), _props("gfx1101")], packs))
assert rocm_gpu_ids_without_torch_kernels() == set()
@pytest.mark.parametrize(
"packs",
[None, [], ["torch_gfx110x.kpack", "torch_gfx1101.kpack"], ["README.txt"]],
ids = ["no_pack_dir", "empty_pack_dir", "family_pack", "no_pack_files"],
)
def test_an_unreadable_pack_layout_keeps_the_arch_list(
self, monkeypatch, no_mask, tmp_path, packs
):
_install(monkeypatch, self._torch(tmp_path, [_props("gfx1036"), _props("gfx1101")], packs))
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_an_explicit_pick_names_only_the_installed_packs(self, monkeypatch, no_mask, tmp_path):
torch = self._torch(
tmp_path, [_props("gfx1036"), _props("gfx1101")], ["torch_gfx1101.kpack"]
)
_install(monkeypatch, torch)
with pytest.raises(ValueError, match = r"kernels for gfx1101 only"):
_hw_module.reject_gpu_ids_without_torch_kernels([0])
class TestIdSpace:
"""Results are consumed as physical ids, so ordinals map through the mask."""
def test_ordinals_map_through_the_mask(self, monkeypatch, no_mask):
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,3")
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
assert rocm_gpu_ids_without_torch_kernels() == {3}
def _rocr_only(self, monkeypatch, platform):
monkeypatch.setattr(sys, "platform", platform)
monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", True)
monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 2)
for var in _MASK_VARS:
monkeypatch.delenv(var, raising = False)
monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "2,3")
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
def test_a_rocr_mask_maps_the_ordinals_on_linux(self, monkeypatch):
self._rocr_only(monkeypatch, "linux")
assert rocm_gpu_ids_without_torch_kernels() == {3}
def test_windows_ignores_a_stray_rocr_mask(self, monkeypatch):
# Windows HIP has no ROCr layer, so reading this mask excludes a nonexistent card.
self._rocr_only(monkeypatch, "win32")
assert rocm_gpu_ids_without_torch_kernels() == {1}
def test_windows_still_honours_a_hip_mask(self, monkeypatch):
self._rocr_only(monkeypatch, "win32")
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,3")
assert rocm_gpu_ids_without_torch_kernels() == {3}
class TestTheOrdinalToIdMapMustBeTotal:
"""device_count() freezes at torch init while the visible spec re-reads the env, so
naming an overflow ordinal into the physical namespace collides with a real id."""
def test_more_ordinals_than_ids_gates_nothing(self, monkeypatch, no_mask):
# Ordinal 2 has no id; reusing it as one would drop physical 2, the good card.
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,0")
_install(
monkeypatch,
_fake_torch(
[_props("gfx1101"), _props("gfx1036"), _props("gfx1036")],
device_count = 3,
),
)
assert rocm_gpu_ids_without_torch_kernels() == set()
def test_the_selector_keeps_the_covered_card(self, monkeypatch, no_mask):
# The regression this guards: an empty list is "no GPU", not "inherit".
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,0")
_install(
monkeypatch,
_fake_torch(
[_props("gfx1101"), _props("gfx1036"), _props("gfx1036")],
device_count = 3,
),
)
monkeypatch.setattr(_hw_module, "get_device", lambda: DeviceType.CUDA)
monkeypatch.setattr(
_hw_module,
"get_visible_gpu_utilization",
lambda: {
"devices": [
{"index": 0, "vram_total_gb": 16.0, "vram_used_gb": 1.0},
{"index": 2, "vram_total_gb": 32.0, "vram_used_gb": 1.0},
]
},
)
gpu_ids, _ = auto_select_gpu_ids("m", required_override_gb = 8.0)
assert gpu_ids == [2]
def test_amd_smi_undercounting_still_gates(self, monkeypatch, no_mask):
# No mask, so the short list is amd-smi missing the iGPU.
monkeypatch.setattr(_hw_module, "get_physical_gpu_count", lambda: 1)
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
assert rocm_gpu_ids_without_torch_kernels() == {1}
def test_an_id_named_twice_still_trips_the_all_uncovered_guard(self, monkeypatch, no_mask):
# Both ordinals are physical 0, so a deduplicated set would read as a partial drop.
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0,0")
_install(
monkeypatch,
_fake_torch([_props("gfx1036"), _props("gfx1036")]),
)
assert rocm_gpu_ids_without_torch_kernels() == set()
class TestSelectorWiring:
DEVICES = {
"devices": [
{"index": 0, "vram_total_gb": 12.0, "vram_used_gb": 0.0},
{"index": 1, "vram_total_gb": 12.0, "vram_used_gb": 0.0},
]
}
def _run(
self,
*,
uncovered,
devices = None,
required = (14.0, {"required_gb": 14.0}),
):
with (
patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
patch(
"utils.hardware.hardware.rocm_gpu_ids_without_torch_kernels",
return_value = uncovered,
),
patch(
"utils.hardware.hardware.estimate_required_model_memory_gb",
return_value = required,
),
patch(
"utils.hardware.hardware.get_visible_gpu_utilization",
return_value = self.DEVICES if devices is None else devices,
),
patch(
"utils.hardware.hardware.get_parent_visible_gpu_ids",
return_value = [0, 1],
),
):
return auto_select_gpu_ids("unsloth/test")
def test_the_ranked_path_never_offers_the_uncovered_gpu(self):
selected, _meta = self._run(uncovered = {1})
assert selected == [0]
def test_the_unestimatable_fallback_is_filtered(self):
selected, metadata = self._run(uncovered = {1}, required = (None, {}))
assert metadata["selection_mode"] == "fallback_all"
assert selected == [0]
def test_the_no_telemetry_fallback_is_filtered(self):
selected, metadata = self._run(uncovered = {1}, devices = {"devices": []})
assert metadata["selection_mode"] == "fallback_all"
assert selected == [0]
def test_a_covered_host_selects_exactly_as_before(self):
assert self._run(uncovered = set())[0] == self._run_ungated()
def _run_ungated(self):
with (
patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
patch(
"utils.hardware.hardware.estimate_required_model_memory_gb",
return_value = (14.0, {"required_gb": 14.0}),
),
patch(
"utils.hardware.hardware.get_visible_gpu_utilization",
return_value = self.DEVICES,
),
patch(
"utils.hardware.hardware.get_parent_visible_gpu_ids",
return_value = [0, 1],
),
):
return auto_select_gpu_ids("unsloth/test")[0]
class TestExportWorkerVisibility:
"""Export workers hide GPUs with missing torch kernels (#11870)."""
def test_the_reported_host_keeps_only_the_discrete_card(self, monkeypatch, no_mask):
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
assert gpu_ids_with_torch_kernels() == [0]
def test_a_covered_host_inherits_the_parent_visibility(self, monkeypatch, no_mask):
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1100")]))
assert gpu_ids_with_torch_kernels() is None
def test_a_cuda_host_inherits_the_parent_visibility(self, monkeypatch, no_mask):
_install(
monkeypatch,
_fake_torch([_props(""), _props("")], arch_list = ["sm_80"], vendor = "nvidia"),
)
assert gpu_ids_with_torch_kernels() is None
def test_only_the_uncovered_card_is_hidden(self, monkeypatch, no_mask):
monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 3)
_install(
monkeypatch,
_fake_torch([_props("gfx1101"), _props("gfx1036"), _props("gfx1100")]),
)
assert gpu_ids_with_torch_kernels() == [0, 2]
def test_a_card_amd_smi_missed_is_kept(self, monkeypatch, no_mask):
monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 1)
_install(monkeypatch, _fake_torch([_props("gfx1036"), _props("gfx1101")]))
assert rocm_gpu_ids_without_torch_kernels() == {0}
assert gpu_ids_with_torch_kernels() == [1]
def test_an_inherited_mask_keeps_its_physical_ids(self, monkeypatch, no_mask):
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,3")
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
assert gpu_ids_with_torch_kernels() == [2]
def test_load_checkpoint_hands_the_covered_ids_to_the_worker(
self, monkeypatch, no_mask, tmp_path
):
from core.export.orchestrator import ExportOrchestrator
_install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
backend = ExportOrchestrator()
spawned = {}
monkeypatch.setattr(backend, "_ensure_subprocess_alive", lambda: False)
monkeypatch.setattr(backend, "_spawn_subprocess", spawned.update)
monkeypatch.setattr(
backend, "_wait_response", lambda *a, **k: {"success": True, "message": "loaded"}
)
assert backend.load_checkpoint(str(tmp_path), load_in_4bit = True)[0]
assert spawned["resolved_gpu_ids"] == [0]
assert spawned["device_backend"] == DeviceType.CUDA.value
with patch.dict(os.environ):
monkeypatch.setattr(sys, "platform", "win32")
apply_gpu_ids(spawned["resolved_gpu_ids"], backend = spawned["device_backend"])
assert os.environ["HIP_VISIBLE_DEVICES"] == "0"
class TestThePinLandsOnTheKeptCard:
"""HIP indexes the agents an inherited ROCr mask left, so a raw physical id misses."""
def _rocr(
self,
monkeypatch,
mask,
*,
devices = ("gfx1101", "gfx1036"),
):
monkeypatch.setattr(sys, "platform", "linux")
monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", True)
monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 2)
_install(monkeypatch, _fake_torch([_props(arch) for arch in devices]))
os.environ.pop("HIP_VISIBLE_DEVICES", None)
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
os.environ["ROCR_VISIBLE_DEVICES"] = mask
def test_a_reordered_mask_pins_the_covered_card(self, monkeypatch):
# Physical id 1 verbatim picks ROCr agent 1, physical 0: the card just excluded.
with patch.dict(os.environ):
self._rocr(monkeypatch, "1,0")
assert rocm_gpu_ids_without_torch_kernels() == {0}
apply_gpu_ids([1], backend = DeviceType.CUDA.value)
assert os.environ["HIP_VISIBLE_DEVICES"] == "0"
assert os.environ["CUDA_VISIBLE_DEVICES"] == "0"
# Clearing it would hand the child every agent, the gfx1036 among them.
assert os.environ["ROCR_VISIBLE_DEVICES"] == "1,0"
def test_a_nonzero_mask_pins_in_range(self, monkeypatch):
# ROCR=2,3 leaves two agents: an untranslated HIP="3" is out of range.
with patch.dict(os.environ):
self._rocr(monkeypatch, "2,3")
assert rocm_gpu_ids_without_torch_kernels() == {3}
apply_gpu_ids([2], backend = DeviceType.CUDA.value)
assert os.environ["HIP_VISIBLE_DEVICES"] == "0"
def test_an_identity_mask_is_written_unchanged(self, monkeypatch):
with patch.dict(os.environ):
self._rocr(monkeypatch, "0,1")
apply_gpu_ids([1], backend = DeviceType.CUDA.value)
assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
assert os.environ["CUDA_VISIBLE_DEVICES"] == "1"
def test_an_inherited_hip_mask_is_already_relative(self, monkeypatch):
# HIP produced the ids, so translating again would map them twice.
with patch.dict(os.environ):
self._rocr(monkeypatch, "1,0")
os.environ["HIP_VISIBLE_DEVICES"] = "1,0"
apply_gpu_ids([1], backend = DeviceType.CUDA.value)
assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
def test_an_inherited_cuda_mask_is_already_relative(self, monkeypatch):
# rocclr already read this as the HIP mask; translating again maps it twice.
with patch.dict(os.environ):
self._rocr(monkeypatch, "1,0")
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
apply_gpu_ids([1], backend = DeviceType.CUDA.value)
assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
assert os.environ["CUDA_VISIBLE_DEVICES"] == "1"
def test_a_uuid_rocr_mask_is_left_alone(self, monkeypatch):
with patch.dict(os.environ):
self._rocr(monkeypatch, "GPU-DEADBEEFDEADBEEF,1")
apply_gpu_ids([1], backend = DeviceType.CUDA.value)
assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
def test_windows_writes_physical_ids(self, monkeypatch):
with patch.dict(os.environ):
self._rocr(monkeypatch, "1,0")
monkeypatch.setattr(sys, "platform", "win32")
apply_gpu_ids([1], backend = DeviceType.CUDA.value)
assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
def test_a_cuda_host_is_untouched(self, monkeypatch):
with patch.dict(os.environ):
monkeypatch.setattr(sys, "platform", "linux")
monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", False)
_install(monkeypatch, _fake_torch([_props("gfx1101")], vendor = "nvidia"))
for var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
os.environ.pop(var, None)
apply_gpu_ids([1, 3], backend = DeviceType.CUDA.value)
assert os.environ["CUDA_VISIBLE_DEVICES"] == "1,3"
assert "HIP_VISIBLE_DEVICES" not in os.environ
def test_a_stale_rocr_var_on_nvidia_does_not_move_the_pin(self, monkeypatch):
# Reading the var's presence as "this is ROCm" writes CUDA="0", not GPU 1.
with patch.dict(os.environ):
monkeypatch.setattr(sys, "platform", "linux")
monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", False)
_install(monkeypatch, _fake_torch([_props("")], vendor = "nvidia"))
os.environ.pop("HIP_VISIBLE_DEVICES", None)
os.environ["ROCR_VISIBLE_DEVICES"] = "1,0"
apply_gpu_ids([1], backend = DeviceType.CUDA.value)
assert os.environ["CUDA_VISIBLE_DEVICES"] == "1"