Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read ZE_AFFINITY_MASK the way Level Zero does (#13528)

Daniel Hance24361df5e8
diff --git a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py
index 63f05af8acb..028a8b73ff3 100644
--- a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py
+++ b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py
@@ -1343,7 +1343,7 @@ def test_a_healthy_xpu_wheel_is_not_called_unavailable(monkeypatch):
assert hw.classify_torch_build() == "torch_cuda_unavailable"
[email protected]("mask", ["", " ", "-1"])
[email protected]("mask", [" ", "-1"])
def test_an_emptied_xpu_mask_is_a_deliberate_hide(monkeypatch, mask):
import sys
@@ -1360,6 +1360,22 @@ def test_an_emptied_xpu_mask_is_a_deliberate_hide(monkeypatch, mask):
assert hw.classify_torch_build() == "torch_cuda_unavailable"
[email protected]("mask", ["", "default"])
+def test_an_empty_or_default_xpu_mask_hides_nothing(monkeypatch, mask):
+ """Level Zero reads "" and "default" as no mask (isAffinityMaskSet): every Intel GPU stays visible."""
+ import sys
+
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch("cuda_dead"))
+ monkeypatch.setattr(
+ hw,
+ "get_physical_gpu_inventory",
+ lambda **_kw: {"available": True, "devices": [{"vendor": "intel"}]},
+ )
+ monkeypatch.setenv("ZE_AFFINITY_MASK", mask)
+ assert hw._vendors_masked_off() == set()
+ assert hw.classify_torch_build() == "torch_cuda_unavailable"
+
+
def test_the_xpu_mask_is_ignored_on_an_nvidia_only_host(monkeypatch):
import sys
@@ -1662,7 +1678,7 @@ def test_a_recovery_re_measures_torch_rather_than_reusing_the_old_answer(monkeyp
def test_a_vendor_mask_does_not_hide_another_vendors_card(monkeypatch):
- """A hybrid NVIDIA + Arc host with ZE_AFFINITY_MASK="".
+ """A hybrid NVIDIA + Arc host with ZE_AFFINITY_MASK="-1".
The mask hides the Arc and nothing else, but it was cancelling the whole
classification, so a CPU-only wheel went unreported for the NVIDIA card the user
@@ -1672,7 +1688,7 @@ def test_a_vendor_mask_does_not_hide_another_vendors_card(monkeypatch):
import sys
monkeypatch.setitem(sys.modules, "torch", _fake_torch("cpu"))
- monkeypatch.setenv("ZE_AFFINITY_MASK", "")
+ monkeypatch.setenv("ZE_AFFINITY_MASK", "-1")
hybrid = [
{"vendor": "nvidia", "name": "NVIDIA RTX A4000"},
{"vendor": "intel", "name": "Intel(R) Arc(TM) A770"},
diff --git a/studio/backend/tests/test_ze_affinity_mask_driver_semantics.py b/studio/backend/tests/test_ze_affinity_mask_driver_semantics.py
new file mode 100644
index 00000000000..7da09fced05
--- /dev/null
+++ b/studio/backend/tests/test_ze_affinity_mask_driver_semantics.py
@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""ZE_AFFINITY_MASK read the way Level Zero reads it. Expected values come from the real
+intel/compute-runtime 26.35 driver with three spoofed COMPOSITE root devices."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+import utils.hardware.hardware as hw
+
+_INTEL = {"vendor": "intel", "source": "sysfs-drm"}
+
+
+def xpu_host(monkeypatch):
+ def _set(
+ mask,
+ *,
+ roots = 3,
+ sysfs = True,
+ torch_count = None,
+ ):
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.XPU)
+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: roots)
+ monkeypatch.setattr(hw.sys, "platform", "linux")
+ monkeypatch.setattr(
+ hw,
+ "_linux_drm_sysfs_records",
+ lambda **_kw: [dict(_INTEL, index = i) for i in range(roots)] if sysfs else None,
+ )
+ monkeypatch.setenv("ZE_FLAT_DEVICE_HIERARCHY", "COMPOSITE")
+ if mask is None:
+ monkeypatch.delenv("ZE_AFFINITY_MASK", raising = False)
+ else:
+ monkeypatch.setenv("ZE_AFFINITY_MASK", mask)
+ monkeypatch.setattr(hw, "_visible_gpu_count", None)
+
+ def _count():
+ if torch_count is None:
+ raise RuntimeError("no torch")
+ return torch_count
+
+ monkeypatch.setitem(
+ __import__("sys").modules,
+ "torch",
+ types.SimpleNamespace(xpu = types.SimpleNamespace(device_count = _count)),
+ )
+
+ return _set
+
+
+ ("mask", "roots"),
+ [
+ (None, [0, 1, 2]),
+ ("", [0, 1, 2]),
+ ("default", [0, 1, 2]),
+ ("0", [0]),
+ ("2", [2]),
+ ("1,0", [0, 1]),
+ ("2,0", [0, 2]),
+ ("0,0", [0]),
+ ("0,5", [0]),
+ ("0x1", [1]),
+ ("01", [1]),
+ ("+1", [1]),
+ ("+0x1", [1]),
+ ("-0", [0]),
+ ("+2,0", [0, 2]),
+ ("-1,1", [1]),
+ ],
+)
+def test_composite_roots_match_the_driver(xpu_host, mask, roots):
+ xpu_host(mask)
+ spec = hw._get_parent_visible_gpu_spec()
+ assert spec["numeric_ids"] == roots
+ assert spec["supports_explicit_gpu_ids"] is True
+ assert hw.get_visible_gpu_count() == len(roots)
+
+
[email protected]("mask", ["", "default"])
+def test_an_empty_or_default_mask_is_unset(xpu_host, mask):
+ xpu_host(mask)
+ assert hw._ze_affinity_mask() is None
+ assert hw._mask_is_emptied("ZE_AFFINITY_MASK") is False
+ assert hw._get_parent_visible_gpu_spec()["raw"] is None
+ assert hw._backend_visible_devices_env() is None
+
+
[email protected]("mask", ["-1", " ", "Default"])
+def test_masks_the_driver_does_not_read_as_unset_keep_their_meaning(xpu_host, mask):
+ """ "-1" is out of range and selects nothing; whitespace and "Default" make stoul abort."""
+ xpu_host(mask)
+ assert hw._ze_affinity_mask() == mask
+ assert hw._mask_is_emptied("ZE_AFFINITY_MASK") is (mask.strip() in ("", "-1"))
+
+
+def test_a_cuda_mask_still_reads_empty_as_none(monkeypatch):
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
+ assert hw._mask_is_emptied("CUDA_VISIBLE_DEVICES") is True
+
+
+def test_a_high_root_is_kept_when_it_is_the_only_one_visible(xpu_host):
+ """torch.xpu.device_count() is filtered by the mask, so it cannot bound root ids."""
+ xpu_host("2", torch_count = 1)
+ assert hw._get_parent_visible_gpu_spec()["numeric_ids"] == [2]
+ assert hw.get_visible_gpu_count() == 1
+
+
+def test_without_a_sysfs_bound_out_of_range_ids_are_kept(xpu_host):
+ xpu_host("0,5", sysfs = False)
+ assert hw._get_parent_visible_gpu_spec()["numeric_ids"] == [0, 5]
+
+
+def test_flat_tile_ids_are_not_bounded_by_cards(xpu_host, monkeypatch):
+ """One two-tile card exposes ids 0 and 1 under FLAT."""
+ xpu_host("0,1", roots = 1)
+ monkeypatch.setenv("ZE_FLAT_DEVICE_HIERARCHY", "FLAT")
+ assert hw.get_visible_gpu_count() == 2
+
+
+def test_torch_still_wins_the_visible_count(xpu_host):
+ xpu_host("", torch_count = 3)
+ assert hw.get_visible_gpu_count() == 3
+
+
+ ("mask", "roots"),
+ [
+ ("2.0,0.1,0.2", [0, 2]),
+ ("08", [0]),
+ ("1abc", [1]),
+ ("abc,1", [1]),
+ ("-1", []),
+ ("1, 0", [0, 1]),
+ ("-18446744073709551615", [1]),
+ ],
+)
+def test_parse_ze_mask_roots(monkeypatch, mask, roots):
+ monkeypatch.setattr(hw.sys, "platform", "linux")
+ assert hw._parse_ze_mask_roots(mask) == roots
+
+
+ ("platform", "mask", "roots"),
+ [
+ ("win32", "-4294967295", [1]), # 32-bit unsigned long
+ ("win32", "-1", []),
+ ("linux", "-4294967295", []),
+ ("win32", "-18446744073709551615", []), # out_of_range on Windows
+ ("linux", "1" + "0" * 5000, []), # past Python's int-digit limit
+ ("linux", "\u20031", []), # not C whitespace
+ ("linux", "\t1", [1]),
+ ("linux", "0" * 40 + "1", [1]),
+ ("linux", "0b10", [2]), # C23 strtoul, linked by compute-runtime 26.35
+ ("linux", "-0b1", []),
+ ("linux", "0b", [0]),
+ ],
+)
+def test_parse_ze_mask_roots_follows_the_platform_stoul(monkeypatch, platform, mask, roots):
+ monkeypatch.setattr(hw.sys, "platform", platform)
+ assert hw._parse_ze_mask_roots(mask) == roots
+
+
+def test_non_linux_hosts_have_no_root_bound(monkeypatch):
+ monkeypatch.setattr(hw.sys, "platform", "win32")
+ assert hw._intel_root_device_bound() is None
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 93c96032baf..9db3a07f414 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -674,9 +674,15 @@ _VISIBILITY_MASK_VENDORS: Dict[str, frozenset] = {
}
+def _ze_affinity_mask() -> Optional[str]:
+ """ZE_AFFINITY_MASK as Level Zero reads it: None when unset, empty or "default", which isAffinityMaskSet treats alike (every device visible)."""
+ value = os.environ.get("ZE_AFFINITY_MASK")
+ return None if value in (None, "", "default") else value
+
+
def _mask_is_emptied(var: str) -> bool:
- """True when ``var`` is set to a value that hides every device it addresses: set-but-empty and "-1". A mask NAMING devices is not this, since that host expects those devices to work."""
- value = os.environ.get(var)
+ """True when ``var`` is set to a value that hides every device it addresses: set-but-empty and "-1". A mask NAMING devices is not this, since that host expects those devices to work. An empty ZE_AFFINITY_MASK hides nothing."""
+ value = _ze_affinity_mask() if var == "ZE_AFFINITY_MASK" else os.environ.get(var)
return value is not None and value.strip() in ("", "-1")
@@ -1463,7 +1469,7 @@ def _detect_hardware_locked() -> DeviceType:
import torch
# Prefer XPU on UNSLOTH_FORCE_XPU=1, or ZE_AFFINITY_MASK set with CUDA hidden/unavailable. A bare mask is NOT enough (it can leak from unrelated Intel tooling); torch.xpu must report a device.
- ze_mask = os.environ.get("ZE_AFFINITY_MASK")
+ ze_mask = _ze_affinity_mask()
cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
cuda_hidden = cvd is not None and cvd.strip() in ("", "-1")
force_xpu = os.environ.get("UNSLOTH_FORCE_XPU") == "1"
@@ -2663,20 +2669,47 @@ def _xpu_hierarchy_is_composite() -> bool:
return hierarchy == "COMPOSITE"
-def _parse_ze_mask_roots(mask: str) -> list[int]:
- """Parse ZE_AFFINITY_MASK into ordered root device IDs, one per token, preserving order and duplicates so logical ordinals map 1-to-1 onto physical root IDs ("2.0,0.1,0.2" -> [2, 0, 0]). Only meaningful in COMPOSITE hierarchy, which callers must gate on."""
- roots: list[int] = []
- if not mask:
- return roots
- for token in mask.split(","):
- token = token.strip()
- if not token:
+# The prefix C stoul(entry, nullptr, 0) consumes: sign, then hex, binary (C23 strtoul, which current
+# compute-runtime builds link), octal or decimal ("08" -> 0, "+1abc" -> 1, "0b10" -> 2).
+_ZE_MASK_ROOT_RE = re.compile(
+ r"([+-]?)(?:0[xX]([0-9a-fA-F]+)|0[bB]([01]+)|(0[0-7]*)|([1-9][0-9]*))"
+)
+
+
+def _parse_ze_mask_roots(mask: str, root_count: Optional[int] = None) -> list[int]:
+ """Root device IDs a ZE_AFFINITY_MASK selects, filtered the way Level Zero does: in root order with duplicates collapsed ("2,0,0" -> [0, 2]), entries parsed like stoul base 0 ("0x1", "01" and "+1" -> 1), ids at or past ``root_count`` dropped. Only meaningful in COMPOSITE hierarchy, which callers must gate on."""
+ roots: set = set()
+ # stoul's unsigned long: 32-bit on Windows (LLP64), 64-bit elsewhere.
+ ulong_limit = 2**32 if sys.platform == "win32" else 2**64
+ for token in (mask or "").split(","):
+ # C isspace only: stoul does not skip Unicode whitespace.
+ match = _ZE_MASK_ROOT_RE.match(token.strip(" \t\n\v\f\r").split(".", 1)[0])
+ if not match:
+ continue
+ sign, hex_digits, binary, octal, decimal = match.groups()
+ digits = hex_digits or binary or octal or decimal
+ # More digits than any unsigned long in any base: stoul throws out_of_range.
+ if len(digits.lstrip("0")) > 64:
continue
- root = token.split(".", 1)[0]
- # isdecimal(), not isdigit(), which accepts Unicode superscripts that then crash int().
- if root.isdecimal():
- roots.append(int(root))
- return roots
+ root = int(digits, 16 if hex_digits else 2 if binary else 8 if octal else 10)
+ if root >= ulong_limit:
+ continue
+ # Negation is unsigned: "-1" wraps past every root, "-0" is 0.
+ if sign == "-":
+ root = -root % ulong_limit
+ # No host has 2**16 roots, so a wrapped value is out of range even without a bound.
+ if root < 2**16:
+ roots.add(root)
+ return sorted(root for root in roots if root_count is None or root < root_count)
+
+
+def _intel_root_device_bound() -> Optional[int]:
+ """Upper bound on Level Zero root devices: every Intel card the Linux DRM drivers bound, allowlisted or not. None when unknown; torch.xpu.device_count() cannot stand in, since it is already filtered by the mask."""
+ if not sys.platform.startswith("linux"):
+ return None
+ records = _linux_drm_sysfs_records(distinguish_failure = True)
+ count = sum(1 for record in records or () if record.get("vendor") == "intel")
+ return count or None
def _smi_query(func_name: str, *args, **kwargs) -> Optional[Dict[str, Any]]:
@@ -4779,7 +4812,7 @@ _visible_gpu_count: Optional[int] = None
def _get_parent_visible_gpu_spec() -> Dict[str, Any]:
# On Intel XPU, visibility is controlled by ZE_AFFINITY_MASK (Level Zero), not CUDA_VISIBLE_DEVICES.
if get_device() == DeviceType.XPU:
- xpu_mask_raw = os.environ.get("ZE_AFFINITY_MASK")
+ xpu_mask_raw = _ze_affinity_mask()
composite = _xpu_hierarchy_is_composite()
if xpu_mask_raw is None:
@@ -4829,10 +4862,10 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]:
"supports_explicit_gpu_ids": False,
}
- # COMPOSITE + pure numeric (subdevice handled above). _parse_ze_mask_roots maps to root GPU IDs, dropping non-decimal tokens so "*"/"GPU-uuid" -> [].
- roots_with_dupes = _parse_ze_mask_roots(xpu_mask)
- if not roots_with_dupes:
- # Unparseable mask ("*", "GPU-uuid"): cannot map to physical root IDs.
+ # COMPOSITE + pure numeric (subdevice handled above). _parse_ze_mask_roots maps to root GPU IDs, dropping non-numeric tokens so "*"/"GPU-uuid" -> [].
+ roots = _parse_ze_mask_roots(xpu_mask, _intel_root_device_bound())
+ if not roots:
+ # Unparseable ("*", "GPU-uuid") or out of range ("5" on 3 roots): no root to map.
return {
"raw": xpu_mask,
"numeric_ids": None,
@@ -4841,7 +4874,7 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]:
return {
"raw": xpu_mask,
- "numeric_ids": roots_with_dupes,
+ "numeric_ids": roots,
"supports_explicit_gpu_ids": True,
}
@@ -6218,7 +6251,7 @@ def get_physical_gpu_count() -> int:
def _backend_visible_devices_env() -> Optional[str]:
"""The raw visibility env string that applies to this backend: ZE_AFFINITY_MASK on XPU, and HIP_/ROCR_VISIBLE_DEVICES ahead of CUDA_VISIBLE_DEVICES on ROCm. Mirrors _get_parent_visible_gpu_spec, so backend_cuda_visible_devices reports the value actually narrowing the visible set."""
if get_device() == DeviceType.XPU:
- return os.environ.get("ZE_AFFINITY_MASK")
+ return _ze_affinity_mask()
if IS_ROCM:
return _get_parent_visible_gpu_spec().get("raw")
return os.environ.get("CUDA_VISIBLE_DEVICES")
@@ -6689,7 +6722,7 @@ def get_visible_gpu_count() -> int:
# Prefer torch.xpu.device_count() on Intel XPU: the Level Zero runtime interprets ZE_AFFINITY_MASK semantics correctly (subdevice syntax "0.0,0.1" collapses onto one root GPU).
if get_device() == DeviceType.XPU:
- xpu_mask_raw = os.environ.get("ZE_AFFINITY_MASK")
+ xpu_mask_raw = _ze_affinity_mask()
xpu_mask_set = xpu_mask_raw is not None
xpu_visible = (xpu_mask_raw or "").strip()
if xpu_mask_set and xpu_visible == "":
@@ -6710,9 +6743,11 @@ def get_visible_gpu_count() -> int:
# Documented wildcard: all physical XPUs visible.
_visible_gpu_count = get_physical_gpu_count()
else:
- roots = _parse_ze_mask_roots(xpu_visible)
+ # FLAT entries are tile handles (one PVC card exposes two), so only COMPOSITE ids are bounded by cards.
+ bound = _intel_root_device_bound() if _xpu_hierarchy_is_composite() else None
+ roots = _parse_ze_mask_roots(xpu_visible, bound)
# Non-parseable masks (",,,", "GPU-abc") yield an empty roots list, treated as 0 visible devices rather than "all visible": no evidence the whole fleet was intended.
- _visible_gpu_count = len(set(roots))
+ _visible_gpu_count = len(roots)
else:
_visible_gpu_count = get_physical_gpu_count()
return _visible_gpu_count
diff --git a/tests/studio/test_xpu_spoof_pipeline.py b/tests/studio/test_xpu_spoof_pipeline.py
index 3570efa71e7..6d28f015b07 100644
--- a/tests/studio/test_xpu_spoof_pipeline.py
+++ b/tests/studio/test_xpu_spoof_pipeline.py
@@ -388,8 +388,15 @@ def test_get_visible_gpu_count_uses_device_count(spoof_xpu):
assert hw.get_visible_gpu_count() == 2
-def test_get_visible_gpu_count_empty_mask_is_zero(spoof_xpu):
- hw, _ = spoof_xpu(ze_mask = "")
+def test_get_visible_gpu_count_empty_mask_is_no_mask(spoof_xpu):
+ """Level Zero reads an empty ZE_AFFINITY_MASK as unset (isAffinityMaskSet), so every device stays visible."""
+ hw, _ = spoof_xpu(ze_mask = "", device_count = 2)
+ hw.detect_hardware()
+ assert hw.get_visible_gpu_count() == 2
+
+
+def test_get_visible_gpu_count_blank_mask_is_zero(spoof_xpu):
+ hw, _ = spoof_xpu(ze_mask = " ")
hw.detect_hardware()
assert hw.get_visible_gpu_count() == 0