Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read ZE_AFFINITY_MASK the way Level Zero does (#13528)
Daniel Hance24361df5e8
diff --git a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.pyindex 63f05af8acb..028a8b73ff3 100644--- a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py+++ b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py@@ -1343,7 +1343,7 @@ def test_a_healthy_xpu_wheel_is_not_called_unavailable(monkeypatch):assert hw.classify_torch_build() == "torch_cuda_unavailable"[email protected]("mask", ["", " ", "-1"])[email protected]("mask", [" ", "-1"])def test_an_emptied_xpu_mask_is_a_deliberate_hide(monkeypatch, mask):import sys@@ -1360,6 +1360,22 @@ def test_an_emptied_xpu_mask_is_a_deliberate_hide(monkeypatch, mask):assert hw.classify_torch_build() == "torch_cuda_unavailable"[email protected]("mask", ["", "default"])+def test_an_empty_or_default_xpu_mask_hides_nothing(monkeypatch, mask):+ """Level Zero reads "" and "default" as no mask (isAffinityMaskSet): every Intel GPU stays visible."""+ import sys++ monkeypatch.setitem(sys.modules, "torch", _fake_torch("cuda_dead"))+ monkeypatch.setattr(+ hw,+ "get_physical_gpu_inventory",+ lambda **_kw: {"available": True, "devices": [{"vendor": "intel"}]},+ )+ monkeypatch.setenv("ZE_AFFINITY_MASK", mask)+ assert hw._vendors_masked_off() == set()+ assert hw.classify_torch_build() == "torch_cuda_unavailable"++def test_the_xpu_mask_is_ignored_on_an_nvidia_only_host(monkeypatch):import sys@@ -1662,7 +1678,7 @@ def test_a_recovery_re_measures_torch_rather_than_reusing_the_old_answer(monkeypdef test_a_vendor_mask_does_not_hide_another_vendors_card(monkeypatch):- """A hybrid NVIDIA + Arc host with ZE_AFFINITY_MASK="".+ """A hybrid NVIDIA + Arc host with ZE_AFFINITY_MASK="-1".The mask hides the Arc and nothing else, but it was cancelling the wholeclassification, so a CPU-only wheel went unreported for the NVIDIA card the user@@ -1672,7 +1688,7 @@ def test_a_vendor_mask_does_not_hide_another_vendors_card(monkeypatch):import sysmonkeypatch.setitem(sys.modules, "torch", _fake_torch("cpu"))- monkeypatch.setenv("ZE_AFFINITY_MASK", "")+ monkeypatch.setenv("ZE_AFFINITY_MASK", "-1")hybrid = [{"vendor": "nvidia", "name": "NVIDIA RTX A4000"},{"vendor": "intel", "name": "Intel(R) Arc(TM) A770"},diff --git a/studio/backend/tests/test_ze_affinity_mask_driver_semantics.py b/studio/backend/tests/test_ze_affinity_mask_driver_semantics.pynew file mode 100644index 00000000000..7da09fced05--- /dev/null+++ b/studio/backend/tests/test_ze_affinity_mask_driver_semantics.py@@ -0,0 +1,171 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""ZE_AFFINITY_MASK read the way Level Zero reads it. Expected values come from the real+intel/compute-runtime 26.35 driver with three spoofed COMPOSITE root devices."""++from __future__ import annotations++import types++import pytest++import utils.hardware.hardware as hw++_INTEL = {"vendor": "intel", "source": "sysfs-drm"}+++def xpu_host(monkeypatch):+ def _set(+ mask,+ *,+ roots = 3,+ sysfs = True,+ torch_count = None,+ ):+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.XPU)+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: roots)+ monkeypatch.setattr(hw.sys, "platform", "linux")+ monkeypatch.setattr(+ hw,+ "_linux_drm_sysfs_records",+ lambda **_kw: [dict(_INTEL, index = i) for i in range(roots)] if sysfs else None,+ )+ monkeypatch.setenv("ZE_FLAT_DEVICE_HIERARCHY", "COMPOSITE")+ if mask is None:+ monkeypatch.delenv("ZE_AFFINITY_MASK", raising = False)+ else:+ monkeypatch.setenv("ZE_AFFINITY_MASK", mask)+ monkeypatch.setattr(hw, "_visible_gpu_count", None)++ def _count():+ if torch_count is None:+ raise RuntimeError("no torch")+ return torch_count++ monkeypatch.setitem(+ __import__("sys").modules,+ "torch",+ types.SimpleNamespace(xpu = types.SimpleNamespace(device_count = _count)),+ )++ return _set+++ ("mask", "roots"),+ [+ (None, [0, 1, 2]),+ ("", [0, 1, 2]),+ ("default", [0, 1, 2]),+ ("0", [0]),+ ("2", [2]),+ ("1,0", [0, 1]),+ ("2,0", [0, 2]),+ ("0,0", [0]),+ ("0,5", [0]),+ ("0x1", [1]),+ ("01", [1]),+ ("+1", [1]),+ ("+0x1", [1]),+ ("-0", [0]),+ ("+2,0", [0, 2]),+ ("-1,1", [1]),+ ],+)+def test_composite_roots_match_the_driver(xpu_host, mask, roots):+ xpu_host(mask)+ spec = hw._get_parent_visible_gpu_spec()+ assert spec["numeric_ids"] == roots+ assert spec["supports_explicit_gpu_ids"] is True+ assert hw.get_visible_gpu_count() == len(roots)++[email protected]("mask", ["", "default"])+def test_an_empty_or_default_mask_is_unset(xpu_host, mask):+ xpu_host(mask)+ assert hw._ze_affinity_mask() is None+ assert hw._mask_is_emptied("ZE_AFFINITY_MASK") is False+ assert hw._get_parent_visible_gpu_spec()["raw"] is None+ assert hw._backend_visible_devices_env() is None++[email protected]("mask", ["-1", " ", "Default"])+def test_masks_the_driver_does_not_read_as_unset_keep_their_meaning(xpu_host, mask):+ """ "-1" is out of range and selects nothing; whitespace and "Default" make stoul abort."""+ xpu_host(mask)+ assert hw._ze_affinity_mask() == mask+ assert hw._mask_is_emptied("ZE_AFFINITY_MASK") is (mask.strip() in ("", "-1"))+++def test_a_cuda_mask_still_reads_empty_as_none(monkeypatch):+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")+ assert hw._mask_is_emptied("CUDA_VISIBLE_DEVICES") is True+++def test_a_high_root_is_kept_when_it_is_the_only_one_visible(xpu_host):+ """torch.xpu.device_count() is filtered by the mask, so it cannot bound root ids."""+ xpu_host("2", torch_count = 1)+ assert hw._get_parent_visible_gpu_spec()["numeric_ids"] == [2]+ assert hw.get_visible_gpu_count() == 1+++def test_without_a_sysfs_bound_out_of_range_ids_are_kept(xpu_host):+ xpu_host("0,5", sysfs = False)+ assert hw._get_parent_visible_gpu_spec()["numeric_ids"] == [0, 5]+++def test_flat_tile_ids_are_not_bounded_by_cards(xpu_host, monkeypatch):+ """One two-tile card exposes ids 0 and 1 under FLAT."""+ xpu_host("0,1", roots = 1)+ monkeypatch.setenv("ZE_FLAT_DEVICE_HIERARCHY", "FLAT")+ assert hw.get_visible_gpu_count() == 2+++def test_torch_still_wins_the_visible_count(xpu_host):+ xpu_host("", torch_count = 3)+ assert hw.get_visible_gpu_count() == 3+++ ("mask", "roots"),+ [+ ("2.0,0.1,0.2", [0, 2]),+ ("08", [0]),+ ("1abc", [1]),+ ("abc,1", [1]),+ ("-1", []),+ ("1, 0", [0, 1]),+ ("-18446744073709551615", [1]),+ ],+)+def test_parse_ze_mask_roots(monkeypatch, mask, roots):+ monkeypatch.setattr(hw.sys, "platform", "linux")+ assert hw._parse_ze_mask_roots(mask) == roots+++ ("platform", "mask", "roots"),+ [+ ("win32", "-4294967295", [1]), # 32-bit unsigned long+ ("win32", "-1", []),+ ("linux", "-4294967295", []),+ ("win32", "-18446744073709551615", []), # out_of_range on Windows+ ("linux", "1" + "0" * 5000, []), # past Python's int-digit limit+ ("linux", "\u20031", []), # not C whitespace+ ("linux", "\t1", [1]),+ ("linux", "0" * 40 + "1", [1]),+ ("linux", "0b10", [2]), # C23 strtoul, linked by compute-runtime 26.35+ ("linux", "-0b1", []),+ ("linux", "0b", [0]),+ ],+)+def test_parse_ze_mask_roots_follows_the_platform_stoul(monkeypatch, platform, mask, roots):+ monkeypatch.setattr(hw.sys, "platform", platform)+ assert hw._parse_ze_mask_roots(mask) == roots+++def test_non_linux_hosts_have_no_root_bound(monkeypatch):+ monkeypatch.setattr(hw.sys, "platform", "win32")+ assert hw._intel_root_device_bound() is Nonediff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 93c96032baf..9db3a07f414 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -674,9 +674,15 @@ _VISIBILITY_MASK_VENDORS: Dict[str, frozenset] = {}+def _ze_affinity_mask() -> Optional[str]:+ """ZE_AFFINITY_MASK as Level Zero reads it: None when unset, empty or "default", which isAffinityMaskSet treats alike (every device visible)."""+ value = os.environ.get("ZE_AFFINITY_MASK")+ return None if value in (None, "", "default") else value++def _mask_is_emptied(var: str) -> bool:- """True when ``var`` is set to a value that hides every device it addresses: set-but-empty and "-1". A mask NAMING devices is not this, since that host expects those devices to work."""- value = os.environ.get(var)+ """True when ``var`` is set to a value that hides every device it addresses: set-but-empty and "-1". A mask NAMING devices is not this, since that host expects those devices to work. An empty ZE_AFFINITY_MASK hides nothing."""+ value = _ze_affinity_mask() if var == "ZE_AFFINITY_MASK" else os.environ.get(var)return value is not None and value.strip() in ("", "-1")@@ -1463,7 +1469,7 @@ def _detect_hardware_locked() -> DeviceType:import torch# Prefer XPU on UNSLOTH_FORCE_XPU=1, or ZE_AFFINITY_MASK set with CUDA hidden/unavailable. A bare mask is NOT enough (it can leak from unrelated Intel tooling); torch.xpu must report a device.- ze_mask = os.environ.get("ZE_AFFINITY_MASK")+ ze_mask = _ze_affinity_mask()cvd = os.environ.get("CUDA_VISIBLE_DEVICES")cuda_hidden = cvd is not None and cvd.strip() in ("", "-1")force_xpu = os.environ.get("UNSLOTH_FORCE_XPU") == "1"@@ -2663,20 +2669,47 @@ def _xpu_hierarchy_is_composite() -> bool:return hierarchy == "COMPOSITE"-def _parse_ze_mask_roots(mask: str) -> list[int]:- """Parse ZE_AFFINITY_MASK into ordered root device IDs, one per token, preserving order and duplicates so logical ordinals map 1-to-1 onto physical root IDs ("2.0,0.1,0.2" -> [2, 0, 0]). Only meaningful in COMPOSITE hierarchy, which callers must gate on."""- roots: list[int] = []- if not mask:- return roots- for token in mask.split(","):- token = token.strip()- if not token:+# The prefix C stoul(entry, nullptr, 0) consumes: sign, then hex, binary (C23 strtoul, which current+# compute-runtime builds link), octal or decimal ("08" -> 0, "+1abc" -> 1, "0b10" -> 2).+_ZE_MASK_ROOT_RE = re.compile(+ r"([+-]?)(?:0[xX]([0-9a-fA-F]+)|0[bB]([01]+)|(0[0-7]*)|([1-9][0-9]*))"+)+++def _parse_ze_mask_roots(mask: str, root_count: Optional[int] = None) -> list[int]:+ """Root device IDs a ZE_AFFINITY_MASK selects, filtered the way Level Zero does: in root order with duplicates collapsed ("2,0,0" -> [0, 2]), entries parsed like stoul base 0 ("0x1", "01" and "+1" -> 1), ids at or past ``root_count`` dropped. Only meaningful in COMPOSITE hierarchy, which callers must gate on."""+ roots: set = set()+ # stoul's unsigned long: 32-bit on Windows (LLP64), 64-bit elsewhere.+ ulong_limit = 2**32 if sys.platform == "win32" else 2**64+ for token in (mask or "").split(","):+ # C isspace only: stoul does not skip Unicode whitespace.+ match = _ZE_MASK_ROOT_RE.match(token.strip(" \t\n\v\f\r").split(".", 1)[0])+ if not match:+ continue+ sign, hex_digits, binary, octal, decimal = match.groups()+ digits = hex_digits or binary or octal or decimal+ # More digits than any unsigned long in any base: stoul throws out_of_range.+ if len(digits.lstrip("0")) > 64:continue- root = token.split(".", 1)[0]- # isdecimal(), not isdigit(), which accepts Unicode superscripts that then crash int().- if root.isdecimal():- roots.append(int(root))- return roots+ root = int(digits, 16 if hex_digits else 2 if binary else 8 if octal else 10)+ if root >= ulong_limit:+ continue+ # Negation is unsigned: "-1" wraps past every root, "-0" is 0.+ if sign == "-":+ root = -root % ulong_limit+ # No host has 2**16 roots, so a wrapped value is out of range even without a bound.+ if root < 2**16:+ roots.add(root)+ return sorted(root for root in roots if root_count is None or root < root_count)+++def _intel_root_device_bound() -> Optional[int]:+ """Upper bound on Level Zero root devices: every Intel card the Linux DRM drivers bound, allowlisted or not. None when unknown; torch.xpu.device_count() cannot stand in, since it is already filtered by the mask."""+ if not sys.platform.startswith("linux"):+ return None+ records = _linux_drm_sysfs_records(distinguish_failure = True)+ count = sum(1 for record in records or () if record.get("vendor") == "intel")+ return count or Nonedef _smi_query(func_name: str, *args, **kwargs) -> Optional[Dict[str, Any]]:@@ -4779,7 +4812,7 @@ _visible_gpu_count: Optional[int] = Nonedef _get_parent_visible_gpu_spec() -> Dict[str, Any]:# On Intel XPU, visibility is controlled by ZE_AFFINITY_MASK (Level Zero), not CUDA_VISIBLE_DEVICES.if get_device() == DeviceType.XPU:- xpu_mask_raw = os.environ.get("ZE_AFFINITY_MASK")+ xpu_mask_raw = _ze_affinity_mask()composite = _xpu_hierarchy_is_composite()if xpu_mask_raw is None:@@ -4829,10 +4862,10 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]:"supports_explicit_gpu_ids": False,}- # COMPOSITE + pure numeric (subdevice handled above). _parse_ze_mask_roots maps to root GPU IDs, dropping non-decimal tokens so "*"/"GPU-uuid" -> [].- roots_with_dupes = _parse_ze_mask_roots(xpu_mask)- if not roots_with_dupes:- # Unparseable mask ("*", "GPU-uuid"): cannot map to physical root IDs.+ # COMPOSITE + pure numeric (subdevice handled above). _parse_ze_mask_roots maps to root GPU IDs, dropping non-numeric tokens so "*"/"GPU-uuid" -> [].+ roots = _parse_ze_mask_roots(xpu_mask, _intel_root_device_bound())+ if not roots:+ # Unparseable ("*", "GPU-uuid") or out of range ("5" on 3 roots): no root to map.return {"raw": xpu_mask,"numeric_ids": None,@@ -4841,7 +4874,7 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]:return {"raw": xpu_mask,- "numeric_ids": roots_with_dupes,+ "numeric_ids": roots,"supports_explicit_gpu_ids": True,}@@ -6218,7 +6251,7 @@ def get_physical_gpu_count() -> int:def _backend_visible_devices_env() -> Optional[str]:"""The raw visibility env string that applies to this backend: ZE_AFFINITY_MASK on XPU, and HIP_/ROCR_VISIBLE_DEVICES ahead of CUDA_VISIBLE_DEVICES on ROCm. Mirrors _get_parent_visible_gpu_spec, so backend_cuda_visible_devices reports the value actually narrowing the visible set."""if get_device() == DeviceType.XPU:- return os.environ.get("ZE_AFFINITY_MASK")+ return _ze_affinity_mask()if IS_ROCM:return _get_parent_visible_gpu_spec().get("raw")return os.environ.get("CUDA_VISIBLE_DEVICES")@@ -6689,7 +6722,7 @@ def get_visible_gpu_count() -> int:# Prefer torch.xpu.device_count() on Intel XPU: the Level Zero runtime interprets ZE_AFFINITY_MASK semantics correctly (subdevice syntax "0.0,0.1" collapses onto one root GPU).if get_device() == DeviceType.XPU:- xpu_mask_raw = os.environ.get("ZE_AFFINITY_MASK")+ xpu_mask_raw = _ze_affinity_mask()xpu_mask_set = xpu_mask_raw is not Nonexpu_visible = (xpu_mask_raw or "").strip()if xpu_mask_set and xpu_visible == "":@@ -6710,9 +6743,11 @@ def get_visible_gpu_count() -> int:# Documented wildcard: all physical XPUs visible._visible_gpu_count = get_physical_gpu_count()else:- roots = _parse_ze_mask_roots(xpu_visible)+ # FLAT entries are tile handles (one PVC card exposes two), so only COMPOSITE ids are bounded by cards.+ bound = _intel_root_device_bound() if _xpu_hierarchy_is_composite() else None+ roots = _parse_ze_mask_roots(xpu_visible, bound)# Non-parseable masks (",,,", "GPU-abc") yield an empty roots list, treated as 0 visible devices rather than "all visible": no evidence the whole fleet was intended.- _visible_gpu_count = len(set(roots))+ _visible_gpu_count = len(roots)else:_visible_gpu_count = get_physical_gpu_count()return _visible_gpu_countdiff --git a/tests/studio/test_xpu_spoof_pipeline.py b/tests/studio/test_xpu_spoof_pipeline.pyindex 3570efa71e7..6d28f015b07 100644--- a/tests/studio/test_xpu_spoof_pipeline.py+++ b/tests/studio/test_xpu_spoof_pipeline.py@@ -388,8 +388,15 @@ def test_get_visible_gpu_count_uses_device_count(spoof_xpu):assert hw.get_visible_gpu_count() == 2-def test_get_visible_gpu_count_empty_mask_is_zero(spoof_xpu):- hw, _ = spoof_xpu(ze_mask = "")+def test_get_visible_gpu_count_empty_mask_is_no_mask(spoof_xpu):+ """Level Zero reads an empty ZE_AFFINITY_MASK as unset (isAffinityMaskSet), so every device stays visible."""+ hw, _ = spoof_xpu(ze_mask = "", device_count = 2)+ hw.detect_hardware()+ assert hw.get_visible_gpu_count() == 2+++def test_get_visible_gpu_count_blank_mask_is_zero(spoof_xpu):+ hw, _ = spoof_xpu(ze_mask = " ")hw.detect_hardware()assert hw.get_visible_gpu_count() == 0