Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: bound Qwen-Image-2.1 attention on Apple Silicon and fail loudly on NaN images (#13188)
Daniel Handfffd23fc28d
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex 87bb85163c1..8afdb0b2308 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -1949,11 +1949,12 @@ def _configure_native_attention(pipe: Any, target: Any, logger: Any) -> None:guard_rocm_fused_sdpa(target, logger)except Exception as exc:_warn(logger, "ROCm fused SDPA check", exc)- if rocm and sdpa_math_only(target):+ mps = getattr(target, "device", None) == "mps"+ if mps or (rocm and sdpa_math_only(target)):try:from .diffusion_qwenimage21_math import installif install(pipe, target, logger):- if logger is not None:+ if logger is not None and not mps:logger.warning("diffusion.attention: Qwen-Image-2.1 is using bounded math attention; ""check the AMD device packages if fused kernels are expected on this GPU"diff --git a/studio/backend/core/inference/diffusion_postprocess.py b/studio/backend/core/inference/diffusion_postprocess.pyindex 4841cc6ffa5..70fcc80c767 100644--- a/studio/backend/core/inference/diffusion_postprocess.py+++ b/studio/backend/core/inference/diffusion_postprocess.py@@ -4,8 +4,8 @@"""diffusers' ``postprocess(output_type="pil")`` uint8 conversion on the device instead of numpy on the host.Stock runs ``(x * 255).round().astype("uint8")`` over host float32 copies, which stalls under host load. Same IEEE ops-on clamped [0, 1] values, so bit-identical; anything else takes the stock path. Kill switch:-``UNSLOTH_DIFFUSION_DEVICE_POSTPROCESS=0``.+on clamped [0, 1] values, so bit-identical; anything else takes the stock path. A decoded image with NaN raises instead+of saving blank. Kill switch: ``UNSLOTH_DIFFUSION_DEVICE_POSTPROCESS=0``."""from __future__ import annotations@@ -15,6 +15,10 @@ from typing import Any, OptionalDEVICE_POSTPROCESS_ENV = "UNSLOTH_DIFFUSION_DEVICE_POSTPROCESS"_MARK = "_unsloth_device_postprocess"+NAN_IMAGE_MESSAGE = (+ "The decoded image contains NaN values and would be saved as a blank image. The model or VAE overflowed at this "+ "resolution and precision; try a smaller resolution or another quant of this model."+)def disabled() -> bool:@@ -67,6 +71,16 @@ def to_pil_on_device(return [Image.fromarray(pixels[i]) for i in range(pixels.shape[0])]+def has_nan(image: Any) -> bool:+ import torch+ return (+ isinstance(image, torch.Tensor)+ and image.is_floating_point()+ and image.device.type != "meta"+ and bool(torch.isnan(image).any())+ )++def uint8_hwc(image: Any) -> Any:"""Values must lie in [0, 1]: the uint8 cast of anything else is undefined."""import torch@@ -91,6 +105,9 @@ def install(pipe: Any, logger: Any = None) -> bool:output_type: str = "pil",do_denormalize: Optional[list] = None,) -> Any:+ if output_type != "latent" and has_nan(image):+ # Stock casts NaN to 0, a blank image that looks like a finished render.+ raise RuntimeError(NAN_IMAGE_MESSAGE)if output_type == "pil":try:out = to_pil_on_device(processor, image, do_denormalize)diff --git a/studio/backend/core/inference/diffusion_qwenimage21_math.py b/studio/backend/core/inference/diffusion_qwenimage21_math.pyindex 75844b71b20..30204349721 100644--- a/studio/backend/core/inference/diffusion_qwenimage21_math.py+++ b/studio/backend/core/inference/diffusion_qwenimage21_math.py@@ -6,9 +6,39 @@from __future__ import annotationsimport functools-from typing import Any+import os+from typing import Any, OptionalQUERY_CHUNK_SIZE = 512+# MiB one score tensor may take before MPS attention is split; unset = 1/8 of the GPU working set.+SCORE_BUDGET_ENV = "UNSLOTH_DIFFUSION_ATTN_SCORE_BUDGET_MB"+# Score elements per MPS call: macOS 15 / torch 2.11 SDPA returns wrong values from about 0.9 * 2**29 elements.+MPS_SCORE_ELEMENTS = 2**28+++def mps_score_budget() -> int:+ """Bytes one ``batch x heads x queries x keys`` score tensor may take on MPS before attention is split."""+ raw = (os.environ.get(SCORE_BUDGET_ENV) or "").strip()+ if raw:+ try:+ return max(0, int(float(raw) * 2**20))+ except (ValueError, OverflowError):+ pass+ try:+ import torch+ total = int(torch.mps.recommended_max_memory())+ except Exception: # noqa: BLE001 - no MPS runtime: the fixed floor below+ total = 0+ return total // 8 if total > 0 else 2**30+++def _query_rows(query, key, budget: Optional[int]) -> int:+ if budget is None:+ return QUERY_CHUNK_SIZE+ per_row = max(query.shape[0] * query.shape[2] * key.shape[1], 1)+ # The element cap is a correctness bound, so it also wins over the 512-row floor (batched 2048x2048).+ rows = max(QUERY_CHUNK_SIZE, budget // (per_row * query.element_size()))+ return max(1, min(rows, MPS_SCORE_ELEMENTS // per_row))def _bounded_attention(@@ -19,16 +49,18 @@ def _bounded_attention(*,mask = None,causal_offset = None,+ budget = None,**kwargs,):import torch+ rows = _query_rows(query, key, budget)outputs = []- for start in range(0, query.shape[1], QUERY_CHUNK_SIZE):- part = query[:, start : start + QUERY_CHUNK_SIZE]+ for start in range(0, query.shape[1], rows):+ part = query[:, start : start + rows]part_mask = maskif mask is not None and mask.shape[-2] != 1:- part_mask = mask[..., start : start + QUERY_CHUNK_SIZE, :]+ part_mask = mask[..., start : start + rows, :]if causal_offset is not None:# The segment can see all preceding segments, then a triangle over its own keys.causal = torch.arange(key.shape[1], device = query.device)[None, :] <= (@@ -74,6 +106,9 @@ def _processor_class():)class BoundedMathQwenImage21AttnProcessor(QwenImage21AttnProcessor):+ # None: fixed QUERY_CHUNK_SIZE rows (math-only ROCm). Bytes: split only past that score size (MPS).+ _unsloth_score_budget = None+def __call__(self,attn,@@ -108,6 +143,7 @@ def _processor_class():value,dispatch_attention_fn,mask = attention_mask,+ budget = self._unsloth_score_budget,backend = self._attention_backend,parallel_config = self._parallel_config,)@@ -122,6 +158,7 @@ def _processor_class():dispatch_attention_fn,mask = None if key_valid is None else key_valid[:, None, None, :end],causal_offset = start if is_text else None,+ budget = self._unsloth_score_budget,backend = None,parallel_config = self._parallel_config,)@@ -134,6 +171,7 @@ def _processor_class():value,dispatch_attention_fn,mask = None if key_valid is None else key_valid[:, None, None, :],+ budget = self._unsloth_score_budget,backend = None,parallel_config = self._parallel_config,)@@ -145,33 +183,47 @@ def _processor_class():return BoundedMathQwenImage21AttnProcessor+def needs_bounded_attention(target: Any) -> bool:+ """ROCm with only SDPA math, or MPS: MPS SDPA ignores ``sdpa_kernel`` and builds the full score matrix (about+ 34 GB per call at 2048x2048 bf16), served from swap."""+ if getattr(target, "device", None) == "mps":+ return True+ if getattr(target, "backend", None) != "rocm":+ return False+ from .diffusion_attention import sdpa_math_only++ return sdpa_math_only(target)++def install(pipe: Any,target: Any,logger: Any = None,) -> bool:- """A correctness fallback for every ROCm architecture, independent of speed mode."""- from .diffusion_attention import sdpa_math_only+ """A correctness fallback for every ROCm architecture and Apple Silicon, independent of speed mode."""from .diffusion_qwenimage21_rocm import _processor_class as speed_processor_classfrom diffusers.models.transformers.transformer_qwenimage21 import QwenImage21AttnProcessor- if getattr(target, "backend", None) != "rocm" or not sdpa_math_only(target):+ if not needs_bounded_attention(target):return Falsetransformer = getattr(pipe, "transformer", None)if type(transformer).__name__ != "QwenImage21Transformer2DModel":return Falsecls = _processor_class()+ budget = mps_score_budget() if getattr(target, "device", None) == "mps" else Nonechanged = Falsefor module in transformer.modules():processor = getattr(module, "processor", None)if type(processor) in (QwenImage21AttnProcessor, speed_processor_class()):replacement = cls()replacement.__dict__.update(processor.__dict__)+ replacement._unsloth_score_budget = budgetmodule.set_processor(replacement)changed = Trueif changed and logger is not None:logger.info(- "diffusion.qwenimage21: bounding math attention to 512 query rows, including prefill"+ "diffusion.qwenimage21: bounding math attention to %s, including prefill",+ "512 query rows" if budget is None else f"{budget / 2**30:.2f} GiB of scores per call",)return bounded_math_attention(pipe)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 57ee1b9c141..fd2841d413c 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -46733,6 +46733,11 @@ _GENERATE_FAILURE_CLASSES: tuple[tuple[tuple[str, ...], str], ...] = (("out of memory", "outofmemory", "oom"),"The device ran out of memory. Try a smaller size, fewer steps, or a smaller batch.",),+ (+ ("decoded image contains nan",),+ "The model or VAE overflowed at this resolution and precision. Try a smaller resolution or another "+ "quant of this model.",+ ),(("sd-server connection lost", "sd-cli exited", "process exited", "ggml_abort", "signal"),"The native image renderer stopped unexpectedly. Switch the engine to diffusers, or see "diff --git a/studio/backend/tests/test_diffusion_postprocess.py b/studio/backend/tests/test_diffusion_postprocess.pyindex 78693351513..3489b533261 100644--- a/studio/backend/tests/test_diffusion_postprocess.py+++ b/studio/backend/tests/test_diffusion_postprocess.py@@ -136,6 +136,18 @@ def test_nan_input_returns_none_for_the_stock_path():assert dp.to_pil_on_device(VaeImageProcessor(), image) is None[email protected]("output_type", ["pil", "np", "pt"])+def test_nan_image_raises_instead_of_saving_blank(output_type):+ proc = VaeImageProcessor()+ dp.install(_Pipe(proc))+ image = _decoded(torch.float32)+ image[0, :, 3, 5] = float("nan")+ with pytest.raises(RuntimeError, match = "NaN"):+ proc.postprocess(image, output_type = output_type)+ # Latents are not an image: they pass through untouched.+ assert proc.postprocess(image, output_type = "latent") is image++def test_stock_path_still_serves_other_output_types_and_cpu():proc = VaeImageProcessor()dp.install(_Pipe(proc))diff --git a/studio/backend/tests/test_diffusion_qwenimage21_large_render.py b/studio/backend/tests/test_diffusion_qwenimage21_large_render.pynew file mode 100644index 00000000000..6596d8d2041--- /dev/null+++ b/studio/backend/tests/test_diffusion_qwenimage21_large_render.py@@ -0,0 +1,155 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Large Qwen-Image-2.1 renders on the device this runs on, MPS first.++The pipeline packs 16x16 pixels per token, so 2048x2048 is 16384 image tokens, as for the real model. Stock MPS SDPA+builds the full score matrix for that; the bounded processors keep each call to 512 query rows."""++from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")++from core.inference import diffusion_qwenimage21_math as bounded++LARGE = 2048+TOKENS = (LARGE // 16) ** 2+++def _device() -> str:+ if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():+ return "mps"+ if torch.cuda.is_available():+ return "cuda"+ return "cpu"+++def _sync(device):+ if device == "mps":+ torch.mps.synchronize()+ elif device == "cuda":+ torch.cuda.synchronize()+++def _reset_peak(device):+ """Bytes the allocator holds now; MPS has no peak counter, so its cache (kept between calls) stands in."""+ _sync(device)+ if device == "mps":+ torch.mps.empty_cache()+ return torch.mps.driver_allocated_memory()+ if device == "cuda":+ torch.cuda.empty_cache()+ torch.cuda.reset_peak_memory_stats()+ return torch.cuda.memory_allocated()+ return None+++def _peak(device):+ _sync(device)+ if device == "mps":+ return torch.mps.driver_allocated_memory()+ if device == "cuda":+ return torch.cuda.max_memory_allocated()+ return None+++def tiny_pipe():+ pipeline_cls = getattr(diffusers, "QwenImage21Pipeline", None)+ if pipeline_cls is None:+ pytest.skip("this diffusers has no QwenImage21Pipeline")+ from huggingface_hub import snapshot_download++ try:+ path = snapshot_download("hf-internal-testing/tiny-qwenimage21-pipe")+ except Exception as exc: # noqa: BLE001+ pytest.skip(f"tiny pipeline not reachable: {exc}")+ device = _device()+ pipe = pipeline_cls.from_pretrained(path, torch_dtype = torch.float32).to(device)+ pipe.set_progress_bar_config(disable = True)+ return pipe, device+++def _render(pipe, size):+ with torch.inference_mode():+ return pipe(+ prompt = "a lighthouse on a cliff at sunset",+ height = size,+ width = size,+ num_inference_steps = 2,+ generator = torch.Generator("cpu").manual_seed(0),+ output_type = "pt",+ ).images++[email protected]_network+def test_large_render_is_bounded_finite_and_matches_stock(tiny_pipe, monkeypatch):+ pipe, device = tiny_pipe+ # Below any real budget, so the large render splits at the 512-row floor on every device.+ monkeypatch.setenv(bounded.SCORE_BUDGET_ENV, "0")+ reference = _render(pipe, 256).cpu()+ # The target Studio resolves on Apple Silicon; install never touches the device itself.+ assert bounded.install(pipe, SimpleNamespace(device = "mps", backend = "mps"))+ assert bounded.bounded_math_attention(pipe)+ torch.testing.assert_close(_render(pipe, 256).cpu(), reference, atol = 2e-5, rtol = 1e-4)++ rows = []+ original = bounded._bounded_attention++ def record(query, key, value, dispatch, **kwargs):+ def checked(q, k, v, **args):+ rows.append((q.shape[1], k.shape[1]))+ return dispatch(q, k, v, **args)++ return original(query, key, value, checked, **kwargs)++ monkeypatch.setattr(bounded, "_bounded_attention", record)+ # The denoise peak, read when the decode starts: the VAE at 2048x2048 is not what this measures.+ peaks = []+ decode = pipe.vae.decode++ def measured_decode(*args, **kwargs):+ peaks.append(_peak(device))+ return decode(*args, **kwargs)++ monkeypatch.setattr(pipe.vae, "decode", measured_decode)+ base = _reset_peak(device)+ image = _render(pipe, LARGE)+ assert image.shape[-2:] == (LARGE, LARGE)+ assert bool(torch.isfinite(image).all())+ assert float(image.std()) > 0+ assert max(q for q, _ in rows) == bounded.QUERY_CHUNK_SIZE+ assert max(k for _, k in rows) >= TOKENS+ # One full fp32 score matrix for 2 heads is 2 * 16384^2 * 4 bytes = 2 GiB; every bounded call stays near 64 MiB.+ full_scores = 2 * TOKENS * TOKENS * 4+ if base is not None:+ grown = peaks[0] - base+ print(+ f"{device}: denoise peak {grown / 2**20:.0f} MiB over the loaded pipeline at {LARGE}x{LARGE}"+ )+ assert grown < full_scores / 4++[email protected](_device() != "mps", reason = "measures torch's MPS SDPA")+def test_mps_sdpa_ignores_kernel_selection_and_materialises_scores():+ """Why Studio cannot read MPS's attention memory off the SDPA probe (a record of the torch it runs on)."""+ from torch.nn.attention import SDPBackend, sdpa_kernel++ q = torch.randn(1, 2, 4096, 16, device = "mps")+ with sdpa_kernel([SDPBackend.FLASH_ATTENTION]):+ flash = torch.nn.functional.scaled_dot_product_attention(q, q, q)+ torch.mps.synchronize()+ base = _reset_peak("mps")+ torch.nn.functional.scaled_dot_product_attention(q, q, q)+ grown = _peak("mps") - base+ scores = 2 * 4096 * 4096 * 4+ print(+ f"torch {torch.__version__}: FLASH-only SDPA ran; {grown / 2**20:.0f} MiB for {scores / 2**20:.0f} MiB of scores"+ )+ assert flash.shape == q.shape+ version = tuple(int(p) for p in torch.__version__.split("+")[0].split(".")[:2])+ if version < (2, 13): # 2.13 adds a fused MPS prefill kernel+ assert grown >= scores // 2diff --git a/studio/backend/tests/test_diffusion_qwenimage21_math.py b/studio/backend/tests/test_diffusion_qwenimage21_math.pyindex c6dd23b07bd..75dafbb7678 100644--- a/studio/backend/tests/test_diffusion_qwenimage21_math.py+++ b/studio/backend/tests/test_diffusion_qwenimage21_math.py@@ -176,3 +176,85 @@ def test_empty_prefix_matches_full_attention():expected = qmod.QwenImage21AttnProcessor()(attn, states, segments = [])actual = bounded._processor_class()()(attn, states, segments = [])torch.testing.assert_close(actual, expected)+++def test_mps_installs_bounded_attention_without_trusting_the_probe(monkeypatch):+ from core.inference.diffusion import _quadratic_attention++ pipe, modules = make_pipe()+ target = SimpleNamespace(backend = "mps", device = "mps", dtype = torch.bfloat16)+ # MPS SDPA ignores sdpa_kernel, so the probe reports fused kernels it does not run.+ monkeypatch.setattr(attention, "sdpa_math_only", lambda t: False)+ warned = []+ monkeypatch.setattr(attention, "warn_if_sdpa_math_only", lambda *a, **k: warned.append(a))+ attention.apply_attention_backend(pipe, None, target = target)+ assert all(type(m.processor) is bounded._processor_class() for m in modules)+ assert bounded.bounded_math_attention(pipe)+ assert not _quadratic_attention(target, None, pipe)+ assert not warned+++ "target, math_only, expected",+ [+ (SimpleNamespace(backend = "mps", device = "mps"), False, True),+ (SimpleNamespace(backend = "rocm", device = "cuda"), True, True),+ (SimpleNamespace(backend = "rocm", device = "cuda"), False, False),+ (SimpleNamespace(backend = "cuda", device = "cuda"), True, False),+ (SimpleNamespace(backend = "cpu", device = "cpu"), True, False),+ ],+)+def test_bounded_attention_targets(monkeypatch, target, math_only, expected):+ monkeypatch.setattr(attention, "sdpa_math_only", lambda t: math_only)+ assert bounded.needs_bounded_attention(target) is expected+++def test_mps_budget_keeps_one_call_per_segment_when_the_scores_fit():+ torch.manual_seed(3)+ attn = qmod.QwenImage21Attention(dim = 16, heads = 2, dim_head = 8)+ stock, patched = qmod.QwenImage21AttnProcessor(), bounded._processor_class()()+ patched._unsloth_score_budget = 2**30+ states = torch.randn(1, 1800, 16)+ segments = [(0, 600, True), (600, 1130, False)]+ with torch.inference_mode():+ expected = stock(attn, states, segments = segments)+ actual = patched(attn, states, segments = segments)+ assert torch.equal(actual, expected)+++ "budget, rows",+ [(None, 512), (0, 512), (2 * 2 * 4096 * 4 * 700, 700), (2**40, 2**28 // (2 * 2 * 4096))],+)+def test_query_rows_follow_the_score_budget(budget, rows):+ q, k = torch.empty(2, 4096, 2, 8), torch.empty(2, 4096, 2, 8)+ assert bounded._query_rows(q, k, budget) == rows+++def test_batched_2048_stays_under_the_mps_element_cap():+ q, k = (+ torch.empty(2, 16384, 32, 1, dtype = torch.bfloat16),+ torch.empty(2, 16640, 32, 1, dtype = torch.bfloat16),+ )+ rows = bounded._query_rows(q, k, 2**34)+ assert (+ 1 <= rows < bounded.QUERY_CHUNK_SIZE and rows * 2 * 32 * 16640 <= bounded.MPS_SCORE_ELEMENTS+ )+++def test_mps_score_budget_reads_the_override(monkeypatch):+ monkeypatch.setenv(bounded.SCORE_BUDGET_ENV, "256")+ assert bounded.mps_score_budget() == 256 * 2**20+ monkeypatch.setattr(torch.mps, "recommended_max_memory", lambda: 16 * 2**30, raising = False)+ for unusable in ("", "inf", "1e309", "nan", "x"):+ monkeypatch.setenv(bounded.SCORE_BUDGET_ENV, unusable)+ assert bounded.mps_score_budget() == 2 * 2**30+++def test_qwen_image_21_1024_splits_below_the_mps_element_cap():+ # One 32 x 4096 x 4352 call returned wrong values on macOS 15; 768x768 (2304 tokens) stays one call.+ k = torch.empty(1, 4352, 32, 1, dtype = torch.bfloat16)+ rows = bounded._query_rows(torch.empty(1, 4096, 32, 1, dtype = torch.bfloat16), k, 2**34)+ assert rows < 4096 and rows * 32 * 4352 <= bounded.MPS_SCORE_ELEMENTS+ k = torch.empty(1, 2560, 32, 1, dtype = torch.bfloat16)+ assert bounded._query_rows(torch.empty(1, 2304, 32, 1, dtype = torch.bfloat16), k, 2**34) >= 2304diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.pyindex 48f527f3e26..f512d3a4957 100644--- a/studio/backend/tests/test_diffusion_routes.py+++ b/studio/backend/tests/test_diffusion_routes.py@@ -710,6 +710,21 @@ def test_generate_native_process_death_names_the_engine_not_its_output(client, massert leak not in detail+def test_generate_nan_image_names_the_overflow(client, monkeypatch):+ from core.inference.diffusion_postprocess import NAN_IMAGE_MESSAGE++ backend = diffusion_module.get_diffusion_backend()+ backend.loaded = True++ def _nan(**kwargs):+ raise RuntimeError(NAN_IMAGE_MESSAGE)++ monkeypatch.setattr(backend, "generate", _nan)+ resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})+ assert resp.status_code == 500+ assert "overflowed at this resolution" in resp.json()["detail"]++def test_generate_execution_error_with_cancelled_substring_is_sanitized_500(client, monkeypatch):# A native execution failure whose raw tail merely CONTAINS "cancelled" must stay a sanitized 500, not misroute to 409.backend = diffusion_module.get_diffusion_backend()