Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: turn on the measured-safe optimizations by default (fused FLUX RoPE, CFG batching, Blackwell native attention, BF16 small DiTs) (#14566)
Daniel Han5a955eff1623
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 5ec8875f948..be22f177fa2 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -54,6 +54,8 @@ from .diffusion_content import ()from .diffusion_gguf_pipeline import load_whole_pipeline_gguf, whole_pipeline_gguf_resident_mibfrom .diffusion_families import (+ SMALL_DIT_BF16_WHEN_RESIDENT,+ small_dit_rule_excludes,named_variant_base,DIFFUSION_CANCELLED_MSG,DIFFUSION_NOT_LOADED_MSG,@@ -2986,8 +2988,13 @@ def _plan_proves_resident(plan: Any) -> bool:)-def _auto_keeps_bf16_reason(fam: Any, kind: Optional[str] = "pipeline") -> Optional[str]:- """Why AUTO keeps bf16 when it fits (measured rule: pipeline loads only), or None; both deciders read this."""+def _auto_keeps_bf16_reason(+ fam: Any,+ kind: Optional[str] = "pipeline",+ base: Optional[str] = None,+) -> Optional[str]:+ """Why AUTO keeps bf16 when it fits (measured rule: pipeline loads only), or None; both deciders read this.+ ``base`` is the resolved base repo: the small-DiT rule skips a family's large variants (FLUX.2 klein 9B)."""if not family_compiles_regionally(fam):return (f"'{getattr(fam, 'name', None)}' cannot be regionally compiled (its transformer declares no "@@ -2999,6 +3006,8 @@ def _auto_keeps_bf16_reason(fam: Any, kind: Optional[str] = "pipeline") -> Optiomeasured = auto_bf16_when_resident_reason(getattr(fam, "name", None))if measured is None:return None+ if measured == SMALL_DIT_BF16_WHEN_RESIDENT and small_dit_rule_excludes(base):+ return Nonereturn (f"'{getattr(fam, 'name', None)}': {measured}; the bf16 weights fit on this GPU as they are")@@ -3009,11 +3018,12 @@ def _auto_quant_eager_reason(plan: Any,prequant_path: Optional[str] = None,kind: Optional[str] = "pipeline",+ base: Optional[str] = None,) -> Optional[str]:"""Why AUTO keeps bf16 for this load, or None; an operator's own checkpoint wins."""if not _plan_proves_resident(plan):return None- reason = _auto_keeps_bf16_reason(fam, kind)+ reason = _auto_keeps_bf16_reason(fam, kind, base)if reason is None:return Noneif prequant_path and local_prequant_path_ready(prequant_path):@@ -5309,7 +5319,7 @@ class DiffusionBackend:is not None):return None- if auto and _auto_keeps_bf16_reason(fam) is not None:+ if auto and _auto_keeps_bf16_reason(fam, base = base or repo_id) is not None:bf16_memory = snapshot_device_memory(target)bf16_plan = self._bf16_table_plan(target,@@ -5326,7 +5336,9 @@ class DiffusionBackend:# Offline without the released shards, bf16 cannot assemble: a cached seed is the only way.if (bf16_plan is not None- and _auto_quant_eager_reason(fam, bf16_plan, transformer_prequant_path)+ and _auto_quant_eager_reason(+ fam, bf16_plan, transformer_prequant_path, base = base or repo_id+ )is not Noneand not (local_files_only@@ -7596,6 +7608,7 @@ class DiffusionBackend:# Why the dense quant did not engage, in the caller's terms. Threaded into `resolved` so a fallback is# visible, and into the refusal so it is actionable.transformer_quant_decline: Optional[str] = None+ small_dit_kept_bf16 = Falsetransformer_quant_decline_status = RESOLVED_FELL_BACK# A GGUF is excluded while it bakes LoRAs: only the dense build can carry them.if (@@ -7604,7 +7617,7 @@ class DiffusionBackend:and dense_quant_supported_kind(kind)and dense_transformer_supported(target)and not (kind == "gguf" and _has_active_lora(loras))- and _auto_keeps_bf16_reason(fam, kind) is not None+ and _auto_keeps_bf16_reason(fam, kind, base) is not Noneand (eager_reason := _auto_quant_eager_reason(fam,@@ -7625,6 +7638,7 @@ class DiffusionBackend:),transformer_prequant_path,kind,+ base,))is not None@@ -7636,6 +7650,10 @@ class DiffusionBackend:transformer_quant_decline = eager_reason# Place the released weights by the plan that proved they fit, not by fp32 shard bytes.plan = eager_plan+ small_dit_kept_bf16 = (+ auto_bf16_when_resident_reason(getattr(fam, "name", None))+ == SMALL_DIT_BF16_WHEN_RESIDENT+ )native_scheme = (native_quant_scheme(target, transformer_quant_pinned, family = getattr(fam, "name", None)@@ -9236,6 +9254,19 @@ class DiffusionBackend:"(quantized transformer must be compiled; eager is ~30x slower)")effective_speed = SPEED_DEFAULT+ if (+ small_dit_kept_bf16+ and speed_mode is None+ and effective_speed == SPEED_OFF+ and not comfy_compile+ and compile_eligible(target, is_gguf = False, family = fam, probe = False)+ and not fp16_compile_explicit_only(target)+ ):+ logger.info(+ "diffusion.speed: speed_mode=default from render 1 (a small denoiser kept bf16 in "+ "place of the INT8 default, which compiled from render 1 too)"+ )+ effective_speed = SPEED_DEFAULT# Deferred speed auto for dense: stay eager and engage `default` on the 3rd image, where compile# amortises. Only when speed was unset.speed_deferred = (@@ -9963,8 +9994,9 @@ class DiffusionBackend:)# After every forward wrapper of the speed / graph layers, so the replica sees the final module.install_cfg_parallel(pipe, offload_policy = effective_policy, logger = logger)- # both branches through one offloaded forward; declines when CFG-parallel took the slot- install_cfg_batch(pipe, offload_policy = effective_policy, logger = logger)+ install_cfg_batch(+ pipe, offload_policy = effective_policy, speed_mode = speed_mode, logger = logger+ )# Z-Image guided steps at the guidance-off batch: turning CFG on compiles and records nothing newinstall_zimage_cfg_split(pipe, offload_policy = effective_policy, logger = logger)te_releaser = te_release.maybe_install(pipe, plan, logger = logger)diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex 696421a4562..2dae44b7c3a 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -15,8 +15,10 @@ Attention is bandwidth-bound, so a better kernel is a real win orthogonal to weibitwise-identical and near free (compiled 1.599s -> 1.572s, 1.02x; eager 0.93x, theper-call sdpa_kernel wrapper cost, which compile folds away) -- but FLASH and EFFICIENTat that same shape run 3.9x and 9.0x slower. Pinning is insurance against the heuristic- picking one of those on another card, head_dim or torch build. Elsewhere stays- ``native``. Only upgrades when a speed profile is active, so ``speed_mode=off`` stays+ picking one of those on another card, head_dim or torch build. On Blackwell (SM100+) auto+ leaves native when torch's own dispatch picks cuDNN at every DiT head dim and the run dtype+ (same kernel, no per-call wrapper; ``UNSLOTH_DIFFUSION_BLACKWELL_NATIVE_ATTN=0`` keeps the+ pin). Elsewhere stays ``native``. Only upgrades when a speed profile is active, so ``speed_mode=off`` staysbit-identical.native - force the default SDPA (bit-identical reference).cudnn - cuDNN fused attention (exact; NVIDIA).@@ -473,7 +475,7 @@ def select_attention_backend(return Nonereturn backendif speed_active and _is_cuda_nvidia(target) and _cudnn_attention_supported():- return "_native_cudnn"+ return AUTO_CUDNNif (speed_active or speed_unset) and _rocm_auto_flash(target, family):return ROCM_FLASH_CANDIDATEreturn None@@ -506,6 +508,22 @@ def _is_cuda_rocm(target: Any) -> bool:ROCM_FLASH_CANDIDATE = "_rocm_flash_candidate"+++class _AutoCudnn(str):+ """The ``auto`` cuDNN pick. Equal to ``"_native_cudnn"`` for every caller; ``apply_attention_backend`` alone tells+ it from an explicit ``cudnn`` request, so it can leave native where torch already dispatches cuDNN (Blackwell)."""++ __slots__ = ()+++AUTO_CUDNN = _AutoCudnn("_native_cudnn")+# B200 torch 2.13 bf16: native SDPA already runs cuDNN at head_dim 64/128 (pin = 0.4-22% slower eager wrapper);+# head_dim 256 picks FLASH, hence the per-head-dim probe.+BLACKWELL_NATIVE_ATTN_ENV = (+ "UNSLOTH_DIFFUSION_BLACKWELL_NATIVE_ATTN" # auto (default) | 0 keeps the cuDNN pin+)+_NATIVE_DISPATCH_CACHE: dict[tuple[str, str, int], bool] = {}ROCM_FLASH_ENV = "UNSLOTH_DIFFUSION_ROCM_FLASH" # auto (timed) | 0 (auto never picks flash) | 1 (verified is enough)ROCM_FLASH_MIN_GAIN = 0.03# Accuracy is judged on the raw attention output (flash vs native SDPA, same inputs), not the block: the residual and@@ -3054,6 +3072,60 @@ def _cudnn_runs_head_dim(target: Any, head_dim: int) -> Optional[bool]:return _CUDNN_HEAD_DIM_CACHE.setdefault(key, bool(ran))+def _native_sdp_choice(device: str, dtype: Any, head_dim: int) -> int:+ """torch's own SDPA dispatch choice (an ``SDPBackend`` value) at ``head_dim`` / ``dtype`` on ``device``."""+ import torch++ q = torch.zeros(1, 2, 256, int(head_dim), device = device, dtype = dtype)+ return int(torch._fused_sdp_choice(q, q, q))+++def _device_capability(device: str) -> tuple[int, int]:+ import torch+ return tuple(torch.cuda.get_device_capability(device)) # type: ignore[return-value]+++def _native_dispatches_cudnn(pipe: Any, target: Any) -> bool:+ """True on a Blackwell (SM100+) NVIDIA card when torch's own SDPA dispatch picks cuDNN for every DiT head dim at the+ run dtype, so the ``auto`` pin would change no kernel. Unknown head dims, dtype or capability keep the pin."""+ if (os.environ.get(BLACKWELL_NATIVE_ATTN_ENV) or "").strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ ):+ return False+ if not _is_cuda_nvidia(target):+ return False+ device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")+ if not device.startswith("cuda"):+ return False+ dims = _dit_head_dims(pipe)+ if not dims:+ return False+ try:+ import torch+ from torch.nn.attention import SDPBackend++ device = _indexed_cuda_device(device)+ if _device_capability(device) < (10, 0):+ return False+ dtype = getattr(target, "dtype", None)+ if dtype not in (torch.bfloat16, torch.float16):+ return False+ for dim in sorted(dims):+ key = (device, str(dtype), int(dim))+ hit = _NATIVE_DISPATCH_CACHE.get(key)+ if hit is None:+ hit = _native_sdp_choice(device, dtype, dim) == int(SDPBackend.CUDNN_ATTENTION)+ _NATIVE_DISPATCH_CACHE[key] = hit+ if not hit:+ return False+ return True+ except Exception: # noqa: BLE001 - unaskable: keep the pin+ return False++def _cudnn_serves_pipe(pipe: Any,target: Any,@@ -3182,6 +3254,20 @@ def apply_attention_backend(and not _cudnn_serves_pipe(pipe, target, logger)):backend = None+ if (+ isinstance(backend, _AutoCudnn)+ and target is not None+ and _native_dispatches_cudnn(pipe, target)+ ):+ if logger is not None:+ logger.info(+ "diffusion.attention: auto keeps native SDPA (it already dispatches cuDNN on this Blackwell card; "+ "%s=0 pins cuDNN)",+ BLACKWELL_NATIVE_ATTN_ENV,+ )+ backend = None+ if backend is not None:+ backend = str(backend)if backend == "flash_4_hub" and not _install_fa4_dispatch_guard():if logger is not None:logger.warning(diff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.pyindex 24c90b12229..2529be91d2c 100644--- a/studio/backend/core/inference/diffusion_cache.py+++ b/studio/backend/core/inference/diffusion_cache.py@@ -63,6 +63,8 @@ def _auto_static_skip_table() -> dict:AUTO_STATIC_SKIP: dict = _auto_static_skip_table()+AUTO_STATIC_SKIP_TIERS = (_TIER_MAX,)+# Load gate on the model's default steps (keeps distilled siblings out); explicit "static" keeps STATIC_MIN_STEPS (12).AUTO_STATIC_MIN_STEPS = 20@@ -106,15 +108,17 @@ def auto_static_skip_plan("""Settings for an auto static skip (``{"every": n, "min_steps": m}``), or None when auto must not pick it.``speed_mode`` is the tier the user ASKED for (an eager downgrade forced by offload does not change what the- skip costs in quality); off / eager are the lossless tiers and never skip."""+ skip costs in quality). Only ``max`` skips automatically: the skip is lossy (mean LPIPS up to 0.05 on the+ ``default`` interval, 0.10 on ``max``, PR #12652) and ComfyUI never skips steps by default, so ``default``, an+ unset speed and the deferred profile render every step. An explicit ``static`` request is still honoured."""if auto_step_skip_disabled(env):return Noneif isinstance(identifiers, str) or identifiers is None:identifiers = (identifiers,)entry = auto_static_skip_entry(*identifiers)- if not entry or speed_mode not in (_TIER_DEFAULT, _TIER_MAX):+ if not entry or speed_mode not in AUTO_STATIC_SKIP_TIERS:return None- every = entry.get(_TIER_DEFAULT) if speed_mode == _TIER_DEFAULT else entry.get(_TIER_MAX)+ every = entry.get(_TIER_MAX)try:steps_ok = default_steps is not None and int(default_steps) >= AUTO_STATIC_MIN_STEPSexcept (TypeError, ValueError):diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.pyindex 186acedefed..90a8b1d27d1 100644--- a/studio/backend/core/inference/diffusion_cfg_batch.py+++ b/studio/backend/core/inference/diffusion_cfg_batch.py@@ -1,12 +1,13 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""CFG batching: both guidance branches of a step through ONE forward of an offloaded denoiser.+"""CFG batching: both guidance branches of a step through ONE forward of the denoiser.-A denoiser streamed from host memory (group / sequential offload) uploads its weights once per call, and a guided step-calls it twice (conditional, then unconditional), so it pays the upload twice. ComfyUI batches the two branches into-one call (``comfy/samplers.py`` ``calc_cond_batch``); diffusers pipelines call the transformer once per branch. This-router batches them for the classes whose forward keeps no state between calls:+A guided step calls the denoiser twice (conditional, then unconditional). ComfyUI batches the two branches into one+call by default whenever memory allows (``comfy/samplers.py`` ``calc_cond_batch``), resident or not; diffusers pipelines+call the transformer once per branch. A denoiser streamed from host memory pays its weight upload once instead of+twice; a resident one gains where one doubled call keeps the GPU busier than two (small images). This router batches+them for the classes whose forward keeps no state between calls:- the first branch of a step builds the second branch's arguments from the previous step's second call (same latentobject, the first branch's per-step tensors, the second branch's constants), runs both as one call of twice the@@ -14,20 +15,28 @@ router batches them for the classes whose forward keeps no state between calls:- the second call is verified against what was batched (every argument bitwise) and gets the kept half; anythingelse (a changed argument, no second call, a third call) reruns that branch plain and counts as a miss.-Priced by measurement, per request shape: a plain pair is timed on the second step of a shape (the first warms up),-the batched pair (concatenate, forward, split, verify) on the next two, and batching stays on only when the faster-batched pair beats the plain pair by ``_MEASURED_FRACTION``. Memory: the doubled call is attempted only when the card-has room for twice the activations a plain call allocated; an out-of-memory reruns plain and turns the shape off.--Not bit-exact: a doubled batch can pick other GEMM / attention tiles (FLUX.2-klein-base, offloaded: LPIPS ~0.01 vs two-calls). ``UNSLOTH_DIFFUSION_CFG_BATCH=0`` keeps the exact one-branch-per-call path; ``force`` also batches a resident-denoiser and an unlisted class. Declined for per-call activation scaling (NVFP4 without frozen global scales), whose-scale would change with the doubled batch, and for denoisers already wrapped by another forward layer.+Priced once per request shape, before any batched result is used: steps 1-2 run plain (step 2 is timed); a resident+pair >= ``_RESIDENT_MAX_PAIR_S`` stays plain untried. Otherwise a discarded trial (plain pairs interleaved with doubled+calls) batches steps 3+ only if the fastest batched step (predict, merge, snapshot, doubled call, split, verify) beats the+fastest plain pair by ``_MEASURED_FRACTION``, so every render of a shape takes one path (same seed, same image). The+doubled call needs room for twice a plain call's activations; an OOM turns the shape plain for that render.++Not always bit-exact (other GEMM / attention tiles): B200 bf16 resident FLUX.2-klein-base-4B is bitwise equal,+Wan2.1-T2V-1.3B LPIPS 0.019 mean; offloaded FLUX.2-klein-base LPIPS ~0.01. ``UNSLOTH_DIFFUSION_CFG_BATCH=0`` keeps the exact+one-branch-per-call path, as does a request with ``speed_mode="off"``; ``force`` batches anyway, an unlisted class too+(the trial's discarded calls must be side-effect free). Declined for per-call activation scaling (NVFP4 without frozen+global scales), whose scale would change with the doubled batch, and for denoisers already wrapped by another forward+layer.++Compiled denoisers: the doubled shape compiles a second graph per block, allowed only in a shape's trial render.+Offloaded compiled denoisers batch; resident compiled ones and shapes judged before a later compile stay plain. A+follow-up-render compile ends batching for the load; under Dynamo tracing the router passes straight through."""from __future__ import annotationsimport os+import sysimport timeimport weakreffrom typing import Any, Optional@@ -52,12 +61,15 @@ _PADDED_TEXT = (("encoder_hidden_states", "encoder_hidden_states_mask"),)_MEASURED_FRACTION = 0.92_BATCHED_SAMPLES = 2+_PLAIN_STEPS = 2+# Compute-bound resident pair: skip the trial (B200 winners 0.03-0.06 s/pair; T4 FLUX.2-klein ~4.2 s, trial +17 s).+_RESIDENT_MAX_PAIR_S = 0.25_MISS_LIMIT = 3_OFF_TRANSIENT = "transient"def cfg_batch_mode() -> str:- """``off`` / ``auto`` (default: an offloaded denoiser of a listed class) / ``force``. Independent of the multi-GPU+ """``off`` / ``auto`` (default: a listed class, resident or offloaded) / ``force``. Independent of the multi-GPUswitches: a single card is the usual case."""raw = (os.environ.get(CFG_BATCH_ENV) or "").strip().lower()if raw in ("0", "off", "false", "no", "disable", "disabled"):@@ -102,6 +114,56 @@ def _foreign_hook(module: Any) -> Optional[str]:return None+# Set on a denoiser whose blocks Studio compiled (regional compile; also read when the compile moves below an+# offload hook, which clears the block's ``_compiled_call_impl``).+_COMPILE_MARKS = ("_unsloth_regional_compile_kwargs", "_unsloth_compile_guard")+++def _compiled_callable(fn: Any) -> bool:+ for _ in range(4):+ if fn is None:+ return False+ if hasattr(fn, "_torchdynamo_orig_callable") or hasattr(fn, "_unsloth_compile_guard"):+ return True+ fn = getattr(fn, "__wrapped__", None)+ return False+++def compiled_denoiser(module: Any) -> Optional[str]:+ """Why ``module`` runs compiled code, else None. A doubled batch there is a new input shape: the batch 1 -> 2 change+ compiles a second graph per compiled block, which is only acceptable inside the warm-up render of a shape."""+ try:+ subs = list(module.named_modules())+ except Exception: # noqa: BLE001 - not a torch module (fakes): nothing compiled+ return None+ for mark in _COMPILE_MARKS:+ if mark in getattr(module, "__dict__", {}):+ return "the denoiser is compiled"+ for name, sub in subs:+ if getattr(sub, "_compiled_call_impl", None) is not None:+ return f"{name or 'the denoiser'} is compiled"+ registry = getattr(sub, "__dict__", {}).get("_diffusers_hook")+ for ref in list(getattr(registry, "_fn_refs", None) or ()):+ if _compiled_callable(getattr(ref, "forward", None)):+ return f"{name or 'the denoiser'} runs a compiled forward below its hooks"+ return None+++def _offloaded(denoiser: Any, offload_policy: str) -> bool:+ return offload_policy in ("group", "streaming", "sequential") and _streamed(denoiser)+++def _graph_count() -> int:+ """Dynamo's compiled-graph count (0 before Dynamo was ever imported)."""+ dynamo_utils = sys.modules.get("torch._dynamo.utils")+ try:+ return (+ int(dynamo_utils.counters["stats"]["unique_graphs"]) if dynamo_utils is not None else 0+ )+ except Exception: # noqa: BLE001+ return 0++def _foreign_layer(module: Any) -> Optional[str]:"""A non-offload diffusers hook (a step cache) or a root forward override that is not offload machinery."""mg = _mg()@@ -132,11 +194,24 @@ def per_call_scaled(module: Any) -> Optional[str]:return None-def cfg_batch_decline(pipe: Any, *, offload_policy: str) -> Optional[str]:- """Why both guidance branches cannot share one forward on this load, else None."""+def _speed_off_requested(speed_mode: Optional[str]) -> bool:+ """The request asked for ``speed_mode="off"`` (unset is not a request): the bit-identical reference path."""+ return speed_mode is not None and str(speed_mode).strip().lower() == "off"+++def cfg_batch_decline(+ pipe: Any,+ *,+ offload_policy: str,+ speed_mode: Optional[str] = None,+) -> Optional[str]:+ """Why both guidance branches cannot share one forward on this load, else None. ``speed_mode`` is the request's own+ value: an explicit ``off`` keeps one branch per call (a doubled batch can pick other tiles); ``force`` overrides."""mode = cfg_batch_mode()if mode == "off":return f"{CFG_BATCH_ENV}=0"+ if mode != "force" and _speed_off_requested(speed_mode):+ return f"speed_mode=off keeps one branch per call ({CFG_BATCH_ENV}=force batches anyway)"torch = _torch()denoiser = getattr(pipe, "transformer", None)if not isinstance(denoiser, torch.nn.Module):@@ -145,6 +220,9 @@ def cfg_batch_decline(pipe: Any, *, offload_policy: str) -> Optional[str]:return "two-expert denoiser"if getattr(denoiser, "_compiled_call_impl", None) is not None:return "denoiser is compiled whole"+ compiled = compiled_denoiser(denoiser)+ if compiled is not None and not _offloaded(denoiser, offload_policy):+ return f"{compiled} and resident"if getattr(pipe, "_unsloth_cuda_graphs", None):return "the denoiser replays CUDA graphs"foreign = _foreign_layer(denoiser)@@ -159,8 +237,9 @@ def cfg_batch_decline(pipe: Any, *, offload_policy: str) -> Optional[str]:"diffusers."):return f"{type(denoiser).__name__} is not on the stateless-forward list ({CFG_BATCH_ENV}=force opts it in)"- if offload_policy not in ("group", "streaming", "sequential") or not _streamed(denoiser):- return "denoiser is resident (two calls upload nothing)"+ if offload_policy == "model":+ # diffusers rebuilds model-offload hooks between renders, evicting the router: later renders would differ.+ return "model offload re-hooks the denoiser every render"return None@@ -186,10 +265,14 @@ class _Shape:self.off: Optional[str] = Noneself.off_kind: Optional[str] = Noneself.misses = 0+ self.judged = False+ self.compiled_at_trial = False+ self.trial_render = 0+ self.render_steps = 0 # steps of this shape in the current renderclass CfgBatchRouter:- """Offloaded denoiser ``forward`` running both guidance branches of a step through one forward."""+ """Denoiser ``forward`` running both guidance branches of a step through one forward."""def __init__(self,@@ -199,6 +282,10 @@ class CfgBatchRouter:logger: Any = None,) -> None:self.module_ref = weakref.ref(module)+ # Under Dynamo tracing: pass straight through (no graph breaks, no doubled-shape recompile).+ self._is_compiling = getattr(getattr(_torch(), "compiler", None), "is_compiling", None) or (+ lambda: False+ )self.orig_forward = module.forwardself.had_override = "forward" in module.__dict__self.prev_override = module.__dict__.get("forward")@@ -214,14 +301,17 @@ class CfgBatchRouter:self.last_shape: Optional[_Shape] = Noneself.render_off: Optional[str] = Noneself.render_checked = False+ self.offloaded = False+ self.pending_disable: Optional[str] = None+ self.pair_t0: Optional[float] = None+ self.renders = 0 # completed renders: a doubled-batch compile is warm-up only in a shape's trial render+ self.render_compiled: Optional[str] = Nonedef _reset_step(self) -> None:self.branch = 0self.b0: Optional[tuple] = Noneself.kept: Optional[dict] = Noneself.shape: Optional[_Shape] = None- self.pending_t0: Optional[float] = None- self.pending_plain: Optional[float] = Nonedef describe(self) -> dict[str, Any]:shape = self.shape or self.last_shape@@ -236,11 +326,17 @@ class CfgBatchRouter:if shape is None or not shape.batched_selse round(min(shape.batched_s), 4),"displaced": self._displaced(),+ "render_off": self.render_off,**self.stats,}def end_render(self) -> None:"""Drop every tensor held between calls (the kept half, the template's constants, the latent)."""+ if self.render_checked:+ self.renders += 1+ if self.pending_disable is not None:+ self.disable(self.pending_disable)+ self.pair_t0 = Noneself.kept = Noneself.template = Noneself.prev_latent = None@@ -249,10 +345,12 @@ class CfgBatchRouter:self.render_checked = Falseself._reset_step()for shape in self.shapes.values():+ shape.render_steps = 0if (shape.off_kind == _OFF_TRANSIENT- ): # memory pressure is per render: retry on the next one+ ): # memory pressure is per render: retry (and re-time) on the next oneshape.off = shape.off_kind = None+ shape.judged = Falsedef disable(self, reason: str) -> None:if self.disabled is None:@@ -279,6 +377,8 @@ class CfgBatchRouter:return module is None or module.__dict__.get("forward") is not selfdef __call__(self, *args: Any, **kwargs: Any) -> Any:+ if self._is_compiling():+ return self.orig_forward(*args, **kwargs)mg = _mg()latent = mg._latent_arg(args, kwargs)if self.disabled is not None or latent is None or self._displaced():@@ -287,6 +387,13 @@ class CfgBatchRouter:# first call of a render: a request-time step cache may have hooked the blocks since installmodule = self.module_ref()self.render_off = _foreign_hook(module) if module is not None else None+ self.render_compiled = compiled_denoiser(module) if module is not None else None+ if self.render_compiled is not None:+ if not self.offloaded:+ self.render_off = f"{self.render_compiled} and resident"+ for shape in self.shapes.values():+ if shape.judged and not shape.compiled_at_trial and shape.off is None:+ self._shape_off(shape, "the denoiser was compiled after this shape's trial")self.render_checked = Trueif self.render_off is not None:return self._plain(args, kwargs)@@ -296,22 +403,44 @@ class CfgBatchRouter:self._new_step(args, kwargs, latent)shape = self.shapeif self.branch == 0:- if self.template is not None and shape.off is None and shape.plain_pair_s is not None:+ if (+ self.template is not None+ and shape.off is None+ and shape.judged+ and shape.render_steps > _PLAIN_STEPS+ ):out = self._batched(args, kwargs, latent, shape)if out is not None:return out- if shape.off is None and shape.plain_pair_s is None and shape.steps >= 2:- return self._timed_first(args, kwargs, latent, shape)+ if shape.off is None and not shape.judged and shape.render_steps == _PLAIN_STEPS:+ self.pair_t0 = time.perf_counter() if self._sync(latent) else None+ if shape.off is None and not shape.judged and shape.activation_mib is None:+ return self._measured_plain(args, kwargs, latent, shape)return self._plain(args, kwargs)if self.branch == 1:if self.kept is not None:out = self._take(args, kwargs, shape)if out is not None:return out+ if shape.off is not None:+ # skip recording on a plain shape: it is a host sync per step (2.5-3.5% on B200)+ return self._plain(args, kwargs)self._record_template(args, kwargs, latent)- if self.pending_t0 is not None:- return self._timed_second(args, kwargs, latent, shape)- return self._plain(args, kwargs)+ out = self._plain(args, kwargs)+ if not shape.judged and self.disabled is None and shape.render_steps == _PLAIN_STEPS:+ t0, self.pair_t0 = self.pair_t0, None+ if t0 is not None and not self.offloaded and self._sync(latent):+ live = time.perf_counter() - t0+ if live >= _RESIDENT_MAX_PAIR_S:+ self._shape_off(+ shape,+ f"compute-bound: a plain pair takes {live:.3f}s, where one doubled call cannot win "+ f"{round((1 - _MEASURED_FRACTION) * 100)}%",+ )+ shape.plain_pair_s = live+ return out+ self._trial((args, kwargs), latent, shape)+ return outself.disable("more than two denoiser calls per step")return self._plain(args, kwargs)@@ -335,6 +464,7 @@ class CfgBatchRouter:None # a template predicts its own shape's second call: one plain pair first)shape.steps += 1+ shape.render_steps += 1self.shape = self.last_shape = shapedef _plain(self, args: tuple, kwargs: dict) -> Any:@@ -347,41 +477,84 @@ class CfgBatchRouter:_torch().cuda.synchronize(latent.device)return True- def _timed_first(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:+ def _measured_plain(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:+ """A plain first-branch call that records the activations it allocated (the doubled call's memory gate)."""torch = _torch()- if not self._sync(latent):- shape.plain_pair_s = 0.0 # no device timer: batch on the memory gate alone+ if getattr(latent.device, "type", None) != "cuda":return self._plain(args, kwargs)# never reset the peak counters (other measurements own them); a call that stayed under an# earlier peak leaves its activations unknownbase = torch.cuda.memory_allocated(latent.device)before = torch.cuda.max_memory_allocated(latent.device)- t0 = time.perf_counter()out = self._plain(args, kwargs)- self._sync(latent)- self.pending_t0 = time.perf_counter() - t0peak = torch.cuda.max_memory_allocated(latent.device)- shape.activation_mib = (int(peak - base) >> 20) if peak > before else None+ if peak > before:+ shape.activation_mib = int(peak - base) >> 20return out- def _timed_second(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:- self._sync(latent)- t0 = time.perf_counter()- out = self._plain(args, kwargs)- self._sync(latent)- shape.plain_pair_s = self.pending_t0 + (time.perf_counter() - t0)- self.pending_t0 = None- return out-- def _judge(self, shape: _Shape) -> None:- if len(shape.batched_s) < _BATCHED_SAMPLES or shape.plain_pair_s in (None, 0.0):+ def _trial(self, second: tuple, latent: Any, shape: _Shape) -> None:+ """Price this shape once on the step's warm arguments; outputs are discarded so renders keep one path."""+ torch = _torch()+ first = self.b0+ if first is None:+ return+ if not self._sync(latent):+ self._judge(shape)+ return+ if not self._memory_ok(latent, shape):+ self._shape_off(shape, "no room for a doubled call", _OFF_TRANSIENT)+ return+ batch = int(latent.shape[0])+ plain_s: list[float] = []+ try:+ graphs = _graph_count()+ for _ in range(_BATCHED_SAMPLES):+ t0 = time.perf_counter()+ self.orig_forward(*first[0], **first[1])+ self.orig_forward(*second[0], **second[1])+ self._sync(latent)+ plain_s.append(time.perf_counter() - t0)+ t0 = time.perf_counter()+ predicted = self._predict(latent) if self.template is not None else second+ b_args, b_kwargs = merge_branches(first, predicted, batch)+ snapshots = _snapshot(predicted)+ split_output(self.orig_forward(*b_args, **b_kwargs), batch)+ _matches(snapshots, second)+ self._sync(latent)+ shape.batched_s.append(time.perf_counter() - t0)+ del b_args, b_kwargs, snapshots, predicted+ shape.plain_pair_s = min(plain_s)+ if _graph_count() != graphs and self.render_compiled is None and not self.offloaded:+ self.disable(+ "the doubled batch compiled a new graph (the denoiser runs compiled code)"+ )+ return+ except _NotBatchable as exc:+ self._shape_off(shape, f"not batchable: {exc}")+ return+ except Exception as exc: # noqa: BLE001 - the shape stays plain; the render's own calls already ran+ if _is_oom(exc):+ try:+ torch.cuda.empty_cache()+ except Exception: # noqa: BLE001+ pass+ self._shape_off(shape, "out of memory on the timed doubled call", _OFF_TRANSIENT)+ return+ self.disable(f"the doubled call failed ({type(exc).__name__}: {exc})")returnbatched = min(shape.batched_s)- if batched > _MEASURED_FRACTION * shape.plain_pair_s:+ if shape.plain_pair_s and batched > _MEASURED_FRACTION * shape.plain_pair_s:self._shape_off(shape,f"measured no gain: batched pair {batched:.4f}s vs plain pair {shape.plain_pair_s:.4f}s",)+ return+ self._judge(shape)++ def _judge(self, shape: _Shape) -> None:+ shape.judged = True+ shape.compiled_at_trial = self.render_compiled is not None+ shape.trial_render = self.rendersdef _record_template(self, args: tuple, kwargs: dict, latent: Any) -> None:"""How the second call's arguments relate to the first's: the latent, a first-branch tensor (same object, or a@@ -436,12 +609,8 @@ class CfgBatchRouter:def _batched(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:torch = _torch()- timing = len(shape.batched_s) < _BATCHED_SAMPLES and shape.plain_pair_s not in (None, 0.0)if not self._memory_ok(latent, shape):return None- if timing:- self._sync(latent)- t0 = time.perf_counter()try:second = self._predict(latent)b_args, b_kwargs = merge_branches((args, kwargs), second, int(latent.shape[0]))@@ -458,6 +627,7 @@ class CfgBatchRouter:passself._shape_off(shape, "out of memory preparing the doubled call", _OFF_TRANSIENT)return None+ graphs = _graph_count()try:out = self.orig_forward(*b_args, **b_kwargs)mine, other = split_output(out, int(latent.shape[0]))@@ -475,10 +645,11 @@ class CfgBatchRouter:return Noneself.disable(f"the doubled call failed ({type(exc).__name__}: {exc})")return None+ if _graph_count() != graphs and self.renders > shape.trial_render:+ self.pending_disable = "the doubled batch compiled a graph after its warm-up render"self.kept = {"out": other,"snapshots": snapshots,- "t0": t0 if timing else None,"latent": latent,}self.stats["batched_steps"] += 1@@ -489,10 +660,6 @@ class CfgBatchRouter:if not _matches(kept["snapshots"], (args, kwargs)):self._miss(shape, "the second branch differed from what was batched")return None- if kept["t0"] is not None:- self._sync(kept["latent"])- shape.batched_s.append(time.perf_counter() - kept["t0"])- self._judge(shape)self.stats["hits"] += 1shape.misses = 0return kept["out"]@@ -662,12 +829,13 @@ def install_cfg_batch(pipe: Any,*,offload_policy: str,+ speed_mode: Optional[str] = None,logger: Any = None,) -> Optional[str]:"""Arm :class:`CfgBatchRouter` on ``pipe.transformer``; the decline reason, or None when armed. An optional lever:- it never fails a load."""+ it never fails a load. ``speed_mode`` is the request's value (None = unset)."""try:- return _install(pipe, offload_policy = offload_policy, logger = logger)+ return _install(pipe, offload_policy = offload_policy, speed_mode = speed_mode, logger = logger)except Exception as exc: # noqa: BLE001if logger is not None:logger.info("diffusion.cfg_batch: not armed (%s: %s)", type(exc).__name__, exc)@@ -678,9 +846,10 @@ def _install(pipe: Any,*,offload_policy: str,+ speed_mode: Optional[str] = None,logger: Any = None,) -> Optional[str]:- reason = cfg_batch_decline(pipe, offload_policy = offload_policy)+ reason = cfg_batch_decline(pipe, offload_policy = offload_policy, speed_mode = speed_mode)if reason is not None:if logger is not None:logger.info("diffusion.cfg_batch: not armed (%s)", reason)@@ -690,9 +859,14 @@ def _install(denoiser,reason = "forced"if cfg_batch_mode() == "force"- else f"offloaded denoiser ({offload_policy})",+ else (+ f"offloaded denoiser ({offload_policy})"+ if _offloaded(denoiser, offload_policy)+ else "resident denoiser"+ ),logger = logger,)+ router.offloaded = _offloaded(denoiser, offload_policy)try:_ROUTERS[pipe] = routerexcept TypeError:diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.pyindex dcc2d2ecd76..66e7a6ceafc 100644--- a/studio/backend/core/inference/diffusion_families.py+++ b/studio/backend/core/inference/diffusion_families.py@@ -323,6 +323,26 @@ _LUMINA2_BF16_WHEN_RESIDENT = (_HIDREAM_BF16_WHEN_RESIDENT = ("INT8 is barely faster than compiled bf16 on a card that holds bf16 and changes image detail")+# Small DiTs (<= ~6B) stay bf16 while it fits: quantising moves detail (L4 vs bf16 LPIPS: FP8 FLUX.2-klein-4B 0.065,+# Z-Image-Turbo 0.103; FLUX.3-klein INT8 0.027). INT8 still when bf16 would offload; such loads compile from render 1.+SMALL_DIT_BF16_WHEN_RESIDENT = (+ "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, "+ "0.103 on Z-Image-Turbo)"+)+# Bases of a small-DiT family that are not small: FLUX.2 klein 9B / base 9B (18.2 GB bf16 transformer, unmeasured)+# keep the quantized auto default. Upstream ids; mirrors fold back through ``canonical_base``.+_SMALL_DIT_LARGE_BASES = frozenset(+ {+ "black-forest-labs/flux.2-klein-9b",+ "black-forest-labs/flux.2-klein-base-9b",+ }+)+++def small_dit_rule_excludes(base_repo: Optional[str]) -> bool:+ """True when ``base_repo`` (vendor id or mirror) is a large variant the small-DiT bf16 rule must not cover."""+ return canonical_base(base_repo).lower() in _SMALL_DIT_LARGE_BASES+# FLUX.1's sd-cli encoders are BFL's own diffusers files, read as-is: sd.cpp loads the CLIP-L text_encoder and the# sharded T5 through its index (the shards sit beside it in the snapshot). The shards are listed so every fetch,@@ -438,6 +458,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (comfy_shift_node = "flux",label = "FLUX.2 Klein",profile = FamilyProfile(+ auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,preview_factors = "flux2",auto_static_skip = ((@@ -523,6 +544,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (comfy_shift_node = "flux",label = "FLUX.3 Klein Base",profile = FamilyProfile(+ auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,# Left dense for EVERY transformer-quant scheme: per-token modulations, timestep embedders, norm_out, the# text-stream per-mode blocks and the 20480-wide text projection. Measured on B200 against bf16 (11-case# recipe set, LPIPS mean): int8 0.097 -> 0.027, fp8 0.063 -> 0.033, mxfp8 0.024 with it; without it mxfp8@@ -578,6 +600,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (comfy_shift_node = "flux",label = "FLUX.3 Klein",profile = FamilyProfile(+ auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,# Left dense for EVERY transformer-quant scheme: per-token modulations, timestep embedders, norm_out, the# text-stream per-mode blocks and the 20480-wide text projection. Measured on B200 against bf16 (11-case# recipe set, LPIPS mean): int8 0.097 -> 0.027, fp8 0.063 -> 0.033, mxfp8 0.024 with it; without it mxfp8@@ -1053,6 +1076,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (default_on = True,),auto_prefer = _GATED_AUTO_PREFER,+ auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,preview_factors = "zimage",),# B200 too: 1 of 4 fresh servers picked R0_BLOCK 1024 (others 4096) and rendered other latents.diff --git a/studio/backend/core/inference/diffusion_rocm_fused.py b/studio/backend/core/inference/diffusion_rocm_fused.pyindex f959e9c7987..92788912bb2 100644--- a/studio/backend/core/inference/diffusion_rocm_fused.py+++ b/studio/backend/core/inference/diffusion_rocm_fused.py@@ -5,8 +5,9 @@RoPE reuses ``diffusion_flux2_rope``'s kernel (bit-identical to stock). AdaLN keeps stock's rounding chain; only therow-reduction order differs. Inference only, never while compiling; any ineligible call runs the forward live at-install. ``UNSLOTH_DIFFUSION_FUSED_ROPE=auto|0|1`` (auto = ROCm only), ``UNSLOTH_DIFFUSION_FUSED_ADALN=auto|0|1``-(auto = off: no gfx1151 speedup and not bit-identical).+install. ``UNSLOTH_DIFFUSION_FUSED_ROPE=auto|0|1`` (auto = ROCm and NVIDIA CUDA on Linux; ComfyUI runs FLUX RoPE+through comfy-kitchen's fused kernel by default), ``UNSLOTH_DIFFUSION_FUSED_ADALN=auto|0|1`` (auto = off: no gfx1151+speedup and not bit-identical)."""from __future__ import annotations@@ -33,6 +34,8 @@ _ADALN_PREV: dict = {}# Survives uninstall, so a fused forward restored later by another layer still reaches a real forward._ADALN_ORIGINAL: dict = {}COUNTS = {"rope_fused": 0, "rope_stock": 0, "adaln_fused": 0, "adaln_stock": 0}+# Set by the first failed RoPE launch (JIT / driver / PTX): the rest of the load runs stock, not one failing JIT per call.+_ROPE_FAILED = [False]def _mode(env: str) -> str:@@ -52,13 +55,33 @@ def _is_rocm() -> bool:return bool(getattr(torch.version, "hip", None))+def _is_nvidia_linux() -> bool:+ """NVIDIA CUDA torch on Linux (WSL included). Windows keeps stock: a Triton JIT failure would retry per call."""+ if not sys.platform.startswith("linux"):+ return False+ try:+ import torch+ except Exception: # noqa: BLE001+ return False+ return getattr(torch.version, "hip", None) is None and bool(+ getattr(torch.version, "cuda", None)+ )++# gfx1151 klein 1024^2: RoPE 8.76 -> 8.09 s pixel-identical; AdaLN adds no speed and moves pixels (LPIPS 0.004)._AUTO_ON = {FUSED_ROPE_ENV: True, FUSED_ADALN_ENV: False}+_AUTO_ON_NVIDIA = {FUSED_ROPE_ENV: True, FUSED_ADALN_ENV: False}def _switch_on(env: str) -> bool:mode = _mode(env)- return mode == "on" or (mode == "auto" and _AUTO_ON.get(env, False) and _is_rocm())+ if mode == "on":+ return True+ if mode != "auto":+ return False+ if _AUTO_ON.get(env, False) and _is_rocm():+ return True+ return _AUTO_ON_NVIDIA.get(env, False) and _is_nvidia_linux()def wanted(@@ -131,7 +154,7 @@ def _make_rope(module_name: str) -> Callable:stock = _ROPE_STOCK.get(module_name) or _ROPE_ORIGINAL[module_name]# Checked first: Dynamo graph-breaks on Triton's import path, so a compiled block stays whole on stock.if not torch.compiler.is_compiling():- launch = _rope_kernel()+ launch = None if _ROPE_FAILED[0] else _rope_kernel()if launch is not None and _rope_eligible(x, freqs_cis, use_real, use_real_unbind_dim, sequence_dim):@@ -140,7 +163,7 @@ def _make_rope(module_name: str) -> Callable:COUNTS["rope_fused"] += 1return outexcept Exception: # noqa: BLE001 -- never fail a render over the fast path- pass+ _ROPE_FAILED[0] = TrueCOUNTS["rope_stock"] += 1return stock(x,@@ -188,6 +211,7 @@ def install_rope(def uninstall_rope() -> None:with _LOCK:+ _ROPE_FAILED[0] = Falsefor name in list(_ROPE_STOCK):stock = _ROPE_STOCK.pop(name)mod = sys.modules.get(name)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 67964b1874f..e59166a39c7 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -9732,8 +9732,9 @@ class VideoBackend:logger = logger,)install_cfg_parallel(pipe, offload_policy = offload_policy, logger = logger)- # both branches through one offloaded forward; declines when CFG-parallel took the slot- install_cfg_batch(pipe, offload_policy = offload_policy, logger = logger)+ install_cfg_batch(+ pipe, offload_policy = offload_policy, speed_mode = speed_mode, logger = logger+ )# Opt-in (UNSLOTH_DIFFUSION_BG_COMPILE=1): off by default so the same seed twice repeats.from . import diffusion_bg_compile as bg_compilediff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.jsonindex 1bff063580b..f125876c89b 100644--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json@@ -1118,7 +1118,7 @@"pad": [],"zero_row": []},- "auto_bf16_when_resident": null,+ "auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)","bf16": {"convrot_filename": null,"convrot_repo": null,@@ -1212,7 +1212,7 @@"pad": [],"zero_row": []},- "auto_bf16_when_resident": null,+ "auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)","bf16": {"convrot_filename": null,"convrot_repo": null,@@ -1335,7 +1335,7 @@"pad": [],"zero_row": []},- "auto_bf16_when_resident": null,+ "auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)","bf16": {"convrot_filename": null,"convrot_repo": null,@@ -3274,7 +3274,7 @@"pad": [],"zero_row": []},- "auto_bf16_when_resident": null,+ "auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)","bf16": {"convrot_filename": null,"convrot_repo": null,@@ -3415,8 +3415,12 @@"wan2.2-ti2v-5b"],"family_auto_bf16_when_resident": {+ "flux.2-klein": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",+ "flux.3-klein": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",+ "flux.3-klein-base": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)","hidream-i1": "INT8 is barely faster than compiled bf16 on a card that holds bf16 and changes image detail",- "lumina-2": "INT8 is no faster than compiled bf16 on a card that holds bf16 and changes image detail"+ "lumina-2": "INT8 is no faster than compiled bf16 on a card that holds bf16 and changes image detail",+ "z-image": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)"},"family_auto_prefer": {"flux.1": {diff --git a/studio/backend/tests/test_diffusion_attention_blackwell.py b/studio/backend/tests/test_diffusion_attention_blackwell.pynew file mode 100644index 00000000000..b4cdb0e1669--- /dev/null+++ b/studio/backend/tests/test_diffusion_attention_blackwell.py@@ -0,0 +1,127 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Auto attention on Blackwell: native SDPA when torch's own dispatch already runs cuDNN at every DiT head dim (the+pin changes no kernel and costs a per-call wrapper), the cuDNN pin everywhere else and for an explicit request."""++from __future__ import annotations++import types++import pytest+import torch++import core.inference.diffusion_attention as att+from core.inference.diffusion_attention import apply_attention_backend, select_attention_backend++CUDNN = int(torch.nn.attention.SDPBackend.CUDNN_ATTENTION)+FLASH = int(torch.nn.attention.SDPBackend.FLASH_ATTENTION)+++class _Dit:+ def __init__(self, head_dim = 128):+ self.config = {"attention_head_dim": head_dim}+ self.set_to = None++ def set_attention_backend(self, name):+ self.set_to = name+++def _target(dtype = torch.bfloat16):+ return types.SimpleNamespace(device = "cuda", dtype = dtype)+++def blackwell(monkeypatch):+ """An NVIDIA SM100 card whose native dispatch picks cuDNN up to head_dim 128 and FLASH above (torch 2.13 / B200)."""+ att._NATIVE_DISPATCH_CACHE.clear()+ monkeypatch.delenv(att.BLACKWELL_NATIVE_ATTN_ENV, raising = False)+ monkeypatch.setenv(att.CUDNN_HEAD_DIM_PROBE_ENV, "0")+ monkeypatch.setattr(att, "_is_cuda_nvidia", lambda target: True)+ monkeypatch.setattr(att, "_cuda_capability", lambda: (10, 0))+ monkeypatch.setattr(att, "_indexed_cuda_device", lambda device: "cuda:0")+ monkeypatch.setattr(att, "guard_rocm_fused_sdpa", lambda target, logger = None: None)+ state = {"cap": (10, 0), "asked": []}+ monkeypatch.setattr(att, "_device_capability", lambda device: state["cap"])++ def choice(device, dtype, dim):+ state["asked"].append((dtype, dim))+ return CUDNN if dim <= 128 else FLASH++ monkeypatch.setattr(att, "_native_sdp_choice", choice)+ yield state+ att._NATIVE_DISPATCH_CACHE.clear()+++def test_auto_pick_still_reads_as_cudnn_for_every_caller(blackwell):+ pick = select_attention_backend(_target(), "auto", speed_active = True)+ assert pick == "_native_cudnn" and isinstance(pick, att._AutoCudnn)+ explicit = select_attention_backend(_target(), "cudnn", speed_active = True)+ assert explicit == "_native_cudnn" and not isinstance(explicit, att._AutoCudnn)+ assert select_attention_backend(_target(), "auto", speed_active = False) is None+++def test_blackwell_auto_leaves_native_when_dispatch_already_picks_cudnn(blackwell):+ dit = _Dit(128)+ pick = select_attention_backend(_target(), "auto", speed_active = True)+ assert (+ apply_attention_backend(types.SimpleNamespace(transformer = dit), pick, target = _target())+ is None+ )+ assert dit.set_to in (None, "native")+ assert blackwell["asked"] == [(torch.bfloat16, 128)]+++def test_an_explicit_cudnn_request_is_still_pinned_on_blackwell(blackwell):+ dit = _Dit(128)+ engaged = apply_attention_backend(+ types.SimpleNamespace(transformer = dit), "_native_cudnn", target = _target()+ )+ assert engaged == "_native_cudnn" and dit.set_to == "_native_cudnn"+ assert blackwell["asked"] == []+++ "case",+ ["hopper", "head_dim_256", "fp32", "kill_switch", "unknown_head_dim", "no_target"],+)+def test_auto_keeps_the_cudnn_pin_wherever_native_could_differ(blackwell, monkeypatch, case):+ dit = _Dit(256 if case == "head_dim_256" else 128)+ target = _target(torch.float32 if case == "fp32" else torch.bfloat16)+ if case == "hopper":+ blackwell["cap"] = (9, 0)+ if case == "kill_switch":+ monkeypatch.setenv(att.BLACKWELL_NATIVE_ATTN_ENV, "0")+ if case == "unknown_head_dim":+ dit.config = {}+ pick = select_attention_backend(target, "auto", speed_active = True)+ engaged = apply_attention_backend(+ types.SimpleNamespace(transformer = dit), pick, target = None if case == "no_target" else target+ )+ assert engaged == "_native_cudnn" and dit.set_to == "_native_cudnn"+ assert type(engaged) is str # the marker never leaks past apply (status, cache keys)+++def test_mixed_head_dims_need_cudnn_at_every_one(blackwell):+ pipe = types.SimpleNamespace(transformer = _Dit(128), transformer_2 = _Dit(256))+ assert att._native_dispatches_cudnn(pipe, _target()) is False+ blackwell["asked"].clear()+ att._NATIVE_DISPATCH_CACHE.clear()+ assert (+ att._native_dispatches_cudnn(types.SimpleNamespace(transformer = _Dit(64)), _target()) is True+ )+ assert (+ att._native_dispatches_cudnn(types.SimpleNamespace(transformer = _Dit(64)), _target()) is True+ )+ assert blackwell["asked"] == [(torch.bfloat16, 64)] # cached per (device, dtype, head_dim)+++def test_an_unaskable_probe_keeps_the_pin(blackwell, monkeypatch):+ def boom(device, dtype, dim):+ raise RuntimeError("no device")++ monkeypatch.setattr(att, "_native_sdp_choice", boom)+ assert (+ att._native_dispatches_cudnn(types.SimpleNamespace(transformer = _Dit(128)), _target())+ is False+ )diff --git a/studio/backend/tests/test_diffusion_autoopt_parity.py b/studio/backend/tests/test_diffusion_autoopt_parity.pyindex 5ea15a38a12..787c0099080 100644--- a/studio/backend/tests/test_diffusion_autoopt_parity.py+++ b/studio/backend/tests/test_diffusion_autoopt_parity.py@@ -473,6 +473,9 @@ def snapshot(entry: Callable) -> dict:patch.setattr(diffusion_flux2_rope, "_kernel", lambda: (lambda *a, **k: None))patch.setattr(video_wan_fused, "_kernels", lambda: {})+ from core.inference import diffusion_rocm_fused++ patch.setattr(diffusion_rocm_fused, "_is_nvidia_linux", lambda: False)out: dict = {}try:for backend, fam in _families():diff --git a/studio/backend/tests/test_diffusion_cfg_batch.py b/studio/backend/tests/test_diffusion_cfg_batch.pyindex c13374ecaa5..65b4e70bf33 100644--- a/studio/backend/tests/test_diffusion_cfg_batch.py+++ b/studio/backend/tests/test_diffusion_cfg_batch.py@@ -1,7 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""CFG batching (diffusion_cfg_batch): both guidance branches through one offloaded forward, CPU fakes."""+"""CFG batching (diffusion_cfg_batch): both guidance branches through one forward (offloaded or resident), CPU fakes."""from __future__ import annotations@@ -95,17 +95,18 @@ def _flux_loop(neg = None,guided = None,between = None,+ device = "cpu",):"""The diffusers FLUX.2-klein loop: per step one latent object, cond then uncond call, CFG combine."""torch.manual_seed(0)- latents = torch.randn(1, 4, 8)- pos = torch.randn(1, 5, 8)- neg = torch.randn(1, 5, 8) if neg is None else neg- txt_ids = torch.zeros(1, 5, 4)- img_ids = torch.ones(1, 4, 4)+ latents = torch.randn(1, 4, 8).to(device)+ pos = torch.randn(1, 5, 8).to(device)+ neg = (torch.randn(1, 5, 8) if neg is None else neg).to(device)+ txt_ids = torch.zeros(1, 5, 4, device = device)+ img_ids = torch.ones(1, 4, 4, device = device)outs = []for i in range(steps):- t = torch.full((1,), 1.0 - i / steps)+ t = torch.full((1,), 1.0 - i / steps, device = device)latent_model_input = latents.clone()cond = model(hidden_states = latent_model_input,@@ -155,10 +156,27 @@ def test_batched_steps_match_two_calls_and_engage(monkeypatch):assert model.calls[-2:] == [2, 2]assert cb.cfg_batch_status(pipe)["batched_steps"] == 4cb.end_cfg_batch_render(pipe)- assert cb.cfg_batch_status(pipe)["plain_pair_s"] is not None or not torch.cuda.is_available()+ assert cb.cfg_batch_status(pipe)["plain_pair_s"] is Noneassert cb.cfg_batch_status(pipe)["shape_off"] is None[email protected](not torch.cuda.is_available(), reason = "needs a CUDA device timer")+def test_cuda_latents_time_the_step_two_pair_and_keep_the_image(monkeypatch):+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()).cuda(), device = "cuda")+ model = Flux2Transformer2DModel().cuda()+ pipe, router = _armed(model, monkeypatch)+ got, _ = _flux_loop(model, device = "cuda")+ torch.testing.assert_close(got, ref)+ status = cb.cfg_batch_status(pipe)+ assert status["plain_pair_s"] is not None and status["batched_pair_s"] is not None+ if status["shape_off"] is None:+ assert router.stats["batched_steps"] == 4 and model.calls[-2:] == [2, 2]+ else:+ assert "measured no gain" in status["shape_off"]+ assert router.stats["batched_steps"] == 0 and model.calls[-2:] == [1, 1]+ cb.end_cfg_batch_render(pipe)++def test_wan_expanded_timesteps_serve_the_second_branch(monkeypatch):"""WanPipeline's stride-0 ``t.expand(batch)`` timestep must compare equal, not disable batching."""@@ -220,7 +238,7 @@ def test_qwen_unequal_prompt_lengths_and_img_shapes_batch(monkeypatch):def test_a_changed_second_branch_reruns_plain_and_stops_after_three_misses(monkeypatch):# uncond prompt rewritten in place: the doubled call must miss, rerun plain, and match the two-call resultdef between(i, neg):- if i >= 2:+ if i >= 3:neg.add_(0.5)ref, _ = _flux_loop(@@ -230,7 +248,7 @@ def test_a_changed_second_branch_reruns_plain_and_stops_after_three_misses(monke_pipe, router = _armed(model, monkeypatch)got, _ = _flux_loop(model, steps = 8, neg = torch.zeros(1, 5, 8), between = between)torch.testing.assert_close(got, ref, rtol = 0, atol = 0)- assert router.stats["misses"] == 3 and router.stats["hits"] == 0+ assert router.stats["misses"] == 3 and router.stats["hits"] == 1assert router.shape.off is not None and "3 consecutive misses" in router.shape.off@@ -287,10 +305,285 @@ def test_switches_are_independent_of_multi_gpu(monkeypatch):assert cb.cfg_batch_decline(pipe, offload_policy = "group") == f"{cb.CFG_BATCH_ENV}=0"-def test_declines(monkeypatch):+def test_a_resident_denoiser_is_armed_by_default(monkeypatch):resident = _Pipe(Flux2Transformer2DModel())- assert "resident" in cb.cfg_batch_decline(resident, offload_policy = "none")+ assert cb.cfg_batch_decline(resident, offload_policy = "none") is None+ assert cb.install_cfg_batch(resident, offload_policy = "none") is None+ assert cb.router_for(resident).reason == "resident denoiser"+ assert "model offload" in cb.cfg_batch_decline(+ _Pipe(Flux2Transformer2DModel()), offload_policy = "model"+ )+ monkeypatch.setenv(cb.CFG_BATCH_ENV, "0")+ assert cb.cfg_batch_decline(_Pipe(Flux2Transformer2DModel()), offload_policy = "none") == (+ f"{cb.CFG_BATCH_ENV}=0"+ )+++class _Clock:+ """A fake device timer: each forward costs ``cost(batch)`` seconds of the router's perf_counter."""++ def __init__(self, cost):+ self.now = 0.0+ self.cost = cost++ def __call__(self):+ return self.now+++def _timed(monkeypatch, cost):+ clock = _Clock(cost)+ monkeypatch.setattr(cb.time, "perf_counter", clock)+ monkeypatch.setattr(cb, "_RESIDENT_MAX_PAIR_S", 1e9)+ monkeypatch.setattr(cb.CfgBatchRouter, "_sync", lambda self, latent: True)++ def forward(self, *a, **k):+ clock.now += clock.cost(int(k["hidden_states"].shape[0]))+ return Flux2Transformer2DModel.forward(self, *a, **k)++ timed = type("Flux2Transformer2DModel", (Flux2Transformer2DModel,), {"forward": forward})+ timed.__module__ = "diffusers.models.fake"+ return timed()++[email protected]("batched_cost, wins", [(1.2, True), (2.0, False)])+def test_the_timing_trial_never_mixes_paths_and_repeats_bit_identically(+ monkeypatch, batched_cost, wins+):+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else batched_cost)+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ router = cb.router_for(pipe)+ first, _ = _flux_loop(model)+ cb.end_cfg_batch_render(pipe)+ calls_after_first = list(model.calls)+ second, _ = _flux_loop(model)+ cb.end_cfg_batch_render(pipe)+ torch.testing.assert_close(first, second, rtol = 0, atol = 0)+ assert calls_after_first[:10] == [1, 1, 1, 1, 1, 1, 2, 1, 1, 2]+ render2 = model.calls[len(calls_after_first) :]+ if wins:+ assert render2 == [1, 1, 1, 1] + [2] * 4 and router.last_shape.off is None+ else:+ assert render2 == [1] * 12 and "measured no gain" in router.last_shape.off+ plain = Flux2Transformer2DModel()+ ref_plain, _ = _flux_loop(plain)+ if not wins:+ torch.testing.assert_close(second, ref_plain, rtol = 0, atol = 0)+ else:+ torch.testing.assert_close(second, ref_plain)+++def test_a_shape_that_stays_plain_records_no_prediction(monkeypatch):+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else 2.0)+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ router = cb.router_for(pipe)+ _flux_loop(model)+ cb.end_cfg_batch_render(pipe)+ assert "measured no gain" in router.last_shape.off+ recorded = []+ monkeypatch.setattr(router, "_record_template", lambda *a, **k: recorded.append(1))+ _flux_loop(model)+ cb.end_cfg_batch_render(pipe)+ assert recorded == [] and model.calls[-12:] == [1] * 12+++def _quiet_flux():+ """The fake FLUX.2 transformer without its call log (a list append Dynamo would have to trace)."""++ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ txt_ids,+ img_ids,+ return_dict = False,+ **_,+ ):+ text = encoder_hidden_states.mean(dim = 1, keepdim = True)+ return (+ hidden_states * self.w+ + text+ + timestep.view(-1, 1, 1)+ + txt_ids.mean()+ + img_ids.mean(),+ )++ cls = type("Flux2Transformer2DModel", (Flux2Transformer2DModel,), {"forward": forward})+ cls.__module__ = "diffusers.models.fake"+ return cls()+++def test_compiled_denoisers_are_seen_in_every_form():+ plain = Flux2Transformer2DModel()+ assert cb.compiled_denoiser(plain) is None+ marked = Flux2Transformer2DModel()+ marked._unsloth_regional_compile_kwargs = {"dynamic": False}+ assert cb.compiled_denoiser(marked) == "the denoiser is compiled"+ block = Flux2Transformer2DModel()+ block.add_module("blocks", torch.nn.ModuleList([torch.nn.Linear(1, 1)]))+ block.blocks[0].compile()+ assert "blocks.0 is compiled" == cb.compiled_denoiser(block)+ # #14528: the compile moved below a group-offload hook, the block's _compiled_call_impl cleared+ hooked = Flux2Transformer2DModel()+ hooked.add_module("blocks", torch.nn.ModuleList([torch.nn.Linear(1, 1)]))+ ref = types.SimpleNamespace(forward = torch.compile(hooked.blocks[0].forward))+ hooked.blocks[0]._diffusers_hook = types.SimpleNamespace(+ hooks = {"group_offloading": object()}, _fn_refs = [ref]+ )+ assert "below its hooks" in cb.compiled_denoiser(hooked)+ assert (+ cb.cfg_batch_decline(_Pipe(marked), offload_policy = "none")+ == "the denoiser is compiled and resident"+ )+ assert cb.cfg_batch_decline(_Pipe(_offloaded(marked)), offload_policy = "group") is None+++def test_a_compile_on_a_resident_denoiser_after_install_keeps_every_render_plain(monkeypatch):+ model = Flux2Transformer2DModel()+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ model._unsloth_regional_compile_kwargs = {+ "dynamic": False+ } # the deferred profile compiled the blocks+ _flux_loop(model)+ cb.end_cfg_batch_render(pipe)+ _flux_loop(model)+ assert model.calls == [1] * 24+ assert cb.cfg_batch_status(pipe)["render_off"] == "the denoiser is compiled and resident"+++def test_an_offloaded_shape_judged_before_a_compile_stays_plain_after_it(monkeypatch):+ model = Flux2Transformer2DModel()+ pipe, router = _armed(model, monkeypatch)+ _flux_loop(model) # eager render: trial on step 1, steps 2-6 batched+ cb.end_cfg_batch_render(pipe)+ assert router.stats["batched_steps"] == 4+ model._unsloth_regional_compile_kwargs = {+ "dynamic": False+ } # compiled below the offload hooks afterwards+ del model.calls[:]+ _flux_loop(model)+ assert model.calls == [1] * 12 # no doubled call reaches the new compile on a follow-up prompt+ assert "compiled after this shape's trial" in router.last_shape.off+++def test_an_offloaded_compiled_denoiser_batches_from_its_trial_render(monkeypatch):+ model = Flux2Transformer2DModel()+ model._unsloth_regional_compile_kwargs = {"dynamic": False}+ pipe, router = _armed(model, monkeypatch)+ for _ in range(2):+ _flux_loop(model)+ cb.end_cfg_batch_render(pipe)+ assert router.stats["batched_steps"] == 8 and router.disabled is None+ assert router.last_shape.compiled_at_trial and router.last_shape.trial_render == 0+++def test_a_doubled_call_compiling_on_a_follow_up_render_ends_batching_after_that_render(+ monkeypatch,+):+ model = Flux2Transformer2DModel()+ pipe, router = _armed(model, monkeypatch)+ _flux_loop(model)+ cb.end_cfg_batch_render(pipe)+ graphs = iter(range(1000))+ monkeypatch.setattr(+ cb, "_graph_count", lambda: next(graphs)+ ) # every doubled call "compiles" from now on+ del model.calls[:]+ _flux_loop(model)+ assert model.calls == [1, 1, 1, 1] + [2] * 4 # render 2 stays on its one path+ cb.end_cfg_batch_render(pipe)+ assert "after its warm-up render" in router.disabled+ del model.calls[:]+ _flux_loop(model)+ assert model.calls == [1] * 12+++def test_a_doubled_call_that_compiles_a_graph_turns_batching_off(monkeypatch):+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else 1.2)+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ graphs = iter(range(100))+ monkeypatch.setattr(cb, "_graph_count", lambda: next(graphs))+ _flux_loop(model)+ router = cb.router_for(pipe)+ assert "compiled a new graph" in router.disabled+ assert (+ model.calls[:4] == [1, 1, 1, 1] and model.calls[-8:] == [1] * 8+ ) # nothing batched after the trial+++def test_a_router_traced_by_dynamo_passes_straight_through():+ model = _quiet_flux()+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ torch._dynamo.reset()+ compiled = torch.compile(model, backend = "eager", fullgraph = True)+ kw = dict(+ timestep = torch.ones(1),+ encoder_hidden_states = torch.randn(1, 5, 8),+ txt_ids = torch.zeros(1, 5, 4),+ img_ids = torch.ones(1, 4, 4),+ )+ for batch in (1, 2):+ x = torch.randn(batch, 4, 8)+ out = compiled(hidden_states = x, **{**kw, "timestep": torch.ones(batch)})[0]+ torch.testing.assert_close(+ out,+ model.forward.orig_forward(hidden_states = x, **{**kw, "timestep": torch.ones(batch)})[0],+ )+ assert (+ cb.router_for(pipe).stats["plain_calls"] == 0+ ) # the traced calls never ran the router's own logic++[email protected]("pair_s, gated", [(0.3, True), (0.1, False)])+def test_a_compute_bound_resident_pair_skips_the_trial(monkeypatch, pair_s, gated):+ model = _timed(monkeypatch, lambda b: pair_s / 2 if b == 1 else pair_s)+ monkeypatch.setattr(cb, "_RESIDENT_MAX_PAIR_S", 0.25)+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ _flux_loop(model)+ off = cb.router_for(pipe).last_shape.off+ if gated:+ assert "compute-bound" in off and model.calls == [1] * 12+ else:+ assert "measured no gain" in off and 2 in model.calls # the trial ran its doubled calls+++def test_an_offloaded_pair_is_never_gated_as_compute_bound(monkeypatch):+ model = _timed(monkeypatch, lambda b: 5.0 if b == 1 else 6.0)+ monkeypatch.setattr(cb, "_RESIDENT_MAX_PAIR_S", 0.25)+ pipe = _Pipe(_offloaded(model))+ assert cb.install_cfg_batch(pipe, offload_policy = "group") is None+ _flux_loop(model)+ assert (+ cb.router_for(pipe).last_shape.off is None and model.calls[-1] == 2+ ) # an upload saved per step wins+++def test_one_slow_plain_sample_does_not_decide_the_trial(monkeypatch):+ seen = {"n": 0}+ def cost(b):+ seen["n"] += 1+ if seen["n"] == 5: # the trial's first plain call (after steps 1 and 2)+ return 50.0+ return 1.0 if b == 1 else 2.0++ model = _timed(monkeypatch, cost)+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ _flux_loop(model)+ cb.end_cfg_batch_render(pipe)+ shape = cb.router_for(pipe).last_shape+ assert shape.off is not None and "measured no gain" in shape.off+ assert shape.plain_pair_s == 2.0 and min(shape.batched_s) == 2.0+++def test_declines(monkeypatch):class ZImageTransformer2DModel(Flux2Transformer2DModel):pass@@ -411,3 +704,52 @@ def test_a_new_request_shape_starts_without_the_previous_shapes_template(monkeypbig = torch.randn(2, 4, 8)router._new_step((), {"hidden_states": big}, big)assert router.last_shape is not first and router.template is None+++def test_the_trial_prices_the_batched_steps_preparation_and_verification(monkeypatch):+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else 1.2)+ clock = cb.time.perf_counter+ real_matches = cb._matches++ def slow_matches(snapshots, call):+ clock.now += 0.7+ return real_matches(snapshots, call)++ monkeypatch.setattr(cb, "_matches", slow_matches)+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ _flux_loop(model)+ shape = cb.router_for(pipe).last_shape+ assert shape.off is not None and "measured no gain" in shape.off+ assert min(shape.batched_s) == pytest.approx(1.9)++[email protected]("offload_policy", ["none", "group"])+def test_an_explicit_speed_off_request_keeps_one_branch_per_call(monkeypatch, offload_policy):+ def pipe():+ model = Flux2Transformer2DModel()+ return _Pipe(_offloaded(model) if offload_policy == "group" else model)++ off = pipe()+ reason = cb.install_cfg_batch(off, offload_policy = offload_policy, speed_mode = "off")+ assert reason is not None and "speed_mode=off" in reason and cb.router_for(off) is None+ assert "forward" not in off.transformer.__dict__+ assert "speed_mode=off" in cb.cfg_batch_decline(+ pipe(), offload_policy = offload_policy, speed_mode = " OFF "+ )+ for speed in (None, "default", "max"): # unset keeps the default-on lever+ assert cb.cfg_batch_decline(pipe(), offload_policy = offload_policy, speed_mode = speed) is None+ monkeypatch.setenv(cb.CFG_BATCH_ENV, "force")+ assert cb.cfg_batch_decline(pipe(), offload_policy = offload_policy, speed_mode = "off") is None+++def test_both_backends_pass_the_requested_speed_mode():+ import inspect++ from core.inference import diffusion, video+ for module in (diffusion, video):+ src = inspect.getsource(module)+ calls = [c for c in src.split("install_cfg_batch(")[1:] if not c.startswith("pipe: Any")]+ assert calls and all(+ "speed_mode = speed_mode" in c.split(")")[0] for c in calls+ ), module.__name__diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.pyindex 0c17ad59d79..9fb4dce6284 100644--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py@@ -1291,6 +1291,7 @@ def test_an_uncompilable_family_loads_an_operators_checkpoint_under_auto(def test_a_compilable_family_loads_without_the_bf16_replan(fake_runtime, monkeypatch):backend, spy = _load_backend(monkeypatch)monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)+ monkeypatch.setattr(dmod, "auto_bf16_when_resident_reason", lambda _name: None)replans = []monkeypatch.setattr(DiffusionBackend,@@ -1405,7 +1406,12 @@ def test_a_measured_bf16_family_quantises_auto_when_the_budget_is_unmeasured(("Alpha-VLLM/Lumina-Image-2.0", True),("HiDream-ai/HiDream-I1-Full", True),("black-forest-labs/FLUX.1-dev", False),- ("Tongyi-MAI/Z-Image-Turbo", False),+ ("Tongyi-MAI/Z-Image-Turbo", True),+ ("black-forest-labs/FLUX.2-klein-4B", True),+ ("black-forest-labs/FLUX.2-klein-base-4B", True),+ ("black-forest-labs/flux-3-image-klein", True),+ ("black-forest-labs/flux-3-image-klein-base", True),+ ("black-forest-labs/FLUX.2-dev", False),("Qwen/Qwen-Image", False),],)@@ -1870,3 +1876,71 @@ def test_the_streamed_seed_credits_host_ram_the_outgoing_pipeline_frees(),)assert (_settle(backend) == "fp8") is expected+++def _small_dit_kept_bf16(monkeypatch, reason):+ _measured_bf16_family(monkeypatch)+ monkeypatch.setattr(dmod, "auto_bf16_when_resident_reason", lambda _name: reason)+ monkeypatch.setattr(dmod, "compile_eligible", lambda *_a, **_k: True)+ monkeypatch.setattr(dmod, "fp16_compile_explicit_only", lambda _t: False)+++ "reason, speed, expect_speed, expect_deferred",+ [+ (dmod.SMALL_DIT_BF16_WHEN_RESIDENT, None, "default", False),+ (dmod.SMALL_DIT_BF16_WHEN_RESIDENT, "off", "off", False),+ ("measured: no faster", None, "off", True),+ ],+)+def test_a_small_dit_kept_bf16_compiles_from_render_1(+ fake_runtime, monkeypatch, reason, speed, expect_speed, expect_deferred+):+ backend, spy = _load_backend(monkeypatch)+ _small_dit_kept_bf16(monkeypatch, reason)+ status = _load(+ backend, speed_mode = speed, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ()+ )++ assert spy.quantised == [] and spy.seeds == []+ assert status["resolved"]["transformer_quant"]["value"] == "off"+ assert backend._state.speed_mode == expect_speed+ assert backend._state.speed_deferred is expect_deferred+++ "repo, kept",+ [+ ("black-forest-labs/FLUX.2-klein-4B", True),+ ("black-forest-labs/FLUX.2-klein-base-4B", True),+ ("unsloth/FLUX.2-klein-4B", True),+ ("black-forest-labs/FLUX.2-klein-9B", False),+ ("black-forest-labs/FLUX.2-klein-base-9B", False),+ ("unsloth/FLUX.2-klein-9B", False),+ ("unsloth/FLUX.2-klein-base-9B", False),+ ],+)+def test_the_small_dit_rule_skips_the_9b_klein_variants(repo, kept):+ fam = detect_family_for_pick(repo, None, None)+ assert fam is not None and fam.name == "flux.2-klein"+ assert (dmod._auto_keeps_bf16_reason(fam, "pipeline", repo) is not None) is kept+ fits = types.SimpleNamespace(+ offload_policy = "none",+ estimates = {"safe_device_budget_mib": 60_000, "resident_required_mib": 30_000},+ )+ assert (dmod._auto_quant_eager_reason(fam, fits, None, "pipeline", repo) is not None) is kept+++def test_a_large_variant_of_a_small_dit_family_quantises_under_auto(fake_runtime, monkeypatch):+ backend, spy = _load_backend(monkeypatch)+ _small_dit_kept_bf16(monkeypatch, dmod.SMALL_DIT_BF16_WHEN_RESIDENT)+ seen = []++ def excludes(base):+ seen.append(base)+ return base == Z_IMAGE_REPO++ monkeypatch.setattr(dmod, "small_dit_rule_excludes", excludes)+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())+ assert Z_IMAGE_REPO in seen # the load hands the decider its resolved base+ assert spy.quantised == ["auto"]diff --git a/studio/backend/tests/test_diffusion_rocm_fused.py b/studio/backend/tests/test_diffusion_rocm_fused.pyindex 2587b41b1a9..f653c96abf6 100644--- a/studio/backend/tests/test_diffusion_rocm_fused.py+++ b/studio/backend/tests/test_diffusion_rocm_fused.py@@ -1,8 +1,9 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Tests for ``diffusion_rocm_fused.py``: the switches (auto = ROCm only), install / uninstall wiring, and on a GPU the-kernels against diffusers (RoPE bit-identical, AdaLN within one rounding of stock) plus a tiny FLUX.1 transformer."""+"""Tests for ``diffusion_rocm_fused.py``: the switches (auto = RoPE on ROCm and on NVIDIA CUDA under Linux, AdaLN+opt-in), install / uninstall wiring, and on a GPU the kernels against diffusers (RoPE bit-identical, AdaLN within one+rounding of stock) plus tiny FLUX.1 / FLUX.2 transformers."""from __future__ import annotations@@ -54,12 +55,59 @@ def fake_kernels(monkeypatch):monkeypatch.setattr(rf, "_adaln_kernel", lambda: (lambda x, scale, shift, eps: None))-def test_auto_is_inert_off_rocm(fake_kernels, monkeypatch):+def _nvidia(monkeypatch, platform = "linux"):monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.setattr(torch.version, "cuda", "13.0", raising = False)+ monkeypatch.setattr(rf.sys, "platform", platform)++[email protected]("dtype", [torch.bfloat16, torch.float16])+def test_auto_engages_rope_only_on_nvidia_linux(fake_kernels, monkeypatch, dtype):+ _nvidia(monkeypatch)+ assert rf.install_for_pipe(_PipeLike(), dtype, "cuda") == {"rope": True, "adaln": 0}+ assert fmod.apply_rotary_emb is rf._ROPE_FNS[rf._MODULES[0]]+ assert nm.AdaLayerNormZero.forward is _STOCK_FWD["AdaLayerNormZero"]++[email protected]("platform", ["win32", "darwin"])+def test_auto_is_inert_on_nvidia_outside_linux(fake_kernels, monkeypatch, platform):+ # Windows keeps stock: a Triton JIT failure there would be retried on every call.+ _nvidia(monkeypatch, platform)+ assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}+ assert not rf.is_installed()+++def test_auto_is_inert_without_a_gpu_torch(fake_kernels, monkeypatch):+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.setattr(torch.version, "cuda", None, raising = False)+ monkeypatch.setattr(rf.sys, "platform", "linux")assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}assert not rf.is_installed()[email protected]("value", ["0", "off", "false"])+def test_kill_switch_on_nvidia(fake_kernels, monkeypatch, value):+ _nvidia(monkeypatch)+ monkeypatch.setenv(rf.FUSED_ROPE_ENV, value)+ assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}+ assert fmod.apply_rotary_emb is stock_rope+++def test_flux2_fp16_keeps_its_own_patch_on_nvidia(fake_kernels, monkeypatch):+ from core.inference import diffusion_flux2_rope as fr++ f2mod = pytest.importorskip("diffusers.models.transformers.transformer_flux2")+ _nvidia(monkeypatch)+ monkeypatch.setattr(fr, "_kernel", lambda: (lambda *a, **k: None))+ pipe = _PipeLike("diffusers.models.transformers.transformer_flux2")+ try:+ assert fr.install_for_pipe(pipe, torch.float16, "cuda")+ assert rf.install_for_pipe(pipe, torch.float16, "cuda") == {"rope": False, "adaln": 0}+ assert f2mod.apply_rotary_emb is fr._fused_apply_rotary_emb+ finally:+ fr.uninstall()++def test_auto_engages_rope_only_on_rocm(fake_kernels, monkeypatch):# auto = the bit-identical RoPE kernel on ROCm; fused AdaLN bought no speed on gfx1151 and is opt-in.monkeypatch.setattr(torch.version, "hip", "7.2.0", raising = False)@@ -329,6 +377,55 @@ def test_tiny_flux1_transformer_end_to_end(monkeypatch):assert rel < 1e-2, float(rel)+@needs_gpu+def test_tiny_flux2_transformer_default_on(monkeypatch):+ f2 = pytest.importorskip("diffusers").Flux2Transformer2DModel+ if not (rf._is_rocm() or rf._is_nvidia_linux()):+ pytest.skip("auto engages on ROCm and NVIDIA Linux only")+ dev, dtype = "cuda", torch.bfloat16+ torch.manual_seed(0)+ model = (+ f2(+ patch_size = 1,+ in_channels = 16,+ num_layers = 2,+ num_single_layers = 2,+ attention_head_dim = 32,+ num_attention_heads = 4,+ joint_attention_dim = 64,+ timestep_guidance_channels = 32,+ axes_dims_rope = (8, 8, 8, 8),+ guidance_embeds = False,+ )+ .to(dev, dtype)+ .eval()+ )+ img, txt = 256, 32+ ids = torch.cartesian_prod(torch.arange(1), torch.arange(16), torch.arange(16), torch.arange(1))+ inputs = dict(+ hidden_states = torch.randn(1, img, 16, device = dev, dtype = dtype),+ encoder_hidden_states = torch.randn(1, txt, 64, device = dev, dtype = dtype),+ timestep = torch.tensor([0.5], device = dev, dtype = dtype),+ img_ids = ids.to(dev)[None].float(),+ txt_ids = torch.cartesian_prod(+ torch.arange(1), torch.arange(1), torch.arange(1), torch.arange(txt)+ )+ .to(dev)[None]+ .float(),+ return_dict = False,+ )+ with torch.no_grad():+ want = model(**inputs)[0]+ assert rf.install_for_pipe(type("P", (), {"transformer": model})(), dtype, dev) == {+ "rope": True,+ "adaln": 0,+ }+ r0 = rf.COUNTS["rope_fused"]+ got = model(**inputs)[0]+ assert rf.COUNTS["rope_fused"] - r0 == 2 * 2 + 2 * 2+ assert torch.equal(got, want)++def test_teardown_survives_a_later_eager_layer(fake_kernels, monkeypatch):# Deferred speed profile: fused AdaLN at load, Studio's eager patch layered on top at the 3rd image, then unload# in the order diffusion.py uses (fused teardown first, eager after).@@ -364,3 +461,27 @@ def test_stock_branch_never_loses_its_previous_forward(fake_kernels, monkeypatchfinally:for n, fwd in _STOCK_FWD.items():getattr(nm, n).forward = fwd+++def test_a_failed_rope_launch_disables_the_fused_path_for_the_load(monkeypatch):+ calls = []++ def failing_launch(x, cos, sin):+ calls.append(1)+ raise RuntimeError("PTX JIT compilation failed")++ monkeypatch.setattr(rf, "_rope_kernel", lambda: failing_launch)+ monkeypatch.setattr(rf, "_rope_eligible", lambda *a, **k: True)+ _nvidia(monkeypatch)+ assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")["rope"] is True+ x = torch.randn(1, 6, 2, 8)+ pos = torch.randn(6, 4, dtype = torch.float64)+ freqs = (pos.cos().repeat_interleave(2, -1).float(), pos.sin().repeat_interleave(2, -1).float())+ want = stock_rope(x, freqs, sequence_dim = 1)+ for _ in range(4):+ assert torch.equal(fmod.apply_rotary_emb(x, freqs, sequence_dim = 1), want)+ assert len(calls) == 1+ rf.uninstall()+ rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")+ fmod.apply_rotary_emb(x, freqs, sequence_dim = 1)+ assert len(calls) == 2diff --git a/studio/backend/tests/test_diffusion_step_skip_defaults.py b/studio/backend/tests/test_diffusion_step_skip_defaults.pyindex bd07b62849c..b97030ef096 100644--- a/studio/backend/tests/test_diffusion_step_skip_defaults.py+++ b/studio/backend/tests/test_diffusion_step_skip_defaults.py@@ -47,7 +47,7 @@ DEV = "black-forest-labs/FLUX.1-dev"@pytest.mark.parametrize("ids, tier, steps, want",[- ((KREA,), "default", 28, {"every": 2, "min_steps": 20}),+ ((KREA,), "default", 28, None),((KREA,), "max", 28, {"every": 3, "min_steps": 20}),((KREA,), "eager", 28, None),((KREA,), "off", 28, None),@@ -56,11 +56,12 @@ DEV = "black-forest-labs/FLUX.1-dev"((DEV,), "default", 28, None), # max-only row((DEV,), "max", 28, {"every": 2, "min_steps": 20}),# The unsloth mirror and a local path with the upstream as its base both resolve to the upstream row.- (("unsloth/FLUX.1-Krea-dev",), "default", 28, {"every": 2, "min_steps": 20}),- (("/models/my-krea", KREA), "default", 28, {"every": 2, "min_steps": 20}),+ (("unsloth/FLUX.1-Krea-dev",), "max", 28, {"every": 3, "min_steps": 20}),+ (("/models/my-krea", KREA), "max", 28, {"every": 3, "min_steps": 20}),+ (("unsloth/FLUX.1-Krea-dev",), "default", 28, None),(("black-forest-labs/FLUX.1-schnell",), "max", 28, None),((None,), "max", 28, None),- (KREA, "default", 28, {"every": 2, "min_steps": 20}), # a bare string+ (KREA, "max", 28, {"every": 3, "min_steps": 20}), # a bare string],)def test_plan_follows_the_tier_and_the_default_steps(table, ids, tier, steps, want):@@ -235,10 +236,10 @@ def _probe_plan(entry):@pytest.mark.parametrize("speed, killed, want_static",[- ("default", False, True),+ ("default", False, False),("max", False, True),("eager", False, False),- ("default", True, False),+ ("max", True, False),],)def test_auto_load_installs_static_for_a_listed_model(@@ -345,11 +346,7 @@ def test_auto_skip_runs_on_txt2img_only_explicit_everywhere(backend.unload()-def test_deferred_default_tier_installs_the_auto_skip_on_the_third_image(- fake_runtime, tmp_path, monkeypatch-):- # Speed unset (the UI default): a dense load stays eager, then the 3rd image engages `default`, and with it the- # model's default-tier skip. The first two renders stay full-step.+def test_deferred_default_tier_never_auto_skips(fake_runtime, tmp_path, monkeypatch):from core.inference import diffusion as dmodfrom .test_diffusion_backend import DiffusionBackend@@ -383,9 +380,9 @@ def test_deferred_default_tier_installs_the_auto_skip_on_the_third_image(backend.generate(prompt = "three")st = backend.status()assert st["speed_mode"] == "default"- assert len(installs) == 1 and installs[0]["every"] == 2 and installs[0]["auto"] is True- assert st["transformer_cache"] == "static"- assert st["resolved"]["transformer_cache"]["value"] == "static"+ assert installs == []+ assert st["transformer_cache"] is None+ assert st["resolved"]["transformer_cache"]["value"] != "static"backend.unload()@@ -492,7 +489,7 @@ def test_a_failed_deferred_profile_takes_the_auto_skip_back_off(backend.generate(prompt = p)st = backend.status()assert st["speed_mode"] == "off"- assert st["transformer_cache"] is None and removed+ assert st["transformer_cache"] is None and removed == []assert st["resolved"]["transformer_cache"]["value"] != "static"backend.unload()@@ -537,7 +534,7 @@ def test_an_explicit_lossless_tier_never_auto_skips(requested, effective, want):tier = dcache.skip_tier(requested, effective)assert tier == wantplan = dcache.auto_static_skip_plan(("Qwen/Qwen-Image",), tier, 20, env = {})- assert (plan is None) == (want in ("off", "eager"))+ assert (plan is None) == (want != "max")@pytest.mark.parametrize("raw, want", [("1", 3), ("abc", 3), ("", 3), ("4", 4)])diff --git a/studio/backend/tests/test_video_static_skip.py b/studio/backend/tests/test_video_static_skip.pyindex 259b8d511ec..98d6bf7dedc 100644--- a/studio/backend/tests/test_video_static_skip.py+++ b/studio/backend/tests/test_video_static_skip.py@@ -213,33 +213,49 @@ def test_fbcache_load_still_reports_an_active_cache(loop_runtime, monkeypatch):backend.unload()[email protected]("request_cache", [None, "auto"])+def test_default_speed_never_auto_skips_on_wan_5b(loop_runtime, monkeypatch, request_cache):+ monkeypatch.delenv(dcache.ENV_AUTO_STEP_SKIP, raising = False)+ backend = VideoBackend()+ status = backend.load_pipeline(+ WAN_5B, model_kind = "pipeline", speed_mode = "default", transformer_cache = request_cache+ )+ assert status["transformer_cache"] is None+ assert "forward" not in loop_runtime["pipe"].transformer.__dict__+ backend.generate(prompt = "a sloth", steps = STEPS)+ assert loop_runtime["pipe"].transformer.computed == 2 * STEPS+ backend.unload()++@pytest.mark.parametrize("request_cache", [None, "auto"])def test_auto_installs_the_measured_static_skip_on_wan_5b(loop_runtime, monkeypatch, request_cache):monkeypatch.delenv(dcache.ENV_AUTO_STEP_SKIP, raising = False)seen = _record_speed(monkeypatch)backend = VideoBackend()status = backend.load_pipeline(- WAN_5B, model_kind = "pipeline", speed_mode = "default", transformer_cache = request_cache+ WAN_5B, model_kind = "pipeline", speed_mode = "max", transformer_cache = request_cache)assert status["transformer_cache"] == "static"entry = status["resolved"]["transformer_cache"]assert entry["value"] == "static" and entry["requested"] is Noneassert "UNSLOTH_DIFFUSION_AUTO_STEP_SKIP" in entry["reason"]layer = loop_runtime["pipe"].transformer.__dict__["forward"]- # Measured at 50 steps, so a shorter render computes every step.- assert (layer.every, layer.auto, layer.min_steps) == (2, True, 50)+ assert (layer.every, layer.auto, layer.min_steps) == (3, True, 50)# Kept graph decisions, and the generate-time FBCache toggle is never armed over it.assert [kw["cache_active"] for kw in seen] == [False]assert backend._state.cache_auto is Falsebackend.generate(prompt = "a sloth", steps = STEPS)- assert loop_runtime["pipe"].transformer.computed == 2 * (STEPS - SKIPPED_PER_BRANCH)+ skipped = len(ss.static_schedule(STEPS, every = 3, min_steps = 50)) - sum(+ ss.static_schedule(STEPS, every = 3, min_steps = 50)+ )+ assert loop_runtime["pipe"].transformer.computed == 2 * (STEPS - skipped)backend.unload()def test_auto_skip_leaves_a_keyframe_clip_unskipped(loop_runtime, monkeypatch):monkeypatch.delenv(dcache.ENV_AUTO_STEP_SKIP, raising = False)backend = VideoBackend()- backend.load_pipeline(WAN_5B, model_kind = "pipeline", speed_mode = "default")+ backend.load_pipeline(WAN_5B, model_kind = "pipeline", speed_mode = "max")real = VideoBackend._resolve_keyframesdef with_first_frame(*args, **kwargs):@@ -462,7 +478,11 @@ def test_hv15_counts_branches_by_context_without_a_step_callback(loop_runtime, mbackend.unload()-def _load_h3(backend, transformer_cache):+def _load_h3(+ backend,+ transformer_cache,+ speed_mode = None,+):from core.inference.video import _detect_load_familydiffusers = sys.modules["diffusers"]@@ -483,6 +503,7 @@ def _load_h3(backend, transformer_cache):_load_token = None,_base_local_dir = None,transformer_cache = transformer_cache,+ speed_mode = speed_mode,)@@ -584,6 +605,11 @@ def test_modular_workflow_auto_follows_the_table_and_the_kill_switch(h3_runtime,)backend = VideoBackend()status = _load_h3(backend, None)+ assert status["transformer_cache"] is None+ assert "forward" not in h3_runtime.instance.transformer.__dict__+ backend.unload()+ backend = VideoBackend()+ status = _load_h3(backend, None, speed_mode = "max")entry = status["resolved"]["transformer_cache"]assert status["transformer_cache"] == "static"assert entry["requested"] is None and entry["value"] == "static"@@ -597,7 +623,7 @@ def test_modular_workflow_auto_follows_the_table_and_the_kill_switch(h3_runtime,monkeypatch.setenv(dcache.ENV_AUTO_STEP_SKIP, "0")backend = VideoBackend()- status = _load_h3(backend, None)+ status = _load_h3(backend, None, speed_mode = "max")assert status["transformer_cache"] is Noneassert "forward" not in h3_runtime.instance.transformer.__dict__backend.unload()