Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: turn on the measured-safe optimizations by default (fused FLUX RoPE, CFG batching, Blackwell native attention, BF16 small DiTs) (#14566)

Daniel Han5a955eff1623
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 5ec8875f948..be22f177fa2 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -54,6 +54,8 @@ from .diffusion_content import (
)
from .diffusion_gguf_pipeline import load_whole_pipeline_gguf, whole_pipeline_gguf_resident_mib
from .diffusion_families import (
+ SMALL_DIT_BF16_WHEN_RESIDENT,
+ small_dit_rule_excludes,
named_variant_base,
DIFFUSION_CANCELLED_MSG,
DIFFUSION_NOT_LOADED_MSG,
@@ -2986,8 +2988,13 @@ def _plan_proves_resident(plan: Any) -> bool:
)
-def _auto_keeps_bf16_reason(fam: Any, kind: Optional[str] = "pipeline") -> Optional[str]:
- """Why AUTO keeps bf16 when it fits (measured rule: pipeline loads only), or None; both deciders read this."""
+def _auto_keeps_bf16_reason(
+ fam: Any,
+ kind: Optional[str] = "pipeline",
+ base: Optional[str] = None,
+) -> Optional[str]:
+ """Why AUTO keeps bf16 when it fits (measured rule: pipeline loads only), or None; both deciders read this.
+ ``base`` is the resolved base repo: the small-DiT rule skips a family's large variants (FLUX.2 klein 9B)."""
if not family_compiles_regionally(fam):
return (
f"'{getattr(fam, 'name', None)}' cannot be regionally compiled (its transformer declares no "
@@ -2999,6 +3006,8 @@ def _auto_keeps_bf16_reason(fam: Any, kind: Optional[str] = "pipeline") -> Optio
measured = auto_bf16_when_resident_reason(getattr(fam, "name", None))
if measured is None:
return None
+ if measured == SMALL_DIT_BF16_WHEN_RESIDENT and small_dit_rule_excludes(base):
+ return None
return (
f"'{getattr(fam, 'name', None)}': {measured}; the bf16 weights fit on this GPU as they are"
)
@@ -3009,11 +3018,12 @@ def _auto_quant_eager_reason(
plan: Any,
prequant_path: Optional[str] = None,
kind: Optional[str] = "pipeline",
+ base: Optional[str] = None,
) -> Optional[str]:
"""Why AUTO keeps bf16 for this load, or None; an operator's own checkpoint wins."""
if not _plan_proves_resident(plan):
return None
- reason = _auto_keeps_bf16_reason(fam, kind)
+ reason = _auto_keeps_bf16_reason(fam, kind, base)
if reason is None:
return None
if prequant_path and local_prequant_path_ready(prequant_path):
@@ -5309,7 +5319,7 @@ class DiffusionBackend:
is not None
):
return None
- if auto and _auto_keeps_bf16_reason(fam) is not None:
+ if auto and _auto_keeps_bf16_reason(fam, base = base or repo_id) is not None:
bf16_memory = snapshot_device_memory(target)
bf16_plan = self._bf16_table_plan(
target,
@@ -5326,7 +5336,9 @@ class DiffusionBackend:
# Offline without the released shards, bf16 cannot assemble: a cached seed is the only way.
if (
bf16_plan is not None
- and _auto_quant_eager_reason(fam, bf16_plan, transformer_prequant_path)
+ and _auto_quant_eager_reason(
+ fam, bf16_plan, transformer_prequant_path, base = base or repo_id
+ )
is not None
and not (
local_files_only
@@ -7596,6 +7608,7 @@ class DiffusionBackend:
# Why the dense quant did not engage, in the caller's terms. Threaded into `resolved` so a fallback is
# visible, and into the refusal so it is actionable.
transformer_quant_decline: Optional[str] = None
+ small_dit_kept_bf16 = False
transformer_quant_decline_status = RESOLVED_FELL_BACK
# A GGUF is excluded while it bakes LoRAs: only the dense build can carry them.
if (
@@ -7604,7 +7617,7 @@ class DiffusionBackend:
and dense_quant_supported_kind(kind)
and dense_transformer_supported(target)
and not (kind == "gguf" and _has_active_lora(loras))
- and _auto_keeps_bf16_reason(fam, kind) is not None
+ and _auto_keeps_bf16_reason(fam, kind, base) is not None
and (
eager_reason := _auto_quant_eager_reason(
fam,
@@ -7625,6 +7638,7 @@ class DiffusionBackend:
),
transformer_prequant_path,
kind,
+ base,
)
)
is not None
@@ -7636,6 +7650,10 @@ class DiffusionBackend:
transformer_quant_decline = eager_reason
# Place the released weights by the plan that proved they fit, not by fp32 shard bytes.
plan = eager_plan
+ small_dit_kept_bf16 = (
+ auto_bf16_when_resident_reason(getattr(fam, "name", None))
+ == SMALL_DIT_BF16_WHEN_RESIDENT
+ )
native_scheme = (
native_quant_scheme(
target, transformer_quant_pinned, family = getattr(fam, "name", None)
@@ -9236,6 +9254,19 @@ class DiffusionBackend:
"(quantized transformer must be compiled; eager is ~30x slower)"
)
effective_speed = SPEED_DEFAULT
+ if (
+ small_dit_kept_bf16
+ and speed_mode is None
+ and effective_speed == SPEED_OFF
+ and not comfy_compile
+ and compile_eligible(target, is_gguf = False, family = fam, probe = False)
+ and not fp16_compile_explicit_only(target)
+ ):
+ logger.info(
+ "diffusion.speed: speed_mode=default from render 1 (a small denoiser kept bf16 in "
+ "place of the INT8 default, which compiled from render 1 too)"
+ )
+ effective_speed = SPEED_DEFAULT
# Deferred speed auto for dense: stay eager and engage `default` on the 3rd image, where compile
# amortises. Only when speed was unset.
speed_deferred = (
@@ -9963,8 +9994,9 @@ class DiffusionBackend:
)
# After every forward wrapper of the speed / graph layers, so the replica sees the final module.
install_cfg_parallel(pipe, offload_policy = effective_policy, logger = logger)
- # both branches through one offloaded forward; declines when CFG-parallel took the slot
- install_cfg_batch(pipe, offload_policy = effective_policy, logger = logger)
+ install_cfg_batch(
+ pipe, offload_policy = effective_policy, speed_mode = speed_mode, logger = logger
+ )
# Z-Image guided steps at the guidance-off batch: turning CFG on compiles and records nothing new
install_zimage_cfg_split(pipe, offload_policy = effective_policy, logger = logger)
te_releaser = te_release.maybe_install(pipe, plan, logger = logger)
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index 696421a4562..2dae44b7c3a 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -15,8 +15,10 @@ Attention is bandwidth-bound, so a better kernel is a real win orthogonal to wei
bitwise-identical and near free (compiled 1.599s -> 1.572s, 1.02x; eager 0.93x, the
per-call sdpa_kernel wrapper cost, which compile folds away) -- but FLASH and EFFICIENT
at that same shape run 3.9x and 9.0x slower. Pinning is insurance against the heuristic
- picking one of those on another card, head_dim or torch build. Elsewhere stays
- ``native``. Only upgrades when a speed profile is active, so ``speed_mode=off`` stays
+ picking one of those on another card, head_dim or torch build. On Blackwell (SM100+) auto
+ leaves native when torch's own dispatch picks cuDNN at every DiT head dim and the run dtype
+ (same kernel, no per-call wrapper; ``UNSLOTH_DIFFUSION_BLACKWELL_NATIVE_ATTN=0`` keeps the
+ pin). Elsewhere stays ``native``. Only upgrades when a speed profile is active, so ``speed_mode=off`` stays
bit-identical.
native - force the default SDPA (bit-identical reference).
cudnn - cuDNN fused attention (exact; NVIDIA).
@@ -473,7 +475,7 @@ def select_attention_backend(
return None
return backend
if speed_active and _is_cuda_nvidia(target) and _cudnn_attention_supported():
- return "_native_cudnn"
+ return AUTO_CUDNN
if (speed_active or speed_unset) and _rocm_auto_flash(target, family):
return ROCM_FLASH_CANDIDATE
return None
@@ -506,6 +508,22 @@ def _is_cuda_rocm(target: Any) -> bool:
ROCM_FLASH_CANDIDATE = "_rocm_flash_candidate"
+
+
+class _AutoCudnn(str):
+ """The ``auto`` cuDNN pick. Equal to ``"_native_cudnn"`` for every caller; ``apply_attention_backend`` alone tells
+ it from an explicit ``cudnn`` request, so it can leave native where torch already dispatches cuDNN (Blackwell)."""
+
+ __slots__ = ()
+
+
+AUTO_CUDNN = _AutoCudnn("_native_cudnn")
+# B200 torch 2.13 bf16: native SDPA already runs cuDNN at head_dim 64/128 (pin = 0.4-22% slower eager wrapper);
+# head_dim 256 picks FLASH, hence the per-head-dim probe.
+BLACKWELL_NATIVE_ATTN_ENV = (
+ "UNSLOTH_DIFFUSION_BLACKWELL_NATIVE_ATTN" # auto (default) | 0 keeps the cuDNN pin
+)
+_NATIVE_DISPATCH_CACHE: dict[tuple[str, str, int], bool] = {}
ROCM_FLASH_ENV = "UNSLOTH_DIFFUSION_ROCM_FLASH" # auto (timed) | 0 (auto never picks flash) | 1 (verified is enough)
ROCM_FLASH_MIN_GAIN = 0.03
# Accuracy is judged on the raw attention output (flash vs native SDPA, same inputs), not the block: the residual and
@@ -3054,6 +3072,60 @@ def _cudnn_runs_head_dim(target: Any, head_dim: int) -> Optional[bool]:
return _CUDNN_HEAD_DIM_CACHE.setdefault(key, bool(ran))
+def _native_sdp_choice(device: str, dtype: Any, head_dim: int) -> int:
+ """torch's own SDPA dispatch choice (an ``SDPBackend`` value) at ``head_dim`` / ``dtype`` on ``device``."""
+ import torch
+
+ q = torch.zeros(1, 2, 256, int(head_dim), device = device, dtype = dtype)
+ return int(torch._fused_sdp_choice(q, q, q))
+
+
+def _device_capability(device: str) -> tuple[int, int]:
+ import torch
+ return tuple(torch.cuda.get_device_capability(device)) # type: ignore[return-value]
+
+
+def _native_dispatches_cudnn(pipe: Any, target: Any) -> bool:
+ """True on a Blackwell (SM100+) NVIDIA card when torch's own SDPA dispatch picks cuDNN for every DiT head dim at the
+ run dtype, so the ``auto`` pin would change no kernel. Unknown head dims, dtype or capability keep the pin."""
+ if (os.environ.get(BLACKWELL_NATIVE_ATTN_ENV) or "").strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ ):
+ return False
+ if not _is_cuda_nvidia(target):
+ return False
+ device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
+ if not device.startswith("cuda"):
+ return False
+ dims = _dit_head_dims(pipe)
+ if not dims:
+ return False
+ try:
+ import torch
+ from torch.nn.attention import SDPBackend
+
+ device = _indexed_cuda_device(device)
+ if _device_capability(device) < (10, 0):
+ return False
+ dtype = getattr(target, "dtype", None)
+ if dtype not in (torch.bfloat16, torch.float16):
+ return False
+ for dim in sorted(dims):
+ key = (device, str(dtype), int(dim))
+ hit = _NATIVE_DISPATCH_CACHE.get(key)
+ if hit is None:
+ hit = _native_sdp_choice(device, dtype, dim) == int(SDPBackend.CUDNN_ATTENTION)
+ _NATIVE_DISPATCH_CACHE[key] = hit
+ if not hit:
+ return False
+ return True
+ except Exception: # noqa: BLE001 - unaskable: keep the pin
+ return False
+
+
def _cudnn_serves_pipe(
pipe: Any,
target: Any,
@@ -3182,6 +3254,20 @@ def apply_attention_backend(
and not _cudnn_serves_pipe(pipe, target, logger)
):
backend = None
+ if (
+ isinstance(backend, _AutoCudnn)
+ and target is not None
+ and _native_dispatches_cudnn(pipe, target)
+ ):
+ if logger is not None:
+ logger.info(
+ "diffusion.attention: auto keeps native SDPA (it already dispatches cuDNN on this Blackwell card; "
+ "%s=0 pins cuDNN)",
+ BLACKWELL_NATIVE_ATTN_ENV,
+ )
+ backend = None
+ if backend is not None:
+ backend = str(backend)
if backend == "flash_4_hub" and not _install_fa4_dispatch_guard():
if logger is not None:
logger.warning(
diff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.py
index 24c90b12229..2529be91d2c 100644
--- a/studio/backend/core/inference/diffusion_cache.py
+++ b/studio/backend/core/inference/diffusion_cache.py
@@ -63,6 +63,8 @@ def _auto_static_skip_table() -> dict:
AUTO_STATIC_SKIP: dict = _auto_static_skip_table()
+AUTO_STATIC_SKIP_TIERS = (_TIER_MAX,)
+
# Load gate on the model's default steps (keeps distilled siblings out); explicit "static" keeps STATIC_MIN_STEPS (12).
AUTO_STATIC_MIN_STEPS = 20
@@ -106,15 +108,17 @@ def auto_static_skip_plan(
"""Settings for an auto static skip (``{"every": n, "min_steps": m}``), or None when auto must not pick it.
``speed_mode`` is the tier the user ASKED for (an eager downgrade forced by offload does not change what the
- skip costs in quality); off / eager are the lossless tiers and never skip."""
+ skip costs in quality). Only ``max`` skips automatically: the skip is lossy (mean LPIPS up to 0.05 on the
+ ``default`` interval, 0.10 on ``max``, PR #12652) and ComfyUI never skips steps by default, so ``default``, an
+ unset speed and the deferred profile render every step. An explicit ``static`` request is still honoured."""
if auto_step_skip_disabled(env):
return None
if isinstance(identifiers, str) or identifiers is None:
identifiers = (identifiers,)
entry = auto_static_skip_entry(*identifiers)
- if not entry or speed_mode not in (_TIER_DEFAULT, _TIER_MAX):
+ if not entry or speed_mode not in AUTO_STATIC_SKIP_TIERS:
return None
- every = entry.get(_TIER_DEFAULT) if speed_mode == _TIER_DEFAULT else entry.get(_TIER_MAX)
+ every = entry.get(_TIER_MAX)
try:
steps_ok = default_steps is not None and int(default_steps) >= AUTO_STATIC_MIN_STEPS
except (TypeError, ValueError):
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.py
index 186acedefed..90a8b1d27d1 100644
--- a/studio/backend/core/inference/diffusion_cfg_batch.py
+++ b/studio/backend/core/inference/diffusion_cfg_batch.py
@@ -1,12 +1,13 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""CFG batching: both guidance branches of a step through ONE forward of an offloaded denoiser.
+"""CFG batching: both guidance branches of a step through ONE forward of the denoiser.
-A denoiser streamed from host memory (group / sequential offload) uploads its weights once per call, and a guided step
-calls it twice (conditional, then unconditional), so it pays the upload twice. ComfyUI batches the two branches into
-one call (``comfy/samplers.py`` ``calc_cond_batch``); diffusers pipelines call the transformer once per branch. This
-router batches them for the classes whose forward keeps no state between calls:
+A guided step calls the denoiser twice (conditional, then unconditional). ComfyUI batches the two branches into one
+call by default whenever memory allows (``comfy/samplers.py`` ``calc_cond_batch``), resident or not; diffusers pipelines
+call the transformer once per branch. A denoiser streamed from host memory pays its weight upload once instead of
+twice; a resident one gains where one doubled call keeps the GPU busier than two (small images). This router batches
+them for the classes whose forward keeps no state between calls:
- the first branch of a step builds the second branch's arguments from the previous step's second call (same latent
object, the first branch's per-step tensors, the second branch's constants), runs both as one call of twice the
@@ -14,20 +15,28 @@ router batches them for the classes whose forward keeps no state between calls:
- the second call is verified against what was batched (every argument bitwise) and gets the kept half; anything
else (a changed argument, no second call, a third call) reruns that branch plain and counts as a miss.
-Priced by measurement, per request shape: a plain pair is timed on the second step of a shape (the first warms up),
-the batched pair (concatenate, forward, split, verify) on the next two, and batching stays on only when the faster
-batched pair beats the plain pair by ``_MEASURED_FRACTION``. Memory: the doubled call is attempted only when the card
-has room for twice the activations a plain call allocated; an out-of-memory reruns plain and turns the shape off.
-
-Not bit-exact: a doubled batch can pick other GEMM / attention tiles (FLUX.2-klein-base, offloaded: LPIPS ~0.01 vs two
-calls). ``UNSLOTH_DIFFUSION_CFG_BATCH=0`` keeps the exact one-branch-per-call path; ``force`` also batches a resident
-denoiser and an unlisted class. Declined for per-call activation scaling (NVFP4 without frozen global scales), whose
-scale would change with the doubled batch, and for denoisers already wrapped by another forward layer.
+Priced once per request shape, before any batched result is used: steps 1-2 run plain (step 2 is timed); a resident
+pair >= ``_RESIDENT_MAX_PAIR_S`` stays plain untried. Otherwise a discarded trial (plain pairs interleaved with doubled
+calls) batches steps 3+ only if the fastest batched step (predict, merge, snapshot, doubled call, split, verify) beats the
+fastest plain pair by ``_MEASURED_FRACTION``, so every render of a shape takes one path (same seed, same image). The
+doubled call needs room for twice a plain call's activations; an OOM turns the shape plain for that render.
+
+Not always bit-exact (other GEMM / attention tiles): B200 bf16 resident FLUX.2-klein-base-4B is bitwise equal,
+Wan2.1-T2V-1.3B LPIPS 0.019 mean; offloaded FLUX.2-klein-base LPIPS ~0.01. ``UNSLOTH_DIFFUSION_CFG_BATCH=0`` keeps the exact
+one-branch-per-call path, as does a request with ``speed_mode="off"``; ``force`` batches anyway, an unlisted class too
+(the trial's discarded calls must be side-effect free). Declined for per-call activation scaling (NVFP4 without frozen
+global scales), whose scale would change with the doubled batch, and for denoisers already wrapped by another forward
+layer.
+
+Compiled denoisers: the doubled shape compiles a second graph per block, allowed only in a shape's trial render.
+Offloaded compiled denoisers batch; resident compiled ones and shapes judged before a later compile stay plain. A
+follow-up-render compile ends batching for the load; under Dynamo tracing the router passes straight through.
"""
from __future__ import annotations
import os
+import sys
import time
import weakref
from typing import Any, Optional
@@ -52,12 +61,15 @@ _PADDED_TEXT = (("encoder_hidden_states", "encoder_hidden_states_mask"),)
_MEASURED_FRACTION = 0.92
_BATCHED_SAMPLES = 2
+_PLAIN_STEPS = 2
+# Compute-bound resident pair: skip the trial (B200 winners 0.03-0.06 s/pair; T4 FLUX.2-klein ~4.2 s, trial +17 s).
+_RESIDENT_MAX_PAIR_S = 0.25
_MISS_LIMIT = 3
_OFF_TRANSIENT = "transient"
def cfg_batch_mode() -> str:
- """``off`` / ``auto`` (default: an offloaded denoiser of a listed class) / ``force``. Independent of the multi-GPU
+ """``off`` / ``auto`` (default: a listed class, resident or offloaded) / ``force``. Independent of the multi-GPU
switches: a single card is the usual case."""
raw = (os.environ.get(CFG_BATCH_ENV) or "").strip().lower()
if raw in ("0", "off", "false", "no", "disable", "disabled"):
@@ -102,6 +114,56 @@ def _foreign_hook(module: Any) -> Optional[str]:
return None
+# Set on a denoiser whose blocks Studio compiled (regional compile; also read when the compile moves below an
+# offload hook, which clears the block's ``_compiled_call_impl``).
+_COMPILE_MARKS = ("_unsloth_regional_compile_kwargs", "_unsloth_compile_guard")
+
+
+def _compiled_callable(fn: Any) -> bool:
+ for _ in range(4):
+ if fn is None:
+ return False
+ if hasattr(fn, "_torchdynamo_orig_callable") or hasattr(fn, "_unsloth_compile_guard"):
+ return True
+ fn = getattr(fn, "__wrapped__", None)
+ return False
+
+
+def compiled_denoiser(module: Any) -> Optional[str]:
+ """Why ``module`` runs compiled code, else None. A doubled batch there is a new input shape: the batch 1 -> 2 change
+ compiles a second graph per compiled block, which is only acceptable inside the warm-up render of a shape."""
+ try:
+ subs = list(module.named_modules())
+ except Exception: # noqa: BLE001 - not a torch module (fakes): nothing compiled
+ return None
+ for mark in _COMPILE_MARKS:
+ if mark in getattr(module, "__dict__", {}):
+ return "the denoiser is compiled"
+ for name, sub in subs:
+ if getattr(sub, "_compiled_call_impl", None) is not None:
+ return f"{name or 'the denoiser'} is compiled"
+ registry = getattr(sub, "__dict__", {}).get("_diffusers_hook")
+ for ref in list(getattr(registry, "_fn_refs", None) or ()):
+ if _compiled_callable(getattr(ref, "forward", None)):
+ return f"{name or 'the denoiser'} runs a compiled forward below its hooks"
+ return None
+
+
+def _offloaded(denoiser: Any, offload_policy: str) -> bool:
+ return offload_policy in ("group", "streaming", "sequential") and _streamed(denoiser)
+
+
+def _graph_count() -> int:
+ """Dynamo's compiled-graph count (0 before Dynamo was ever imported)."""
+ dynamo_utils = sys.modules.get("torch._dynamo.utils")
+ try:
+ return (
+ int(dynamo_utils.counters["stats"]["unique_graphs"]) if dynamo_utils is not None else 0
+ )
+ except Exception: # noqa: BLE001
+ return 0
+
+
def _foreign_layer(module: Any) -> Optional[str]:
"""A non-offload diffusers hook (a step cache) or a root forward override that is not offload machinery."""
mg = _mg()
@@ -132,11 +194,24 @@ def per_call_scaled(module: Any) -> Optional[str]:
return None
-def cfg_batch_decline(pipe: Any, *, offload_policy: str) -> Optional[str]:
- """Why both guidance branches cannot share one forward on this load, else None."""
+def _speed_off_requested(speed_mode: Optional[str]) -> bool:
+ """The request asked for ``speed_mode="off"`` (unset is not a request): the bit-identical reference path."""
+ return speed_mode is not None and str(speed_mode).strip().lower() == "off"
+
+
+def cfg_batch_decline(
+ pipe: Any,
+ *,
+ offload_policy: str,
+ speed_mode: Optional[str] = None,
+) -> Optional[str]:
+ """Why both guidance branches cannot share one forward on this load, else None. ``speed_mode`` is the request's own
+ value: an explicit ``off`` keeps one branch per call (a doubled batch can pick other tiles); ``force`` overrides."""
mode = cfg_batch_mode()
if mode == "off":
return f"{CFG_BATCH_ENV}=0"
+ if mode != "force" and _speed_off_requested(speed_mode):
+ return f"speed_mode=off keeps one branch per call ({CFG_BATCH_ENV}=force batches anyway)"
torch = _torch()
denoiser = getattr(pipe, "transformer", None)
if not isinstance(denoiser, torch.nn.Module):
@@ -145,6 +220,9 @@ def cfg_batch_decline(pipe: Any, *, offload_policy: str) -> Optional[str]:
return "two-expert denoiser"
if getattr(denoiser, "_compiled_call_impl", None) is not None:
return "denoiser is compiled whole"
+ compiled = compiled_denoiser(denoiser)
+ if compiled is not None and not _offloaded(denoiser, offload_policy):
+ return f"{compiled} and resident"
if getattr(pipe, "_unsloth_cuda_graphs", None):
return "the denoiser replays CUDA graphs"
foreign = _foreign_layer(denoiser)
@@ -159,8 +237,9 @@ def cfg_batch_decline(pipe: Any, *, offload_policy: str) -> Optional[str]:
"diffusers."
):
return f"{type(denoiser).__name__} is not on the stateless-forward list ({CFG_BATCH_ENV}=force opts it in)"
- if offload_policy not in ("group", "streaming", "sequential") or not _streamed(denoiser):
- return "denoiser is resident (two calls upload nothing)"
+ if offload_policy == "model":
+ # diffusers rebuilds model-offload hooks between renders, evicting the router: later renders would differ.
+ return "model offload re-hooks the denoiser every render"
return None
@@ -186,10 +265,14 @@ class _Shape:
self.off: Optional[str] = None
self.off_kind: Optional[str] = None
self.misses = 0
+ self.judged = False
+ self.compiled_at_trial = False
+ self.trial_render = 0
+ self.render_steps = 0 # steps of this shape in the current render
class CfgBatchRouter:
- """Offloaded denoiser ``forward`` running both guidance branches of a step through one forward."""
+ """Denoiser ``forward`` running both guidance branches of a step through one forward."""
def __init__(
self,
@@ -199,6 +282,10 @@ class CfgBatchRouter:
logger: Any = None,
) -> None:
self.module_ref = weakref.ref(module)
+ # Under Dynamo tracing: pass straight through (no graph breaks, no doubled-shape recompile).
+ self._is_compiling = getattr(getattr(_torch(), "compiler", None), "is_compiling", None) or (
+ lambda: False
+ )
self.orig_forward = module.forward
self.had_override = "forward" in module.__dict__
self.prev_override = module.__dict__.get("forward")
@@ -214,14 +301,17 @@ class CfgBatchRouter:
self.last_shape: Optional[_Shape] = None
self.render_off: Optional[str] = None
self.render_checked = False
+ self.offloaded = False
+ self.pending_disable: Optional[str] = None
+ self.pair_t0: Optional[float] = None
+ self.renders = 0 # completed renders: a doubled-batch compile is warm-up only in a shape's trial render
+ self.render_compiled: Optional[str] = None
def _reset_step(self) -> None:
self.branch = 0
self.b0: Optional[tuple] = None
self.kept: Optional[dict] = None
self.shape: Optional[_Shape] = None
- self.pending_t0: Optional[float] = None
- self.pending_plain: Optional[float] = None
def describe(self) -> dict[str, Any]:
shape = self.shape or self.last_shape
@@ -236,11 +326,17 @@ class CfgBatchRouter:
if shape is None or not shape.batched_s
else round(min(shape.batched_s), 4),
"displaced": self._displaced(),
+ "render_off": self.render_off,
**self.stats,
}
def end_render(self) -> None:
"""Drop every tensor held between calls (the kept half, the template's constants, the latent)."""
+ if self.render_checked:
+ self.renders += 1
+ if self.pending_disable is not None:
+ self.disable(self.pending_disable)
+ self.pair_t0 = None
self.kept = None
self.template = None
self.prev_latent = None
@@ -249,10 +345,12 @@ class CfgBatchRouter:
self.render_checked = False
self._reset_step()
for shape in self.shapes.values():
+ shape.render_steps = 0
if (
shape.off_kind == _OFF_TRANSIENT
- ): # memory pressure is per render: retry on the next one
+ ): # memory pressure is per render: retry (and re-time) on the next one
shape.off = shape.off_kind = None
+ shape.judged = False
def disable(self, reason: str) -> None:
if self.disabled is None:
@@ -279,6 +377,8 @@ class CfgBatchRouter:
return module is None or module.__dict__.get("forward") is not self
def __call__(self, *args: Any, **kwargs: Any) -> Any:
+ if self._is_compiling():
+ return self.orig_forward(*args, **kwargs)
mg = _mg()
latent = mg._latent_arg(args, kwargs)
if self.disabled is not None or latent is None or self._displaced():
@@ -287,6 +387,13 @@ class CfgBatchRouter:
# first call of a render: a request-time step cache may have hooked the blocks since install
module = self.module_ref()
self.render_off = _foreign_hook(module) if module is not None else None
+ self.render_compiled = compiled_denoiser(module) if module is not None else None
+ if self.render_compiled is not None:
+ if not self.offloaded:
+ self.render_off = f"{self.render_compiled} and resident"
+ for shape in self.shapes.values():
+ if shape.judged and not shape.compiled_at_trial and shape.off is None:
+ self._shape_off(shape, "the denoiser was compiled after this shape's trial")
self.render_checked = True
if self.render_off is not None:
return self._plain(args, kwargs)
@@ -296,22 +403,44 @@ class CfgBatchRouter:
self._new_step(args, kwargs, latent)
shape = self.shape
if self.branch == 0:
- if self.template is not None and shape.off is None and shape.plain_pair_s is not None:
+ if (
+ self.template is not None
+ and shape.off is None
+ and shape.judged
+ and shape.render_steps > _PLAIN_STEPS
+ ):
out = self._batched(args, kwargs, latent, shape)
if out is not None:
return out
- if shape.off is None and shape.plain_pair_s is None and shape.steps >= 2:
- return self._timed_first(args, kwargs, latent, shape)
+ if shape.off is None and not shape.judged and shape.render_steps == _PLAIN_STEPS:
+ self.pair_t0 = time.perf_counter() if self._sync(latent) else None
+ if shape.off is None and not shape.judged and shape.activation_mib is None:
+ return self._measured_plain(args, kwargs, latent, shape)
return self._plain(args, kwargs)
if self.branch == 1:
if self.kept is not None:
out = self._take(args, kwargs, shape)
if out is not None:
return out
+ if shape.off is not None:
+ # skip recording on a plain shape: it is a host sync per step (2.5-3.5% on B200)
+ return self._plain(args, kwargs)
self._record_template(args, kwargs, latent)
- if self.pending_t0 is not None:
- return self._timed_second(args, kwargs, latent, shape)
- return self._plain(args, kwargs)
+ out = self._plain(args, kwargs)
+ if not shape.judged and self.disabled is None and shape.render_steps == _PLAIN_STEPS:
+ t0, self.pair_t0 = self.pair_t0, None
+ if t0 is not None and not self.offloaded and self._sync(latent):
+ live = time.perf_counter() - t0
+ if live >= _RESIDENT_MAX_PAIR_S:
+ self._shape_off(
+ shape,
+ f"compute-bound: a plain pair takes {live:.3f}s, where one doubled call cannot win "
+ f"{round((1 - _MEASURED_FRACTION) * 100)}%",
+ )
+ shape.plain_pair_s = live
+ return out
+ self._trial((args, kwargs), latent, shape)
+ return out
self.disable("more than two denoiser calls per step")
return self._plain(args, kwargs)
@@ -335,6 +464,7 @@ class CfgBatchRouter:
None # a template predicts its own shape's second call: one plain pair first
)
shape.steps += 1
+ shape.render_steps += 1
self.shape = self.last_shape = shape
def _plain(self, args: tuple, kwargs: dict) -> Any:
@@ -347,41 +477,84 @@ class CfgBatchRouter:
_torch().cuda.synchronize(latent.device)
return True
- def _timed_first(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
+ def _measured_plain(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
+ """A plain first-branch call that records the activations it allocated (the doubled call's memory gate)."""
torch = _torch()
- if not self._sync(latent):
- shape.plain_pair_s = 0.0 # no device timer: batch on the memory gate alone
+ if getattr(latent.device, "type", None) != "cuda":
return self._plain(args, kwargs)
# never reset the peak counters (other measurements own them); a call that stayed under an
# earlier peak leaves its activations unknown
base = torch.cuda.memory_allocated(latent.device)
before = torch.cuda.max_memory_allocated(latent.device)
- t0 = time.perf_counter()
out = self._plain(args, kwargs)
- self._sync(latent)
- self.pending_t0 = time.perf_counter() - t0
peak = torch.cuda.max_memory_allocated(latent.device)
- shape.activation_mib = (int(peak - base) >> 20) if peak > before else None
+ if peak > before:
+ shape.activation_mib = int(peak - base) >> 20
return out
- def _timed_second(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
- self._sync(latent)
- t0 = time.perf_counter()
- out = self._plain(args, kwargs)
- self._sync(latent)
- shape.plain_pair_s = self.pending_t0 + (time.perf_counter() - t0)
- self.pending_t0 = None
- return out
-
- def _judge(self, shape: _Shape) -> None:
- if len(shape.batched_s) < _BATCHED_SAMPLES or shape.plain_pair_s in (None, 0.0):
+ def _trial(self, second: tuple, latent: Any, shape: _Shape) -> None:
+ """Price this shape once on the step's warm arguments; outputs are discarded so renders keep one path."""
+ torch = _torch()
+ first = self.b0
+ if first is None:
+ return
+ if not self._sync(latent):
+ self._judge(shape)
+ return
+ if not self._memory_ok(latent, shape):
+ self._shape_off(shape, "no room for a doubled call", _OFF_TRANSIENT)
+ return
+ batch = int(latent.shape[0])
+ plain_s: list[float] = []
+ try:
+ graphs = _graph_count()
+ for _ in range(_BATCHED_SAMPLES):
+ t0 = time.perf_counter()
+ self.orig_forward(*first[0], **first[1])
+ self.orig_forward(*second[0], **second[1])
+ self._sync(latent)
+ plain_s.append(time.perf_counter() - t0)
+ t0 = time.perf_counter()
+ predicted = self._predict(latent) if self.template is not None else second
+ b_args, b_kwargs = merge_branches(first, predicted, batch)
+ snapshots = _snapshot(predicted)
+ split_output(self.orig_forward(*b_args, **b_kwargs), batch)
+ _matches(snapshots, second)
+ self._sync(latent)
+ shape.batched_s.append(time.perf_counter() - t0)
+ del b_args, b_kwargs, snapshots, predicted
+ shape.plain_pair_s = min(plain_s)
+ if _graph_count() != graphs and self.render_compiled is None and not self.offloaded:
+ self.disable(
+ "the doubled batch compiled a new graph (the denoiser runs compiled code)"
+ )
+ return
+ except _NotBatchable as exc:
+ self._shape_off(shape, f"not batchable: {exc}")
+ return
+ except Exception as exc: # noqa: BLE001 - the shape stays plain; the render's own calls already ran
+ if _is_oom(exc):
+ try:
+ torch.cuda.empty_cache()
+ except Exception: # noqa: BLE001
+ pass
+ self._shape_off(shape, "out of memory on the timed doubled call", _OFF_TRANSIENT)
+ return
+ self.disable(f"the doubled call failed ({type(exc).__name__}: {exc})")
return
batched = min(shape.batched_s)
- if batched > _MEASURED_FRACTION * shape.plain_pair_s:
+ if shape.plain_pair_s and batched > _MEASURED_FRACTION * shape.plain_pair_s:
self._shape_off(
shape,
f"measured no gain: batched pair {batched:.4f}s vs plain pair {shape.plain_pair_s:.4f}s",
)
+ return
+ self._judge(shape)
+
+ def _judge(self, shape: _Shape) -> None:
+ shape.judged = True
+ shape.compiled_at_trial = self.render_compiled is not None
+ shape.trial_render = self.renders
def _record_template(self, args: tuple, kwargs: dict, latent: Any) -> None:
"""How the second call's arguments relate to the first's: the latent, a first-branch tensor (same object, or a
@@ -436,12 +609,8 @@ class CfgBatchRouter:
def _batched(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
torch = _torch()
- timing = len(shape.batched_s) < _BATCHED_SAMPLES and shape.plain_pair_s not in (None, 0.0)
if not self._memory_ok(latent, shape):
return None
- if timing:
- self._sync(latent)
- t0 = time.perf_counter()
try:
second = self._predict(latent)
b_args, b_kwargs = merge_branches((args, kwargs), second, int(latent.shape[0]))
@@ -458,6 +627,7 @@ class CfgBatchRouter:
pass
self._shape_off(shape, "out of memory preparing the doubled call", _OFF_TRANSIENT)
return None
+ graphs = _graph_count()
try:
out = self.orig_forward(*b_args, **b_kwargs)
mine, other = split_output(out, int(latent.shape[0]))
@@ -475,10 +645,11 @@ class CfgBatchRouter:
return None
self.disable(f"the doubled call failed ({type(exc).__name__}: {exc})")
return None
+ if _graph_count() != graphs and self.renders > shape.trial_render:
+ self.pending_disable = "the doubled batch compiled a graph after its warm-up render"
self.kept = {
"out": other,
"snapshots": snapshots,
- "t0": t0 if timing else None,
"latent": latent,
}
self.stats["batched_steps"] += 1
@@ -489,10 +660,6 @@ class CfgBatchRouter:
if not _matches(kept["snapshots"], (args, kwargs)):
self._miss(shape, "the second branch differed from what was batched")
return None
- if kept["t0"] is not None:
- self._sync(kept["latent"])
- shape.batched_s.append(time.perf_counter() - kept["t0"])
- self._judge(shape)
self.stats["hits"] += 1
shape.misses = 0
return kept["out"]
@@ -662,12 +829,13 @@ def install_cfg_batch(
pipe: Any,
*,
offload_policy: str,
+ speed_mode: Optional[str] = None,
logger: Any = None,
) -> Optional[str]:
"""Arm :class:`CfgBatchRouter` on ``pipe.transformer``; the decline reason, or None when armed. An optional lever:
- it never fails a load."""
+ it never fails a load. ``speed_mode`` is the request's value (None = unset)."""
try:
- return _install(pipe, offload_policy = offload_policy, logger = logger)
+ return _install(pipe, offload_policy = offload_policy, speed_mode = speed_mode, logger = logger)
except Exception as exc: # noqa: BLE001
if logger is not None:
logger.info("diffusion.cfg_batch: not armed (%s: %s)", type(exc).__name__, exc)
@@ -678,9 +846,10 @@ def _install(
pipe: Any,
*,
offload_policy: str,
+ speed_mode: Optional[str] = None,
logger: Any = None,
) -> Optional[str]:
- reason = cfg_batch_decline(pipe, offload_policy = offload_policy)
+ reason = cfg_batch_decline(pipe, offload_policy = offload_policy, speed_mode = speed_mode)
if reason is not None:
if logger is not None:
logger.info("diffusion.cfg_batch: not armed (%s)", reason)
@@ -690,9 +859,14 @@ def _install(
denoiser,
reason = "forced"
if cfg_batch_mode() == "force"
- else f"offloaded denoiser ({offload_policy})",
+ else (
+ f"offloaded denoiser ({offload_policy})"
+ if _offloaded(denoiser, offload_policy)
+ else "resident denoiser"
+ ),
logger = logger,
)
+ router.offloaded = _offloaded(denoiser, offload_policy)
try:
_ROUTERS[pipe] = router
except TypeError:
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index dcc2d2ecd76..66e7a6ceafc 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -323,6 +323,26 @@ _LUMINA2_BF16_WHEN_RESIDENT = (
_HIDREAM_BF16_WHEN_RESIDENT = (
"INT8 is barely faster than compiled bf16 on a card that holds bf16 and changes image detail"
)
+# Small DiTs (<= ~6B) stay bf16 while it fits: quantising moves detail (L4 vs bf16 LPIPS: FP8 FLUX.2-klein-4B 0.065,
+# Z-Image-Turbo 0.103; FLUX.3-klein INT8 0.027). INT8 still when bf16 would offload; such loads compile from render 1.
+SMALL_DIT_BF16_WHEN_RESIDENT = (
+ "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, "
+ "0.103 on Z-Image-Turbo)"
+)
+# Bases of a small-DiT family that are not small: FLUX.2 klein 9B / base 9B (18.2 GB bf16 transformer, unmeasured)
+# keep the quantized auto default. Upstream ids; mirrors fold back through ``canonical_base``.
+_SMALL_DIT_LARGE_BASES = frozenset(
+ {
+ "black-forest-labs/flux.2-klein-9b",
+ "black-forest-labs/flux.2-klein-base-9b",
+ }
+)
+
+
+def small_dit_rule_excludes(base_repo: Optional[str]) -> bool:
+ """True when ``base_repo`` (vendor id or mirror) is a large variant the small-DiT bf16 rule must not cover."""
+ return canonical_base(base_repo).lower() in _SMALL_DIT_LARGE_BASES
+
# FLUX.1's sd-cli encoders are BFL's own diffusers files, read as-is: sd.cpp loads the CLIP-L text_encoder and the
# sharded T5 through its index (the shards sit beside it in the snapshot). The shards are listed so every fetch,
@@ -438,6 +458,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
comfy_shift_node = "flux",
label = "FLUX.2 Klein",
profile = FamilyProfile(
+ auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,
preview_factors = "flux2",
auto_static_skip = (
(
@@ -523,6 +544,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
comfy_shift_node = "flux",
label = "FLUX.3 Klein Base",
profile = FamilyProfile(
+ auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,
# Left dense for EVERY transformer-quant scheme: per-token modulations, timestep embedders, norm_out, the
# text-stream per-mode blocks and the 20480-wide text projection. Measured on B200 against bf16 (11-case
# recipe set, LPIPS mean): int8 0.097 -> 0.027, fp8 0.063 -> 0.033, mxfp8 0.024 with it; without it mxfp8
@@ -578,6 +600,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
comfy_shift_node = "flux",
label = "FLUX.3 Klein",
profile = FamilyProfile(
+ auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,
# Left dense for EVERY transformer-quant scheme: per-token modulations, timestep embedders, norm_out, the
# text-stream per-mode blocks and the 20480-wide text projection. Measured on B200 against bf16 (11-case
# recipe set, LPIPS mean): int8 0.097 -> 0.027, fp8 0.063 -> 0.033, mxfp8 0.024 with it; without it mxfp8
@@ -1053,6 +1076,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
default_on = True,
),
auto_prefer = _GATED_AUTO_PREFER,
+ auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,
preview_factors = "zimage",
),
# B200 too: 1 of 4 fresh servers picked R0_BLOCK 1024 (others 4096) and rendered other latents.
diff --git a/studio/backend/core/inference/diffusion_rocm_fused.py b/studio/backend/core/inference/diffusion_rocm_fused.py
index f959e9c7987..92788912bb2 100644
--- a/studio/backend/core/inference/diffusion_rocm_fused.py
+++ b/studio/backend/core/inference/diffusion_rocm_fused.py
@@ -5,8 +5,9 @@
RoPE reuses ``diffusion_flux2_rope``'s kernel (bit-identical to stock). AdaLN keeps stock's rounding chain; only the
row-reduction order differs. Inference only, never while compiling; any ineligible call runs the forward live at
-install. ``UNSLOTH_DIFFUSION_FUSED_ROPE=auto|0|1`` (auto = ROCm only), ``UNSLOTH_DIFFUSION_FUSED_ADALN=auto|0|1``
-(auto = off: no gfx1151 speedup and not bit-identical).
+install. ``UNSLOTH_DIFFUSION_FUSED_ROPE=auto|0|1`` (auto = ROCm and NVIDIA CUDA on Linux; ComfyUI runs FLUX RoPE
+through comfy-kitchen's fused kernel by default), ``UNSLOTH_DIFFUSION_FUSED_ADALN=auto|0|1`` (auto = off: no gfx1151
+speedup and not bit-identical).
"""
from __future__ import annotations
@@ -33,6 +34,8 @@ _ADALN_PREV: dict = {}
# Survives uninstall, so a fused forward restored later by another layer still reaches a real forward.
_ADALN_ORIGINAL: dict = {}
COUNTS = {"rope_fused": 0, "rope_stock": 0, "adaln_fused": 0, "adaln_stock": 0}
+# Set by the first failed RoPE launch (JIT / driver / PTX): the rest of the load runs stock, not one failing JIT per call.
+_ROPE_FAILED = [False]
def _mode(env: str) -> str:
@@ -52,13 +55,33 @@ def _is_rocm() -> bool:
return bool(getattr(torch.version, "hip", None))
+def _is_nvidia_linux() -> bool:
+ """NVIDIA CUDA torch on Linux (WSL included). Windows keeps stock: a Triton JIT failure would retry per call."""
+ if not sys.platform.startswith("linux"):
+ return False
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return False
+ return getattr(torch.version, "hip", None) is None and bool(
+ getattr(torch.version, "cuda", None)
+ )
+
+
# gfx1151 klein 1024^2: RoPE 8.76 -> 8.09 s pixel-identical; AdaLN adds no speed and moves pixels (LPIPS 0.004).
_AUTO_ON = {FUSED_ROPE_ENV: True, FUSED_ADALN_ENV: False}
+_AUTO_ON_NVIDIA = {FUSED_ROPE_ENV: True, FUSED_ADALN_ENV: False}
def _switch_on(env: str) -> bool:
mode = _mode(env)
- return mode == "on" or (mode == "auto" and _AUTO_ON.get(env, False) and _is_rocm())
+ if mode == "on":
+ return True
+ if mode != "auto":
+ return False
+ if _AUTO_ON.get(env, False) and _is_rocm():
+ return True
+ return _AUTO_ON_NVIDIA.get(env, False) and _is_nvidia_linux()
def wanted(
@@ -131,7 +154,7 @@ def _make_rope(module_name: str) -> Callable:
stock = _ROPE_STOCK.get(module_name) or _ROPE_ORIGINAL[module_name]
# Checked first: Dynamo graph-breaks on Triton's import path, so a compiled block stays whole on stock.
if not torch.compiler.is_compiling():
- launch = _rope_kernel()
+ launch = None if _ROPE_FAILED[0] else _rope_kernel()
if launch is not None and _rope_eligible(
x, freqs_cis, use_real, use_real_unbind_dim, sequence_dim
):
@@ -140,7 +163,7 @@ def _make_rope(module_name: str) -> Callable:
COUNTS["rope_fused"] += 1
return out
except Exception: # noqa: BLE001 -- never fail a render over the fast path
- pass
+ _ROPE_FAILED[0] = True
COUNTS["rope_stock"] += 1
return stock(
x,
@@ -188,6 +211,7 @@ def install_rope(
def uninstall_rope() -> None:
with _LOCK:
+ _ROPE_FAILED[0] = False
for name in list(_ROPE_STOCK):
stock = _ROPE_STOCK.pop(name)
mod = sys.modules.get(name)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 67964b1874f..e59166a39c7 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -9732,8 +9732,9 @@ class VideoBackend:
logger = logger,
)
install_cfg_parallel(pipe, offload_policy = offload_policy, logger = logger)
- # both branches through one offloaded forward; declines when CFG-parallel took the slot
- install_cfg_batch(pipe, offload_policy = offload_policy, logger = logger)
+ install_cfg_batch(
+ pipe, offload_policy = offload_policy, speed_mode = speed_mode, logger = logger
+ )
# Opt-in (UNSLOTH_DIFFUSION_BG_COMPILE=1): off by default so the same seed twice repeats.
from . import diffusion_bg_compile as bg_compile
diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
index 1bff063580b..f125876c89b 100644
--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
@@ -1118,7 +1118,7 @@
"pad": [],
"zero_row": []
},
- "auto_bf16_when_resident": null,
+ "auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
"bf16": {
"convrot_filename": null,
"convrot_repo": null,
@@ -1212,7 +1212,7 @@
"pad": [],
"zero_row": []
},
- "auto_bf16_when_resident": null,
+ "auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
"bf16": {
"convrot_filename": null,
"convrot_repo": null,
@@ -1335,7 +1335,7 @@
"pad": [],
"zero_row": []
},
- "auto_bf16_when_resident": null,
+ "auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
"bf16": {
"convrot_filename": null,
"convrot_repo": null,
@@ -3274,7 +3274,7 @@
"pad": [],
"zero_row": []
},
- "auto_bf16_when_resident": null,
+ "auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
"bf16": {
"convrot_filename": null,
"convrot_repo": null,
@@ -3415,8 +3415,12 @@
"wan2.2-ti2v-5b"
],
"family_auto_bf16_when_resident": {
+ "flux.2-klein": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
+ "flux.3-klein": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
+ "flux.3-klein-base": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
"hidream-i1": "INT8 is barely faster than compiled bf16 on a card that holds bf16 and changes image detail",
- "lumina-2": "INT8 is no faster than compiled bf16 on a card that holds bf16 and changes image detail"
+ "lumina-2": "INT8 is no faster than compiled bf16 on a card that holds bf16 and changes image detail",
+ "z-image": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)"
},
"family_auto_prefer": {
"flux.1": {
diff --git a/studio/backend/tests/test_diffusion_attention_blackwell.py b/studio/backend/tests/test_diffusion_attention_blackwell.py
new file mode 100644
index 00000000000..b4cdb0e1669
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_attention_blackwell.py
@@ -0,0 +1,127 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Auto attention on Blackwell: native SDPA when torch's own dispatch already runs cuDNN at every DiT head dim (the
+pin changes no kernel and costs a per-call wrapper), the cuDNN pin everywhere else and for an explicit request."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+import torch
+
+import core.inference.diffusion_attention as att
+from core.inference.diffusion_attention import apply_attention_backend, select_attention_backend
+
+CUDNN = int(torch.nn.attention.SDPBackend.CUDNN_ATTENTION)
+FLASH = int(torch.nn.attention.SDPBackend.FLASH_ATTENTION)
+
+
+class _Dit:
+ def __init__(self, head_dim = 128):
+ self.config = {"attention_head_dim": head_dim}
+ self.set_to = None
+
+ def set_attention_backend(self, name):
+ self.set_to = name
+
+
+def _target(dtype = torch.bfloat16):
+ return types.SimpleNamespace(device = "cuda", dtype = dtype)
+
+
+def blackwell(monkeypatch):
+ """An NVIDIA SM100 card whose native dispatch picks cuDNN up to head_dim 128 and FLASH above (torch 2.13 / B200)."""
+ att._NATIVE_DISPATCH_CACHE.clear()
+ monkeypatch.delenv(att.BLACKWELL_NATIVE_ATTN_ENV, raising = False)
+ monkeypatch.setenv(att.CUDNN_HEAD_DIM_PROBE_ENV, "0")
+ monkeypatch.setattr(att, "_is_cuda_nvidia", lambda target: True)
+ monkeypatch.setattr(att, "_cuda_capability", lambda: (10, 0))
+ monkeypatch.setattr(att, "_indexed_cuda_device", lambda device: "cuda:0")
+ monkeypatch.setattr(att, "guard_rocm_fused_sdpa", lambda target, logger = None: None)
+ state = {"cap": (10, 0), "asked": []}
+ monkeypatch.setattr(att, "_device_capability", lambda device: state["cap"])
+
+ def choice(device, dtype, dim):
+ state["asked"].append((dtype, dim))
+ return CUDNN if dim <= 128 else FLASH
+
+ monkeypatch.setattr(att, "_native_sdp_choice", choice)
+ yield state
+ att._NATIVE_DISPATCH_CACHE.clear()
+
+
+def test_auto_pick_still_reads_as_cudnn_for_every_caller(blackwell):
+ pick = select_attention_backend(_target(), "auto", speed_active = True)
+ assert pick == "_native_cudnn" and isinstance(pick, att._AutoCudnn)
+ explicit = select_attention_backend(_target(), "cudnn", speed_active = True)
+ assert explicit == "_native_cudnn" and not isinstance(explicit, att._AutoCudnn)
+ assert select_attention_backend(_target(), "auto", speed_active = False) is None
+
+
+def test_blackwell_auto_leaves_native_when_dispatch_already_picks_cudnn(blackwell):
+ dit = _Dit(128)
+ pick = select_attention_backend(_target(), "auto", speed_active = True)
+ assert (
+ apply_attention_backend(types.SimpleNamespace(transformer = dit), pick, target = _target())
+ is None
+ )
+ assert dit.set_to in (None, "native")
+ assert blackwell["asked"] == [(torch.bfloat16, 128)]
+
+
+def test_an_explicit_cudnn_request_is_still_pinned_on_blackwell(blackwell):
+ dit = _Dit(128)
+ engaged = apply_attention_backend(
+ types.SimpleNamespace(transformer = dit), "_native_cudnn", target = _target()
+ )
+ assert engaged == "_native_cudnn" and dit.set_to == "_native_cudnn"
+ assert blackwell["asked"] == []
+
+
+ "case",
+ ["hopper", "head_dim_256", "fp32", "kill_switch", "unknown_head_dim", "no_target"],
+)
+def test_auto_keeps_the_cudnn_pin_wherever_native_could_differ(blackwell, monkeypatch, case):
+ dit = _Dit(256 if case == "head_dim_256" else 128)
+ target = _target(torch.float32 if case == "fp32" else torch.bfloat16)
+ if case == "hopper":
+ blackwell["cap"] = (9, 0)
+ if case == "kill_switch":
+ monkeypatch.setenv(att.BLACKWELL_NATIVE_ATTN_ENV, "0")
+ if case == "unknown_head_dim":
+ dit.config = {}
+ pick = select_attention_backend(target, "auto", speed_active = True)
+ engaged = apply_attention_backend(
+ types.SimpleNamespace(transformer = dit), pick, target = None if case == "no_target" else target
+ )
+ assert engaged == "_native_cudnn" and dit.set_to == "_native_cudnn"
+ assert type(engaged) is str # the marker never leaks past apply (status, cache keys)
+
+
+def test_mixed_head_dims_need_cudnn_at_every_one(blackwell):
+ pipe = types.SimpleNamespace(transformer = _Dit(128), transformer_2 = _Dit(256))
+ assert att._native_dispatches_cudnn(pipe, _target()) is False
+ blackwell["asked"].clear()
+ att._NATIVE_DISPATCH_CACHE.clear()
+ assert (
+ att._native_dispatches_cudnn(types.SimpleNamespace(transformer = _Dit(64)), _target()) is True
+ )
+ assert (
+ att._native_dispatches_cudnn(types.SimpleNamespace(transformer = _Dit(64)), _target()) is True
+ )
+ assert blackwell["asked"] == [(torch.bfloat16, 64)] # cached per (device, dtype, head_dim)
+
+
+def test_an_unaskable_probe_keeps_the_pin(blackwell, monkeypatch):
+ def boom(device, dtype, dim):
+ raise RuntimeError("no device")
+
+ monkeypatch.setattr(att, "_native_sdp_choice", boom)
+ assert (
+ att._native_dispatches_cudnn(types.SimpleNamespace(transformer = _Dit(128)), _target())
+ is False
+ )
diff --git a/studio/backend/tests/test_diffusion_autoopt_parity.py b/studio/backend/tests/test_diffusion_autoopt_parity.py
index 5ea15a38a12..787c0099080 100644
--- a/studio/backend/tests/test_diffusion_autoopt_parity.py
+++ b/studio/backend/tests/test_diffusion_autoopt_parity.py
@@ -473,6 +473,9 @@ def snapshot(entry: Callable) -> dict:
patch.setattr(diffusion_flux2_rope, "_kernel", lambda: (lambda *a, **k: None))
patch.setattr(video_wan_fused, "_kernels", lambda: {})
+ from core.inference import diffusion_rocm_fused
+
+ patch.setattr(diffusion_rocm_fused, "_is_nvidia_linux", lambda: False)
out: dict = {}
try:
for backend, fam in _families():
diff --git a/studio/backend/tests/test_diffusion_cfg_batch.py b/studio/backend/tests/test_diffusion_cfg_batch.py
index c13374ecaa5..65b4e70bf33 100644
--- a/studio/backend/tests/test_diffusion_cfg_batch.py
+++ b/studio/backend/tests/test_diffusion_cfg_batch.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""CFG batching (diffusion_cfg_batch): both guidance branches through one offloaded forward, CPU fakes."""
+"""CFG batching (diffusion_cfg_batch): both guidance branches through one forward (offloaded or resident), CPU fakes."""
from __future__ import annotations
@@ -95,17 +95,18 @@ def _flux_loop(
neg = None,
guided = None,
between = None,
+ device = "cpu",
):
"""The diffusers FLUX.2-klein loop: per step one latent object, cond then uncond call, CFG combine."""
torch.manual_seed(0)
- latents = torch.randn(1, 4, 8)
- pos = torch.randn(1, 5, 8)
- neg = torch.randn(1, 5, 8) if neg is None else neg
- txt_ids = torch.zeros(1, 5, 4)
- img_ids = torch.ones(1, 4, 4)
+ latents = torch.randn(1, 4, 8).to(device)
+ pos = torch.randn(1, 5, 8).to(device)
+ neg = (torch.randn(1, 5, 8) if neg is None else neg).to(device)
+ txt_ids = torch.zeros(1, 5, 4, device = device)
+ img_ids = torch.ones(1, 4, 4, device = device)
outs = []
for i in range(steps):
- t = torch.full((1,), 1.0 - i / steps)
+ t = torch.full((1,), 1.0 - i / steps, device = device)
latent_model_input = latents.clone()
cond = model(
hidden_states = latent_model_input,
@@ -155,10 +156,27 @@ def test_batched_steps_match_two_calls_and_engage(monkeypatch):
assert model.calls[-2:] == [2, 2]
assert cb.cfg_batch_status(pipe)["batched_steps"] == 4
cb.end_cfg_batch_render(pipe)
- assert cb.cfg_batch_status(pipe)["plain_pair_s"] is not None or not torch.cuda.is_available()
+ assert cb.cfg_batch_status(pipe)["plain_pair_s"] is None
assert cb.cfg_batch_status(pipe)["shape_off"] is None
[email protected](not torch.cuda.is_available(), reason = "needs a CUDA device timer")
+def test_cuda_latents_time_the_step_two_pair_and_keep_the_image(monkeypatch):
+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()).cuda(), device = "cuda")
+ model = Flux2Transformer2DModel().cuda()
+ pipe, router = _armed(model, monkeypatch)
+ got, _ = _flux_loop(model, device = "cuda")
+ torch.testing.assert_close(got, ref)
+ status = cb.cfg_batch_status(pipe)
+ assert status["plain_pair_s"] is not None and status["batched_pair_s"] is not None
+ if status["shape_off"] is None:
+ assert router.stats["batched_steps"] == 4 and model.calls[-2:] == [2, 2]
+ else:
+ assert "measured no gain" in status["shape_off"]
+ assert router.stats["batched_steps"] == 0 and model.calls[-2:] == [1, 1]
+ cb.end_cfg_batch_render(pipe)
+
+
def test_wan_expanded_timesteps_serve_the_second_branch(monkeypatch):
"""WanPipeline's stride-0 ``t.expand(batch)`` timestep must compare equal, not disable batching."""
@@ -220,7 +238,7 @@ def test_qwen_unequal_prompt_lengths_and_img_shapes_batch(monkeypatch):
def test_a_changed_second_branch_reruns_plain_and_stops_after_three_misses(monkeypatch):
# uncond prompt rewritten in place: the doubled call must miss, rerun plain, and match the two-call result
def between(i, neg):
- if i >= 2:
+ if i >= 3:
neg.add_(0.5)
ref, _ = _flux_loop(
@@ -230,7 +248,7 @@ def test_a_changed_second_branch_reruns_plain_and_stops_after_three_misses(monke
_pipe, router = _armed(model, monkeypatch)
got, _ = _flux_loop(model, steps = 8, neg = torch.zeros(1, 5, 8), between = between)
torch.testing.assert_close(got, ref, rtol = 0, atol = 0)
- assert router.stats["misses"] == 3 and router.stats["hits"] == 0
+ assert router.stats["misses"] == 3 and router.stats["hits"] == 1
assert router.shape.off is not None and "3 consecutive misses" in router.shape.off
@@ -287,10 +305,285 @@ def test_switches_are_independent_of_multi_gpu(monkeypatch):
assert cb.cfg_batch_decline(pipe, offload_policy = "group") == f"{cb.CFG_BATCH_ENV}=0"
-def test_declines(monkeypatch):
+def test_a_resident_denoiser_is_armed_by_default(monkeypatch):
resident = _Pipe(Flux2Transformer2DModel())
- assert "resident" in cb.cfg_batch_decline(resident, offload_policy = "none")
+ assert cb.cfg_batch_decline(resident, offload_policy = "none") is None
+ assert cb.install_cfg_batch(resident, offload_policy = "none") is None
+ assert cb.router_for(resident).reason == "resident denoiser"
+ assert "model offload" in cb.cfg_batch_decline(
+ _Pipe(Flux2Transformer2DModel()), offload_policy = "model"
+ )
+ monkeypatch.setenv(cb.CFG_BATCH_ENV, "0")
+ assert cb.cfg_batch_decline(_Pipe(Flux2Transformer2DModel()), offload_policy = "none") == (
+ f"{cb.CFG_BATCH_ENV}=0"
+ )
+
+
+class _Clock:
+ """A fake device timer: each forward costs ``cost(batch)`` seconds of the router's perf_counter."""
+
+ def __init__(self, cost):
+ self.now = 0.0
+ self.cost = cost
+
+ def __call__(self):
+ return self.now
+
+
+def _timed(monkeypatch, cost):
+ clock = _Clock(cost)
+ monkeypatch.setattr(cb.time, "perf_counter", clock)
+ monkeypatch.setattr(cb, "_RESIDENT_MAX_PAIR_S", 1e9)
+ monkeypatch.setattr(cb.CfgBatchRouter, "_sync", lambda self, latent: True)
+
+ def forward(self, *a, **k):
+ clock.now += clock.cost(int(k["hidden_states"].shape[0]))
+ return Flux2Transformer2DModel.forward(self, *a, **k)
+
+ timed = type("Flux2Transformer2DModel", (Flux2Transformer2DModel,), {"forward": forward})
+ timed.__module__ = "diffusers.models.fake"
+ return timed()
+
+
[email protected]("batched_cost, wins", [(1.2, True), (2.0, False)])
+def test_the_timing_trial_never_mixes_paths_and_repeats_bit_identically(
+ monkeypatch, batched_cost, wins
+):
+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else batched_cost)
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ router = cb.router_for(pipe)
+ first, _ = _flux_loop(model)
+ cb.end_cfg_batch_render(pipe)
+ calls_after_first = list(model.calls)
+ second, _ = _flux_loop(model)
+ cb.end_cfg_batch_render(pipe)
+ torch.testing.assert_close(first, second, rtol = 0, atol = 0)
+ assert calls_after_first[:10] == [1, 1, 1, 1, 1, 1, 2, 1, 1, 2]
+ render2 = model.calls[len(calls_after_first) :]
+ if wins:
+ assert render2 == [1, 1, 1, 1] + [2] * 4 and router.last_shape.off is None
+ else:
+ assert render2 == [1] * 12 and "measured no gain" in router.last_shape.off
+ plain = Flux2Transformer2DModel()
+ ref_plain, _ = _flux_loop(plain)
+ if not wins:
+ torch.testing.assert_close(second, ref_plain, rtol = 0, atol = 0)
+ else:
+ torch.testing.assert_close(second, ref_plain)
+
+
+def test_a_shape_that_stays_plain_records_no_prediction(monkeypatch):
+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else 2.0)
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ router = cb.router_for(pipe)
+ _flux_loop(model)
+ cb.end_cfg_batch_render(pipe)
+ assert "measured no gain" in router.last_shape.off
+ recorded = []
+ monkeypatch.setattr(router, "_record_template", lambda *a, **k: recorded.append(1))
+ _flux_loop(model)
+ cb.end_cfg_batch_render(pipe)
+ assert recorded == [] and model.calls[-12:] == [1] * 12
+
+
+def _quiet_flux():
+ """The fake FLUX.2 transformer without its call log (a list append Dynamo would have to trace)."""
+
+ def forward(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states,
+ txt_ids,
+ img_ids,
+ return_dict = False,
+ **_,
+ ):
+ text = encoder_hidden_states.mean(dim = 1, keepdim = True)
+ return (
+ hidden_states * self.w
+ + text
+ + timestep.view(-1, 1, 1)
+ + txt_ids.mean()
+ + img_ids.mean(),
+ )
+
+ cls = type("Flux2Transformer2DModel", (Flux2Transformer2DModel,), {"forward": forward})
+ cls.__module__ = "diffusers.models.fake"
+ return cls()
+
+
+def test_compiled_denoisers_are_seen_in_every_form():
+ plain = Flux2Transformer2DModel()
+ assert cb.compiled_denoiser(plain) is None
+ marked = Flux2Transformer2DModel()
+ marked._unsloth_regional_compile_kwargs = {"dynamic": False}
+ assert cb.compiled_denoiser(marked) == "the denoiser is compiled"
+ block = Flux2Transformer2DModel()
+ block.add_module("blocks", torch.nn.ModuleList([torch.nn.Linear(1, 1)]))
+ block.blocks[0].compile()
+ assert "blocks.0 is compiled" == cb.compiled_denoiser(block)
+ # #14528: the compile moved below a group-offload hook, the block's _compiled_call_impl cleared
+ hooked = Flux2Transformer2DModel()
+ hooked.add_module("blocks", torch.nn.ModuleList([torch.nn.Linear(1, 1)]))
+ ref = types.SimpleNamespace(forward = torch.compile(hooked.blocks[0].forward))
+ hooked.blocks[0]._diffusers_hook = types.SimpleNamespace(
+ hooks = {"group_offloading": object()}, _fn_refs = [ref]
+ )
+ assert "below its hooks" in cb.compiled_denoiser(hooked)
+ assert (
+ cb.cfg_batch_decline(_Pipe(marked), offload_policy = "none")
+ == "the denoiser is compiled and resident"
+ )
+ assert cb.cfg_batch_decline(_Pipe(_offloaded(marked)), offload_policy = "group") is None
+
+
+def test_a_compile_on_a_resident_denoiser_after_install_keeps_every_render_plain(monkeypatch):
+ model = Flux2Transformer2DModel()
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ model._unsloth_regional_compile_kwargs = {
+ "dynamic": False
+ } # the deferred profile compiled the blocks
+ _flux_loop(model)
+ cb.end_cfg_batch_render(pipe)
+ _flux_loop(model)
+ assert model.calls == [1] * 24
+ assert cb.cfg_batch_status(pipe)["render_off"] == "the denoiser is compiled and resident"
+
+
+def test_an_offloaded_shape_judged_before_a_compile_stays_plain_after_it(monkeypatch):
+ model = Flux2Transformer2DModel()
+ pipe, router = _armed(model, monkeypatch)
+ _flux_loop(model) # eager render: trial on step 1, steps 2-6 batched
+ cb.end_cfg_batch_render(pipe)
+ assert router.stats["batched_steps"] == 4
+ model._unsloth_regional_compile_kwargs = {
+ "dynamic": False
+ } # compiled below the offload hooks afterwards
+ del model.calls[:]
+ _flux_loop(model)
+ assert model.calls == [1] * 12 # no doubled call reaches the new compile on a follow-up prompt
+ assert "compiled after this shape's trial" in router.last_shape.off
+
+
+def test_an_offloaded_compiled_denoiser_batches_from_its_trial_render(monkeypatch):
+ model = Flux2Transformer2DModel()
+ model._unsloth_regional_compile_kwargs = {"dynamic": False}
+ pipe, router = _armed(model, monkeypatch)
+ for _ in range(2):
+ _flux_loop(model)
+ cb.end_cfg_batch_render(pipe)
+ assert router.stats["batched_steps"] == 8 and router.disabled is None
+ assert router.last_shape.compiled_at_trial and router.last_shape.trial_render == 0
+
+
+def test_a_doubled_call_compiling_on_a_follow_up_render_ends_batching_after_that_render(
+ monkeypatch,
+):
+ model = Flux2Transformer2DModel()
+ pipe, router = _armed(model, monkeypatch)
+ _flux_loop(model)
+ cb.end_cfg_batch_render(pipe)
+ graphs = iter(range(1000))
+ monkeypatch.setattr(
+ cb, "_graph_count", lambda: next(graphs)
+ ) # every doubled call "compiles" from now on
+ del model.calls[:]
+ _flux_loop(model)
+ assert model.calls == [1, 1, 1, 1] + [2] * 4 # render 2 stays on its one path
+ cb.end_cfg_batch_render(pipe)
+ assert "after its warm-up render" in router.disabled
+ del model.calls[:]
+ _flux_loop(model)
+ assert model.calls == [1] * 12
+
+
+def test_a_doubled_call_that_compiles_a_graph_turns_batching_off(monkeypatch):
+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else 1.2)
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ graphs = iter(range(100))
+ monkeypatch.setattr(cb, "_graph_count", lambda: next(graphs))
+ _flux_loop(model)
+ router = cb.router_for(pipe)
+ assert "compiled a new graph" in router.disabled
+ assert (
+ model.calls[:4] == [1, 1, 1, 1] and model.calls[-8:] == [1] * 8
+ ) # nothing batched after the trial
+
+
+def test_a_router_traced_by_dynamo_passes_straight_through():
+ model = _quiet_flux()
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ torch._dynamo.reset()
+ compiled = torch.compile(model, backend = "eager", fullgraph = True)
+ kw = dict(
+ timestep = torch.ones(1),
+ encoder_hidden_states = torch.randn(1, 5, 8),
+ txt_ids = torch.zeros(1, 5, 4),
+ img_ids = torch.ones(1, 4, 4),
+ )
+ for batch in (1, 2):
+ x = torch.randn(batch, 4, 8)
+ out = compiled(hidden_states = x, **{**kw, "timestep": torch.ones(batch)})[0]
+ torch.testing.assert_close(
+ out,
+ model.forward.orig_forward(hidden_states = x, **{**kw, "timestep": torch.ones(batch)})[0],
+ )
+ assert (
+ cb.router_for(pipe).stats["plain_calls"] == 0
+ ) # the traced calls never ran the router's own logic
+
+
[email protected]("pair_s, gated", [(0.3, True), (0.1, False)])
+def test_a_compute_bound_resident_pair_skips_the_trial(monkeypatch, pair_s, gated):
+ model = _timed(monkeypatch, lambda b: pair_s / 2 if b == 1 else pair_s)
+ monkeypatch.setattr(cb, "_RESIDENT_MAX_PAIR_S", 0.25)
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ _flux_loop(model)
+ off = cb.router_for(pipe).last_shape.off
+ if gated:
+ assert "compute-bound" in off and model.calls == [1] * 12
+ else:
+ assert "measured no gain" in off and 2 in model.calls # the trial ran its doubled calls
+
+
+def test_an_offloaded_pair_is_never_gated_as_compute_bound(monkeypatch):
+ model = _timed(monkeypatch, lambda b: 5.0 if b == 1 else 6.0)
+ monkeypatch.setattr(cb, "_RESIDENT_MAX_PAIR_S", 0.25)
+ pipe = _Pipe(_offloaded(model))
+ assert cb.install_cfg_batch(pipe, offload_policy = "group") is None
+ _flux_loop(model)
+ assert (
+ cb.router_for(pipe).last_shape.off is None and model.calls[-1] == 2
+ ) # an upload saved per step wins
+
+
+def test_one_slow_plain_sample_does_not_decide_the_trial(monkeypatch):
+ seen = {"n": 0}
+ def cost(b):
+ seen["n"] += 1
+ if seen["n"] == 5: # the trial's first plain call (after steps 1 and 2)
+ return 50.0
+ return 1.0 if b == 1 else 2.0
+
+ model = _timed(monkeypatch, cost)
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ _flux_loop(model)
+ cb.end_cfg_batch_render(pipe)
+ shape = cb.router_for(pipe).last_shape
+ assert shape.off is not None and "measured no gain" in shape.off
+ assert shape.plain_pair_s == 2.0 and min(shape.batched_s) == 2.0
+
+
+def test_declines(monkeypatch):
class ZImageTransformer2DModel(Flux2Transformer2DModel):
pass
@@ -411,3 +704,52 @@ def test_a_new_request_shape_starts_without_the_previous_shapes_template(monkeyp
big = torch.randn(2, 4, 8)
router._new_step((), {"hidden_states": big}, big)
assert router.last_shape is not first and router.template is None
+
+
+def test_the_trial_prices_the_batched_steps_preparation_and_verification(monkeypatch):
+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else 1.2)
+ clock = cb.time.perf_counter
+ real_matches = cb._matches
+
+ def slow_matches(snapshots, call):
+ clock.now += 0.7
+ return real_matches(snapshots, call)
+
+ monkeypatch.setattr(cb, "_matches", slow_matches)
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ _flux_loop(model)
+ shape = cb.router_for(pipe).last_shape
+ assert shape.off is not None and "measured no gain" in shape.off
+ assert min(shape.batched_s) == pytest.approx(1.9)
+
+
[email protected]("offload_policy", ["none", "group"])
+def test_an_explicit_speed_off_request_keeps_one_branch_per_call(monkeypatch, offload_policy):
+ def pipe():
+ model = Flux2Transformer2DModel()
+ return _Pipe(_offloaded(model) if offload_policy == "group" else model)
+
+ off = pipe()
+ reason = cb.install_cfg_batch(off, offload_policy = offload_policy, speed_mode = "off")
+ assert reason is not None and "speed_mode=off" in reason and cb.router_for(off) is None
+ assert "forward" not in off.transformer.__dict__
+ assert "speed_mode=off" in cb.cfg_batch_decline(
+ pipe(), offload_policy = offload_policy, speed_mode = " OFF "
+ )
+ for speed in (None, "default", "max"): # unset keeps the default-on lever
+ assert cb.cfg_batch_decline(pipe(), offload_policy = offload_policy, speed_mode = speed) is None
+ monkeypatch.setenv(cb.CFG_BATCH_ENV, "force")
+ assert cb.cfg_batch_decline(pipe(), offload_policy = offload_policy, speed_mode = "off") is None
+
+
+def test_both_backends_pass_the_requested_speed_mode():
+ import inspect
+
+ from core.inference import diffusion, video
+ for module in (diffusion, video):
+ src = inspect.getsource(module)
+ calls = [c for c in src.split("install_cfg_batch(")[1:] if not c.startswith("pipe: Any")]
+ assert calls and all(
+ "speed_mode = speed_mode" in c.split(")")[0] for c in calls
+ ), module.__name__
diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.py
index 0c17ad59d79..9fb4dce6284 100644
--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py
+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py
@@ -1291,6 +1291,7 @@ def test_an_uncompilable_family_loads_an_operators_checkpoint_under_auto(
def test_a_compilable_family_loads_without_the_bf16_replan(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch)
monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)
+ monkeypatch.setattr(dmod, "auto_bf16_when_resident_reason", lambda _name: None)
replans = []
monkeypatch.setattr(
DiffusionBackend,
@@ -1405,7 +1406,12 @@ def test_a_measured_bf16_family_quantises_auto_when_the_budget_is_unmeasured(
("Alpha-VLLM/Lumina-Image-2.0", True),
("HiDream-ai/HiDream-I1-Full", True),
("black-forest-labs/FLUX.1-dev", False),
- ("Tongyi-MAI/Z-Image-Turbo", False),
+ ("Tongyi-MAI/Z-Image-Turbo", True),
+ ("black-forest-labs/FLUX.2-klein-4B", True),
+ ("black-forest-labs/FLUX.2-klein-base-4B", True),
+ ("black-forest-labs/flux-3-image-klein", True),
+ ("black-forest-labs/flux-3-image-klein-base", True),
+ ("black-forest-labs/FLUX.2-dev", False),
("Qwen/Qwen-Image", False),
],
)
@@ -1870,3 +1876,71 @@ def test_the_streamed_seed_credits_host_ram_the_outgoing_pipeline_frees(
),
)
assert (_settle(backend) == "fp8") is expected
+
+
+def _small_dit_kept_bf16(monkeypatch, reason):
+ _measured_bf16_family(monkeypatch)
+ monkeypatch.setattr(dmod, "auto_bf16_when_resident_reason", lambda _name: reason)
+ monkeypatch.setattr(dmod, "compile_eligible", lambda *_a, **_k: True)
+ monkeypatch.setattr(dmod, "fp16_compile_explicit_only", lambda _t: False)
+
+
+ "reason, speed, expect_speed, expect_deferred",
+ [
+ (dmod.SMALL_DIT_BF16_WHEN_RESIDENT, None, "default", False),
+ (dmod.SMALL_DIT_BF16_WHEN_RESIDENT, "off", "off", False),
+ ("measured: no faster", None, "off", True),
+ ],
+)
+def test_a_small_dit_kept_bf16_compiles_from_render_1(
+ fake_runtime, monkeypatch, reason, speed, expect_speed, expect_deferred
+):
+ backend, spy = _load_backend(monkeypatch)
+ _small_dit_kept_bf16(monkeypatch, reason)
+ status = _load(
+ backend, speed_mode = speed, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ()
+ )
+
+ assert spy.quantised == [] and spy.seeds == []
+ assert status["resolved"]["transformer_quant"]["value"] == "off"
+ assert backend._state.speed_mode == expect_speed
+ assert backend._state.speed_deferred is expect_deferred
+
+
+ "repo, kept",
+ [
+ ("black-forest-labs/FLUX.2-klein-4B", True),
+ ("black-forest-labs/FLUX.2-klein-base-4B", True),
+ ("unsloth/FLUX.2-klein-4B", True),
+ ("black-forest-labs/FLUX.2-klein-9B", False),
+ ("black-forest-labs/FLUX.2-klein-base-9B", False),
+ ("unsloth/FLUX.2-klein-9B", False),
+ ("unsloth/FLUX.2-klein-base-9B", False),
+ ],
+)
+def test_the_small_dit_rule_skips_the_9b_klein_variants(repo, kept):
+ fam = detect_family_for_pick(repo, None, None)
+ assert fam is not None and fam.name == "flux.2-klein"
+ assert (dmod._auto_keeps_bf16_reason(fam, "pipeline", repo) is not None) is kept
+ fits = types.SimpleNamespace(
+ offload_policy = "none",
+ estimates = {"safe_device_budget_mib": 60_000, "resident_required_mib": 30_000},
+ )
+ assert (dmod._auto_quant_eager_reason(fam, fits, None, "pipeline", repo) is not None) is kept
+
+
+def test_a_large_variant_of_a_small_dit_family_quantises_under_auto(fake_runtime, monkeypatch):
+ backend, spy = _load_backend(monkeypatch)
+ _small_dit_kept_bf16(monkeypatch, dmod.SMALL_DIT_BF16_WHEN_RESIDENT)
+ seen = []
+
+ def excludes(base):
+ seen.append(base)
+ return base == Z_IMAGE_REPO
+
+ monkeypatch.setattr(dmod, "small_dit_rule_excludes", excludes)
+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
+ assert Z_IMAGE_REPO in seen # the load hands the decider its resolved base
+ assert spy.quantised == ["auto"]
diff --git a/studio/backend/tests/test_diffusion_rocm_fused.py b/studio/backend/tests/test_diffusion_rocm_fused.py
index 2587b41b1a9..f653c96abf6 100644
--- a/studio/backend/tests/test_diffusion_rocm_fused.py
+++ b/studio/backend/tests/test_diffusion_rocm_fused.py
@@ -1,8 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Tests for ``diffusion_rocm_fused.py``: the switches (auto = ROCm only), install / uninstall wiring, and on a GPU the
-kernels against diffusers (RoPE bit-identical, AdaLN within one rounding of stock) plus a tiny FLUX.1 transformer."""
+"""Tests for ``diffusion_rocm_fused.py``: the switches (auto = RoPE on ROCm and on NVIDIA CUDA under Linux, AdaLN
+opt-in), install / uninstall wiring, and on a GPU the kernels against diffusers (RoPE bit-identical, AdaLN within one
+rounding of stock) plus tiny FLUX.1 / FLUX.2 transformers."""
from __future__ import annotations
@@ -54,12 +55,59 @@ def fake_kernels(monkeypatch):
monkeypatch.setattr(rf, "_adaln_kernel", lambda: (lambda x, scale, shift, eps: None))
-def test_auto_is_inert_off_rocm(fake_kernels, monkeypatch):
+def _nvidia(monkeypatch, platform = "linux"):
monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setattr(torch.version, "cuda", "13.0", raising = False)
+ monkeypatch.setattr(rf.sys, "platform", platform)
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
+def test_auto_engages_rope_only_on_nvidia_linux(fake_kernels, monkeypatch, dtype):
+ _nvidia(monkeypatch)
+ assert rf.install_for_pipe(_PipeLike(), dtype, "cuda") == {"rope": True, "adaln": 0}
+ assert fmod.apply_rotary_emb is rf._ROPE_FNS[rf._MODULES[0]]
+ assert nm.AdaLayerNormZero.forward is _STOCK_FWD["AdaLayerNormZero"]
+
+
[email protected]("platform", ["win32", "darwin"])
+def test_auto_is_inert_on_nvidia_outside_linux(fake_kernels, monkeypatch, platform):
+ # Windows keeps stock: a Triton JIT failure there would be retried on every call.
+ _nvidia(monkeypatch, platform)
+ assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}
+ assert not rf.is_installed()
+
+
+def test_auto_is_inert_without_a_gpu_torch(fake_kernels, monkeypatch):
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setattr(torch.version, "cuda", None, raising = False)
+ monkeypatch.setattr(rf.sys, "platform", "linux")
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}
assert not rf.is_installed()
[email protected]("value", ["0", "off", "false"])
+def test_kill_switch_on_nvidia(fake_kernels, monkeypatch, value):
+ _nvidia(monkeypatch)
+ monkeypatch.setenv(rf.FUSED_ROPE_ENV, value)
+ assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}
+ assert fmod.apply_rotary_emb is stock_rope
+
+
+def test_flux2_fp16_keeps_its_own_patch_on_nvidia(fake_kernels, monkeypatch):
+ from core.inference import diffusion_flux2_rope as fr
+
+ f2mod = pytest.importorskip("diffusers.models.transformers.transformer_flux2")
+ _nvidia(monkeypatch)
+ monkeypatch.setattr(fr, "_kernel", lambda: (lambda *a, **k: None))
+ pipe = _PipeLike("diffusers.models.transformers.transformer_flux2")
+ try:
+ assert fr.install_for_pipe(pipe, torch.float16, "cuda")
+ assert rf.install_for_pipe(pipe, torch.float16, "cuda") == {"rope": False, "adaln": 0}
+ assert f2mod.apply_rotary_emb is fr._fused_apply_rotary_emb
+ finally:
+ fr.uninstall()
+
+
def test_auto_engages_rope_only_on_rocm(fake_kernels, monkeypatch):
# auto = the bit-identical RoPE kernel on ROCm; fused AdaLN bought no speed on gfx1151 and is opt-in.
monkeypatch.setattr(torch.version, "hip", "7.2.0", raising = False)
@@ -329,6 +377,55 @@ def test_tiny_flux1_transformer_end_to_end(monkeypatch):
assert rel < 1e-2, float(rel)
+@needs_gpu
+def test_tiny_flux2_transformer_default_on(monkeypatch):
+ f2 = pytest.importorskip("diffusers").Flux2Transformer2DModel
+ if not (rf._is_rocm() or rf._is_nvidia_linux()):
+ pytest.skip("auto engages on ROCm and NVIDIA Linux only")
+ dev, dtype = "cuda", torch.bfloat16
+ torch.manual_seed(0)
+ model = (
+ f2(
+ patch_size = 1,
+ in_channels = 16,
+ num_layers = 2,
+ num_single_layers = 2,
+ attention_head_dim = 32,
+ num_attention_heads = 4,
+ joint_attention_dim = 64,
+ timestep_guidance_channels = 32,
+ axes_dims_rope = (8, 8, 8, 8),
+ guidance_embeds = False,
+ )
+ .to(dev, dtype)
+ .eval()
+ )
+ img, txt = 256, 32
+ ids = torch.cartesian_prod(torch.arange(1), torch.arange(16), torch.arange(16), torch.arange(1))
+ inputs = dict(
+ hidden_states = torch.randn(1, img, 16, device = dev, dtype = dtype),
+ encoder_hidden_states = torch.randn(1, txt, 64, device = dev, dtype = dtype),
+ timestep = torch.tensor([0.5], device = dev, dtype = dtype),
+ img_ids = ids.to(dev)[None].float(),
+ txt_ids = torch.cartesian_prod(
+ torch.arange(1), torch.arange(1), torch.arange(1), torch.arange(txt)
+ )
+ .to(dev)[None]
+ .float(),
+ return_dict = False,
+ )
+ with torch.no_grad():
+ want = model(**inputs)[0]
+ assert rf.install_for_pipe(type("P", (), {"transformer": model})(), dtype, dev) == {
+ "rope": True,
+ "adaln": 0,
+ }
+ r0 = rf.COUNTS["rope_fused"]
+ got = model(**inputs)[0]
+ assert rf.COUNTS["rope_fused"] - r0 == 2 * 2 + 2 * 2
+ assert torch.equal(got, want)
+
+
def test_teardown_survives_a_later_eager_layer(fake_kernels, monkeypatch):
# Deferred speed profile: fused AdaLN at load, Studio's eager patch layered on top at the 3rd image, then unload
# in the order diffusion.py uses (fused teardown first, eager after).
@@ -364,3 +461,27 @@ def test_stock_branch_never_loses_its_previous_forward(fake_kernels, monkeypatch
finally:
for n, fwd in _STOCK_FWD.items():
getattr(nm, n).forward = fwd
+
+
+def test_a_failed_rope_launch_disables_the_fused_path_for_the_load(monkeypatch):
+ calls = []
+
+ def failing_launch(x, cos, sin):
+ calls.append(1)
+ raise RuntimeError("PTX JIT compilation failed")
+
+ monkeypatch.setattr(rf, "_rope_kernel", lambda: failing_launch)
+ monkeypatch.setattr(rf, "_rope_eligible", lambda *a, **k: True)
+ _nvidia(monkeypatch)
+ assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")["rope"] is True
+ x = torch.randn(1, 6, 2, 8)
+ pos = torch.randn(6, 4, dtype = torch.float64)
+ freqs = (pos.cos().repeat_interleave(2, -1).float(), pos.sin().repeat_interleave(2, -1).float())
+ want = stock_rope(x, freqs, sequence_dim = 1)
+ for _ in range(4):
+ assert torch.equal(fmod.apply_rotary_emb(x, freqs, sequence_dim = 1), want)
+ assert len(calls) == 1
+ rf.uninstall()
+ rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")
+ fmod.apply_rotary_emb(x, freqs, sequence_dim = 1)
+ assert len(calls) == 2
diff --git a/studio/backend/tests/test_diffusion_step_skip_defaults.py b/studio/backend/tests/test_diffusion_step_skip_defaults.py
index bd07b62849c..b97030ef096 100644
--- a/studio/backend/tests/test_diffusion_step_skip_defaults.py
+++ b/studio/backend/tests/test_diffusion_step_skip_defaults.py
@@ -47,7 +47,7 @@ DEV = "black-forest-labs/FLUX.1-dev"
@pytest.mark.parametrize(
"ids, tier, steps, want",
[
- ((KREA,), "default", 28, {"every": 2, "min_steps": 20}),
+ ((KREA,), "default", 28, None),
((KREA,), "max", 28, {"every": 3, "min_steps": 20}),
((KREA,), "eager", 28, None),
((KREA,), "off", 28, None),
@@ -56,11 +56,12 @@ DEV = "black-forest-labs/FLUX.1-dev"
((DEV,), "default", 28, None), # max-only row
((DEV,), "max", 28, {"every": 2, "min_steps": 20}),
# The unsloth mirror and a local path with the upstream as its base both resolve to the upstream row.
- (("unsloth/FLUX.1-Krea-dev",), "default", 28, {"every": 2, "min_steps": 20}),
- (("/models/my-krea", KREA), "default", 28, {"every": 2, "min_steps": 20}),
+ (("unsloth/FLUX.1-Krea-dev",), "max", 28, {"every": 3, "min_steps": 20}),
+ (("/models/my-krea", KREA), "max", 28, {"every": 3, "min_steps": 20}),
+ (("unsloth/FLUX.1-Krea-dev",), "default", 28, None),
(("black-forest-labs/FLUX.1-schnell",), "max", 28, None),
((None,), "max", 28, None),
- (KREA, "default", 28, {"every": 2, "min_steps": 20}), # a bare string
+ (KREA, "max", 28, {"every": 3, "min_steps": 20}), # a bare string
],
)
def test_plan_follows_the_tier_and_the_default_steps(table, ids, tier, steps, want):
@@ -235,10 +236,10 @@ def _probe_plan(entry):
@pytest.mark.parametrize(
"speed, killed, want_static",
[
- ("default", False, True),
+ ("default", False, False),
("max", False, True),
("eager", False, False),
- ("default", True, False),
+ ("max", True, False),
],
)
def test_auto_load_installs_static_for_a_listed_model(
@@ -345,11 +346,7 @@ def test_auto_skip_runs_on_txt2img_only_explicit_everywhere(
backend.unload()
-def test_deferred_default_tier_installs_the_auto_skip_on_the_third_image(
- fake_runtime, tmp_path, monkeypatch
-):
- # Speed unset (the UI default): a dense load stays eager, then the 3rd image engages `default`, and with it the
- # model's default-tier skip. The first two renders stay full-step.
+def test_deferred_default_tier_never_auto_skips(fake_runtime, tmp_path, monkeypatch):
from core.inference import diffusion as dmod
from .test_diffusion_backend import DiffusionBackend
@@ -383,9 +380,9 @@ def test_deferred_default_tier_installs_the_auto_skip_on_the_third_image(
backend.generate(prompt = "three")
st = backend.status()
assert st["speed_mode"] == "default"
- assert len(installs) == 1 and installs[0]["every"] == 2 and installs[0]["auto"] is True
- assert st["transformer_cache"] == "static"
- assert st["resolved"]["transformer_cache"]["value"] == "static"
+ assert installs == []
+ assert st["transformer_cache"] is None
+ assert st["resolved"]["transformer_cache"]["value"] != "static"
backend.unload()
@@ -492,7 +489,7 @@ def test_a_failed_deferred_profile_takes_the_auto_skip_back_off(
backend.generate(prompt = p)
st = backend.status()
assert st["speed_mode"] == "off"
- assert st["transformer_cache"] is None and removed
+ assert st["transformer_cache"] is None and removed == []
assert st["resolved"]["transformer_cache"]["value"] != "static"
backend.unload()
@@ -537,7 +534,7 @@ def test_an_explicit_lossless_tier_never_auto_skips(requested, effective, want):
tier = dcache.skip_tier(requested, effective)
assert tier == want
plan = dcache.auto_static_skip_plan(("Qwen/Qwen-Image",), tier, 20, env = {})
- assert (plan is None) == (want in ("off", "eager"))
+ assert (plan is None) == (want != "max")
@pytest.mark.parametrize("raw, want", [("1", 3), ("abc", 3), ("", 3), ("4", 4)])
diff --git a/studio/backend/tests/test_video_static_skip.py b/studio/backend/tests/test_video_static_skip.py
index 259b8d511ec..98d6bf7dedc 100644
--- a/studio/backend/tests/test_video_static_skip.py
+++ b/studio/backend/tests/test_video_static_skip.py
@@ -213,33 +213,49 @@ def test_fbcache_load_still_reports_an_active_cache(loop_runtime, monkeypatch):
backend.unload()
[email protected]("request_cache", [None, "auto"])
+def test_default_speed_never_auto_skips_on_wan_5b(loop_runtime, monkeypatch, request_cache):
+ monkeypatch.delenv(dcache.ENV_AUTO_STEP_SKIP, raising = False)
+ backend = VideoBackend()
+ status = backend.load_pipeline(
+ WAN_5B, model_kind = "pipeline", speed_mode = "default", transformer_cache = request_cache
+ )
+ assert status["transformer_cache"] is None
+ assert "forward" not in loop_runtime["pipe"].transformer.__dict__
+ backend.generate(prompt = "a sloth", steps = STEPS)
+ assert loop_runtime["pipe"].transformer.computed == 2 * STEPS
+ backend.unload()
+
+
@pytest.mark.parametrize("request_cache", [None, "auto"])
def test_auto_installs_the_measured_static_skip_on_wan_5b(loop_runtime, monkeypatch, request_cache):
monkeypatch.delenv(dcache.ENV_AUTO_STEP_SKIP, raising = False)
seen = _record_speed(monkeypatch)
backend = VideoBackend()
status = backend.load_pipeline(
- WAN_5B, model_kind = "pipeline", speed_mode = "default", transformer_cache = request_cache
+ WAN_5B, model_kind = "pipeline", speed_mode = "max", transformer_cache = request_cache
)
assert status["transformer_cache"] == "static"
entry = status["resolved"]["transformer_cache"]
assert entry["value"] == "static" and entry["requested"] is None
assert "UNSLOTH_DIFFUSION_AUTO_STEP_SKIP" in entry["reason"]
layer = loop_runtime["pipe"].transformer.__dict__["forward"]
- # Measured at 50 steps, so a shorter render computes every step.
- assert (layer.every, layer.auto, layer.min_steps) == (2, True, 50)
+ assert (layer.every, layer.auto, layer.min_steps) == (3, True, 50)
# Kept graph decisions, and the generate-time FBCache toggle is never armed over it.
assert [kw["cache_active"] for kw in seen] == [False]
assert backend._state.cache_auto is False
backend.generate(prompt = "a sloth", steps = STEPS)
- assert loop_runtime["pipe"].transformer.computed == 2 * (STEPS - SKIPPED_PER_BRANCH)
+ skipped = len(ss.static_schedule(STEPS, every = 3, min_steps = 50)) - sum(
+ ss.static_schedule(STEPS, every = 3, min_steps = 50)
+ )
+ assert loop_runtime["pipe"].transformer.computed == 2 * (STEPS - skipped)
backend.unload()
def test_auto_skip_leaves_a_keyframe_clip_unskipped(loop_runtime, monkeypatch):
monkeypatch.delenv(dcache.ENV_AUTO_STEP_SKIP, raising = False)
backend = VideoBackend()
- backend.load_pipeline(WAN_5B, model_kind = "pipeline", speed_mode = "default")
+ backend.load_pipeline(WAN_5B, model_kind = "pipeline", speed_mode = "max")
real = VideoBackend._resolve_keyframes
def with_first_frame(*args, **kwargs):
@@ -462,7 +478,11 @@ def test_hv15_counts_branches_by_context_without_a_step_callback(loop_runtime, m
backend.unload()
-def _load_h3(backend, transformer_cache):
+def _load_h3(
+ backend,
+ transformer_cache,
+ speed_mode = None,
+):
from core.inference.video import _detect_load_family
diffusers = sys.modules["diffusers"]
@@ -483,6 +503,7 @@ def _load_h3(backend, transformer_cache):
_load_token = None,
_base_local_dir = None,
transformer_cache = transformer_cache,
+ speed_mode = speed_mode,
)
@@ -584,6 +605,11 @@ def test_modular_workflow_auto_follows_the_table_and_the_kill_switch(h3_runtime,
)
backend = VideoBackend()
status = _load_h3(backend, None)
+ assert status["transformer_cache"] is None
+ assert "forward" not in h3_runtime.instance.transformer.__dict__
+ backend.unload()
+ backend = VideoBackend()
+ status = _load_h3(backend, None, speed_mode = "max")
entry = status["resolved"]["transformer_cache"]
assert status["transformer_cache"] == "static"
assert entry["requested"] is None and entry["value"] == "static"
@@ -597,7 +623,7 @@ def test_modular_workflow_auto_follows_the_table_and_the_kill_switch(h3_runtime,
monkeypatch.setenv(dcache.ENV_AUTO_STEP_SKIP, "0")
backend = VideoBackend()
- status = _load_h3(backend, None)
+ status = _load_h3(backend, None, speed_mode = "max")
assert status["transformer_cache"] is None
assert "forward" not in h3_runtime.instance.transformer.__dict__
backend.unload()