Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep an eagerly decoded image VAE contiguous on NVIDIA (#12035)
Daniel Han186b486d1d06
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex b8856c19712..7acaabbf63a 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -447,6 +447,19 @@ def apply_speed_optims(eager_when_tiled = _vae_eager_when_tiled(pipe),)+ if applied["channels_last"] and not _channels_last_decode_wins(+ pipe, target, applied["compiled_vae_decode"], offload_active+ ):+ applied["channels_last"] = not _vae_contiguous(pipe, logger)+ elif applied["compiled_vae_decode"] and not _channels_last_decode_wins(+ pipe, target, False, offload_active+ ):+ # A decode compile that fails at first call falls back eager: relayout then.+ try:+ pipe.vae._unsloth_eager_contiguous = True+ except Exception: # noqa: BLE001+ pass+if mode == SPEED_MAX:if on_cuda:applied["tf32"] = _enable_tf32(logger)@@ -1040,6 +1053,46 @@ def _vae_eager_when_tiled(pipe: Any) -> bool:return os.environ.get(COMPILE_VAE_ENV, "").strip().lower() not in _VAE_TRUE_TOKENS+def _vae_contiguous(pipe: Any, logger: Any) -> bool:+ try:+ import torch+ pipe.vae.to(memory_format = torch.contiguous_format)+ return True+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "contiguous vae", exc)+ return False+++# VAE classes whose decode layout was measured; any other keeps channels_last.+_VAE_LAYOUT_MEASURED: frozenset[str] = frozenset({"AutoencoderKL", "AutoencoderKLFlux2"})+++def _channels_last_decode_wins(+ pipe: Any,+ target: Any,+ compiled_decode: bool,+ offload_active: bool = False,+) -> bool:+ """NVIDIA-measured: eager 16-bit is faster contiguous (no NHWC GroupNorm) but offload keeps channels_last (lower peak); compiled 16-bit wins channels_last; fp32 wins contiguous."""+ vae = getattr(pipe, "vae", None)+ if (+ getattr(target, "backend", "cuda") != "cuda"+ or getattr(target, "device", None) != "cuda"+ or type(vae).__name__ not in _VAE_LAYOUT_MEASURED+ ):+ return True+ config = getattr(vae, "config", None)+ # SDXL pipelines upcast a force_upcast fp16 VAE to fp32 for the decode.+ fp32_decode = str(getattr(vae, "dtype", "")).endswith("float32") or (+ bool(getattr(config, "force_upcast", False))+ and _is_float16(getattr(vae, "dtype", None))+ and _denoiser_unet(pipe) is not None+ )+ if fp32_decode:+ return False+ return compiled_decode or offload_active++def _guard_compiled_decode(vae: Any,compiled: Any,@@ -1079,6 +1132,12 @@ def _guard_compiled_decode(del owner.decodeexcept Exception: # noqa: BLE001 - `failed` still routes this wrapper to eagerpass+ if getattr(vae, "_unsloth_eager_contiguous", False):+ try:+ import torch+ vae.to(memory_format = torch.contiguous_format)+ except Exception: # noqa: BLE001 - optimisation only+ passif logger is not None:logger.warning("diffusion.speed: torch.compile failed on the VAE decode (%s); decoding eager",diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex ffacf8beb17..4784cd6e9d6 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -80,7 +80,9 @@ def _compile_runtime_independent_of_the_host(monkeypatch):def _stub_torch(monkeypatch):torch = types.ModuleType("torch")torch.bfloat16 = "bfloat16" # _is_bfloat16 compares by identity then str fallback+ torch.float16 = "float16"torch.channels_last = "channels_last"+ torch.contiguous_format = "contiguous_format"torch.backends = types.SimpleNamespace(cuda = types.SimpleNamespace(matmul = types.SimpleNamespace(allow_tf32 = False)),cudnn = types.SimpleNamespace(allow_tf32 = False, benchmark = False),@@ -451,7 +453,7 @@ def test_speed_default_dense_falls_back_to_regional_compile(monkeypatch):applied = apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_DEFAULT)- assert applied["channels_last"] is True and pipe.vae.mem_format == torch.channels_last+ assert applied["channels_last"] is False and pipe.vae.mem_format == torch.contiguous_formatassert applied["compiled"] is True and pipe.compiled is True# default compiles with dynamic=True and no autotune mode: fast cold start, resolution-robust, sidesteps the CUDA-graph crash.assert pipe.compile_kwargs == {"fullgraph": True, "dynamic": True}@@ -487,7 +489,7 @@ def test_speed_default_gguf_compiles_only_dequant(monkeypatch):applied = apply_speed_optims(pipe, _target(), is_gguf = True, family = _family(), speed_mode = SPEED_DEFAULT)- assert applied["channels_last"] is True+ assert applied["channels_last"] is Falseassert applied["compiled_dequant"] is True# The transformer block is NOT regionally compiled under GGUF default.assert applied["compiled"] is False and pipe.compiled is False@@ -610,7 +612,7 @@ class _UNetPipe:def __init__(self, unet = None):self.mem_format = Noneself.fused = False- self.vae = types.SimpleNamespace(to = self._vae_to, decode = lambda z: z)+ self.vae = AutoencoderKL(to = self._vae_to, decode = lambda z: z)self.unet = UNet2DConditionModel() if unet is None else unetdef _vae_to(self, *, memory_format):@@ -662,6 +664,103 @@ def test_dit_default_tier_keeps_fuse_off_and_leaves_the_vae_decode_eager(monkeypassert ds_mod.vae_decode_compile_allowed(pipe, SPEED_MAX) is True[email protected]("tier", [SPEED_EAGER, SPEED_DEFAULT])+def test_eager_vae_decode_keeps_contiguous_weights(monkeypatch, tier):+ torch = _stub_torch(monkeypatch)+ monkeypatch.delenv(ds_mod.COMPILE_VAE_ENV, raising = False)+ pipe = _Pipe(with_compile = True)+ applied = apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = tier)+ assert applied["compiled_vae_decode"] is False+ assert applied["channels_last"] is False and pipe.vae.mem_format == torch.contiguous_format+++ "backend, device", [("rocm", "cuda"), ("mps", "mps"), ("xpu", "xpu"), ("cpu", "cpu")]+)+def test_eager_vae_decode_layout_unchanged_off_nvidia(monkeypatch, backend, device):+ torch = _stub_torch(monkeypatch)+ pipe = _Pipe(with_compile = True)+ target = _target(device = device)+ target.backend = backend+ applied = apply_speed_optims(+ pipe, target, is_gguf = False, family = _family(), speed_mode = SPEED_EAGER+ )+ assert applied["channels_last"] is True and pipe.vae.mem_format == torch.channels_last+++ "dtype, expect_channels_last",+ [("torch.bfloat16", True), ("float16", True), ("torch.float32", False)],+)+def test_offloaded_eager_decode_keeps_channels_last_unless_fp32(+ monkeypatch, dtype, expect_channels_last+):+ # An eager 16-bit contiguous decode peaks ~0.63 GiB/MP higher; an fp32 one is smaller contiguous.+ torch = _stub_torch(monkeypatch)+ pipe = _Pipe(with_compile = True)+ pipe.vae.dtype = dtype+ applied = apply_speed_optims(+ pipe,+ _target(),+ is_gguf = False,+ family = _family(),+ speed_mode = SPEED_EAGER,+ offload_active = True,+ )+ assert applied["channels_last"] is expect_channels_last+ assert pipe.vae.mem_format == (+ torch.channels_last if expect_channels_last else torch.contiguous_format+ )+++def test_unmeasured_vae_class_keeps_channels_last(monkeypatch):+ torch = _stub_torch(monkeypatch)+ pipe = _Pipe(with_compile = True, vae_cls = type("HYVAE2D", (types.SimpleNamespace,), {}))+ applied = apply_speed_optims(+ pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_EAGER+ )+ assert applied["channels_last"] is True and pipe.vae.mem_format == torch.channels_last+++def test_compiled_vae_decode_keeps_channels_last(monkeypatch):+ torch = _stub_torch(monkeypatch)+ monkeypatch.delenv(ds_mod.COMPILE_VAE_ENV, raising = False)+ pipe = _Pipe(with_compile = True)+ pipe.vae.dtype = "torch.bfloat16"+ applied = apply_speed_optims(+ pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_MAX+ )+ assert applied["compiled_vae_decode"] is True+ assert applied["channels_last"] is True and pipe.vae.mem_format == torch.channels_last+++ "dtype, force_upcast, expect_channels_last",+ [+ ("torch.float32", False, False),+ ("float16", True, False),+ ("float16", False, True),+ ("bfloat16", True, True),+ ],+)+def test_unet_compiled_decode_in_fp32_keeps_contiguous_weights(+ monkeypatch, dtype, force_upcast, expect_channels_last+):+ # SDXL pipelines decode a force_upcast fp16 VAE in fp32, where channels_last measured 0.89x even compiled (T4).+ torch = _stub_torch(monkeypatch)+ pipe = _UNetPipe()+ pipe.vae.dtype = dtype+ pipe.vae.config = types.SimpleNamespace(force_upcast = force_upcast)+ applied = apply_speed_optims(+ pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_DEFAULT+ )+ assert applied["compiled_vae_decode"] is True+ assert applied["channels_last"] is expect_channels_last+ assert pipe.mem_format == (+ torch.channels_last if expect_channels_last else torch.contiguous_format+ )++def test_dit_default_tier_vae_decode_compile_forced_on_by_env(monkeypatch):torch = _stub_torch(monkeypatch)monkeypatch.setenv(ds_mod.COMPILE_VAE_ENV, "1")@@ -1916,6 +2015,30 @@ def test_a_tiled_dit_decode_stays_eager_unless_the_compile_is_forced(monkeypatchassert calls["compiled"] == (2 if forced else 1)[email protected]("offload_active", [False, True])+def test_vae_decode_compile_fallback_relayouts_an_eager_16bit_decode(monkeypatch, offload_active):+ monkeypatch.delenv(ds_mod.COMPILE_VAE_ENV, raising = False)+ _stub_lazy_compile(monkeypatch, lambda: _BackendCompilerFailed("LoweringException"))+ torch = sys.modules["torch"]+ pipe = _Pipe(with_compile = True)+ pipe.vae.dtype = "torch.bfloat16"+ applied = apply_speed_optims(+ pipe,+ _target(),+ is_gguf = False,+ family = _family(),+ speed_mode = SPEED_MAX,+ offload_active = offload_active,+ )+ assert applied["compiled_vae_decode"] is True+ assert pipe.vae.mem_format == torch.channels_last+ assert pipe.vae.decode(1) == 1+ assert pipe.vae._unsloth_compiled_decode is False+ assert pipe.vae.mem_format == (+ torch.channels_last if offload_active else torch.contiguous_format+ )++@pytest.mark.parametrize("kind", ["runtime", "oom"])def test_vae_decode_non_compile_errors_are_not_swallowed(monkeypatch, kind):def failure():