Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: fp32 VAE upcasts keep the offloaded VAE's pinned host weights (SDXL-class force_upcast VAEs) (#14946)
Daniel Han30e76889e225
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 8f56e3cfe33..ea7cbab442d 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -270,6 +270,7 @@ from .diffusion_speed import (snapshot_backend_flags,)from .diffusion_vae_fp16 import enable_fp16_vae_decode+from .diffusion_vae_upcast import pipeline_upcasts_keep_storagefrom .diffusion_vae_tiling import install as install_wide_vae_tilesfrom .diffusion_attention import (apply_attention_backend,@@ -13412,10 +13413,13 @@ class DiffusionBackend:pipe, fam, sd_controls, chunk_kwargs, guidance = guidance):weighting_used[0] = True- with sd_controls_mod.variation_latents(- pipe,- chunk_variation,- sd_controls.variation_strength if sd_controls is not None else None,+ with (+ sd_controls_mod.variation_latents(+ pipe,+ chunk_variation,+ sd_controls.variation_strength if sd_controls is not None else None,+ ),+ pipeline_upcasts_keep_storage(pipe),):return pipe(**chunk_kwargs).imagesdiff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 01baa29f897..a27622813c1 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -122,6 +122,8 @@ def _resolve_host_memory_reclaimer() -> Optional[Callable[[], None]]:OFFLOAD_KEEP_CPU_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_KEEP_CPU"_KEEP_ATTR = "_unsloth_offload_keep_cpu"+# diffusion_vae_upcast.DTYPE_ONLY_TO_ATTR+_DTYPE_ONLY_TO_ATTR = "_unsloth_dtype_only_to"def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:@@ -757,8 +759,10 @@ def _wrap_cpu_offload_hook(type(mod).__name__,)layout = state.get("pin_layout") if onload and _fast_upload_enabled() else None- if layout and "to" in mod.__dict__:- # An instance-level mover is already installed (stable arena, pinned swap): defer to it.+ prior_to = mod.__dict__.get("to")+ if layout and prior_to is not None and not getattr(prior_to, _DTYPE_ONLY_TO_ATTR, False):+ # An instance-level mover is already installed (stable arena, pinned swap): defer to it. A dtype-only+ # one (diffusion_vae_upcast) passes device moves through, so the pinned chunks still go up first.layout = Noneif not layout:out = pre_forward(mod, *args, **kwargs)@@ -778,8 +782,14 @@ def _wrap_cpu_offload_hook(# The stock pre_forward offloads the previous component, then calls module.to(device) only if the module# is not there yet. Shadow that one call so the pinned chunks go up first; module.to then moves whatever# the chunks did not cover (unpinned or replaced weights) exactly as before.+ def _put_back_to() -> None:+ if prior_to is None:+ mod.__dict__.pop("to", None)+ else:+ mod.__dict__["to"] = prior_to+def _to(*to_args: Any, **to_kwargs: Any) -> Any:- mod.__dict__.pop("to", None)+ _put_back_to()import torchtarget = to_args[0] if to_args else to_kwargs.get("device")@@ -799,7 +809,7 @@ def _wrap_cpu_offload_hook(try:out = pre_forward(mod, *args, **kwargs)finally:- mod.__dict__.pop("to", None)+ _put_back_to()if onload:for name, p in mod.named_parameters():if name in host and owner.get(name) is p and p.device.type != "cpu":diff --git a/studio/backend/core/inference/diffusion_vae_fp16.py b/studio/backend/core/inference/diffusion_vae_fp16.pyindex 0706915003e..da0674d3c24 100644--- a/studio/backend/core/inference/diffusion_vae_fp16.py+++ b/studio/backend/core/inference/diffusion_vae_fp16.py@@ -15,6 +15,8 @@ import functoolsimport typesfrom typing import Any, Optional+from .diffusion_vae_upcast import fp32_keeping_storage+_SCALE_LOG2 = 8@@ -116,13 +118,9 @@ def enable_fp16_vae_decode(slot = types.SimpleNamespace(decode = decode)def _fp32_call(fn: Any, parts: list, x: Any, *args: Any, **kwargs: Any) -> Any:- for part in parts:- part.to(torch.float32)- try:+ # Leaves the stored fp16 weights in place, so they stay pinned under model offload.+ with fp32_keeping_storage(vae, parts):return fn(x.to(torch.float32), *args, **kwargs)- finally:- for part in parts:- part.to(torch.float16)@functools.wraps(decode)def fp16_decode(z: Any, *args: Any, **kwargs: Any) -> Any:diff --git a/studio/backend/core/inference/diffusion_vae_upcast.py b/studio/backend/core/inference/diffusion_vae_upcast.pynew file mode 100644index 00000000000..8335fbb932f--- /dev/null+++ b/studio/backend/core/inference/diffusion_vae_upcast.py@@ -0,0 +1,178 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""fp32 VAE runs that leave a 16-bit VAE's stored weights in place."""++from __future__ import annotations++import contextlib+from typing import Any, Iterable, Optional++# Marks the instance-level ``to`` below; the kept offload hook's pinned onload passes device moves through it.+DTYPE_ONLY_TO_ATTR = "_unsloth_dtype_only_to"+_KEEP_ATTR = "_unsloth_offload_keep_cpu"+++def _first_param(module: Any) -> Any:+ try:+ return next(iter(module.parameters()), None)+ except Exception: # noqa: BLE001 - duck-typed module+ return None+++def _onload_through_kept_hook(owner: Any) -> None:+ """Onload via the kept hook (pinned chunks) so the upcast happens on device, not on the host weights."""+ hook = getattr(owner, "_hf_hook", None)+ if not getattr(hook, _KEEP_ATTR, False) or not callable(getattr(hook, "pre_forward", None)):+ return+ first = _first_param(owner)+ if first is None or first.device.type != "cpu":+ return+ try:+ hook.pre_forward(owner)+ except Exception: # noqa: BLE001 - the forward's own hook onloads it as before+ pass+++def _float_tensors(parts: Iterable[Any], dtype: Any) -> tuple[list, list]:+ """``(module, name)`` of every floating parameter and buffer not already in ``dtype``. Keyed by registration, not+ object: sequential offload's leaf hooks register a new Parameter on every onload and offload."""+ params, buffers, seen = [], [], set()+ for part in parts:+ for module in part.modules():+ for name, p in list(getattr(module, "_parameters", {}).items()):+ if (+ p is not None+ and p.is_floating_point()+ and p.dtype != dtype+ and id(p) not in seen+ ):+ seen.add(id(p))+ params.append((module, name))+ for name, b in list(getattr(module, "_buffers", {}).items()):+ if b is not None and b.is_floating_point() and b.dtype != dtype:+ buffers.append((module, name))+ return params, buffers+++def upcast_keeping_storage(+ owner: Any,+ parts: Optional[list] = None,+ dtype: Any = None,+) -> list:+ import torch++ dtype = torch.float32 if dtype is None else dtype+ parts = [owner] if parts is None else [p for p in parts if p is not None]+ _onload_through_kept_hook(owner)+ params, buffers = _float_tensors(parts, dtype)+ saved = [("p", m, name, m._parameters[name].data) for m, name in params]+ saved += [("b", m, name, m._buffers[name]) for m, name in buffers]+ for part in parts:+ # The class method: an instance-level ``to`` (ours, an offload arena's) only handles device moves.+ type(part).to(part, dtype = dtype)+ token: list = []+ for kind, m, name, orig in saved:+ cast = (m._parameters if kind == "p" else m._buffers).get(name)+ token.append((kind, m, name, orig, cast.data_ptr() if cast is not None else None))+ return token+++def _restorable(current: Any, orig: Any, cast_ptr: Any) -> bool:+ """``current`` holds ``orig``'s values (diffusers < 0.37 ``upcast_vae`` casts some submodules back)."""+ import torch++ if current.device != orig.device or current.shape != orig.shape:+ return False+ if current.dtype != orig.dtype:+ return current.data_ptr() == cast_ptr+ if current.device.type == "meta" or current.data_ptr() == orig.data_ptr():+ return current.device.type == "meta"+ return bool(torch.equal(current, orig))+++def restore_storage(token: list) -> None:+ """Put back the original tensors on the current registrations; one that has since moved or been changed is cast+ back to its load dtype instead, as the stock cast does."""+ for kind, m, name, orig, cast_ptr in token:+ try:+ table = m._parameters if kind == "p" else m._buffers+ current = table.get(name)+ if current is None:+ continue+ data = current.data if kind == "p" else current+ if _restorable(data, orig, cast_ptr):+ if kind == "p":+ current.data = orig+ else:+ table[name] = orig+ elif data.dtype != orig.dtype:+ if kind == "p":+ current.data = data.to(orig.dtype)+ else:+ table[name] = data.to(orig.dtype)+ except Exception: # noqa: BLE001 - best effort; leaves that tensor as the cast made it+ pass+ token.clear()+++def fp32_keeping_storage(owner: Any, parts: Optional[list] = None):+ token = upcast_keeping_storage(owner, parts)+ try:+ yield+ finally:+ restore_storage(token)+++def _is_16bit(dtype: Any) -> bool:+ import torch+ return dtype in (torch.float16, torch.bfloat16)+++def pipeline_upcasts_keep_storage(pipe: Any):+ """Route diffusers' fp32 / load-dtype round trip through ``upcast_keeping_storage``; other ``to`` calls run as is."""+ vae = getattr(pipe, "vae", None)+ first = _first_param(vae) if vae is not None else None+ config = getattr(vae, "config", None)+ if (+ first is None+ or not _is_16bit(first.dtype)+ or not getattr(config, "force_upcast", False)+ or "to" in getattr(vae, "__dict__", {})+ ):+ yield+ return+ import torch++ load_dtype = first.dtype+ stock_to = type(vae).to+ state: dict = {"token": None}++ def to(*args: Any, **kwargs: Any) -> Any:+ try:+ device, dtype, _non_blocking, fmt = torch._C._nn._parse_to(*args, **kwargs)+ except Exception: # noqa: BLE001 - not a tensor-style call: the stock method reports it+ return stock_to(vae, *args, **kwargs)+ if device is None and fmt is None and dtype is not None:+ current = getattr(_first_param(vae), "dtype", None)+ if dtype == torch.float32 and state["token"] is None and current == load_dtype:+ state["token"] = upcast_keeping_storage(vae)+ return vae+ if dtype == load_dtype and state["token"] is not None:+ token, state["token"] = state["token"], None+ restore_storage(token)+ return vae+ return stock_to(vae, *args, **kwargs)++ setattr(to, DTYPE_ONLY_TO_ATTR, True)+ vae.__dict__["to"] = to+ try:+ yield+ finally:+ if vae.__dict__.get("to") is to:+ vae.__dict__.pop("to", None)+ if state["token"] is not None:+ # A raise between the upcast and the cast back: leave the VAE in its load dtype, as the next call expects.+ restore_storage(state["token"])diff --git a/studio/backend/tests/test_diffusion_vae_upcast_storage.py b/studio/backend/tests/test_diffusion_vae_upcast_storage.pynew file mode 100644index 00000000000..7410f80d085--- /dev/null+++ b/studio/backend/tests/test_diffusion_vae_upcast_storage.py@@ -0,0 +1,257 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""An fp32 run of a force_upcast 16-bit VAE keeps its stored (pinned) tensors and matches stock fp32 output."""++from __future__ import annotations++import contextlib+import copy+import types++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")++from core.inference import diffusion_vae_fp16 as vf # noqa: E402++CUDA = types.SimpleNamespace(device = "cuda")+++def _vu():+ # Imported per test, so the behavioural tests below also run (and fail) against a tree without the helper.+ from core.inference import diffusion_vae_upcast+ return diffusion_vae_upcast+++def _tiny_vae(dtype = torch.float16):+ torch.manual_seed(0)+ return (+ diffusers.AutoencoderKL(+ in_channels = 3,+ out_channels = 3,+ down_block_types = ("DownEncoderBlock2D",) * 3,+ up_block_types = ("UpDecoderBlock2D",) * 3,+ block_out_channels = (16, 32, 32),+ layers_per_block = 1,+ latent_channels = 4,+ norm_num_groups = 8,+ force_upcast = True,+ )+ .eval()+ .to(dtype)+ )+++def _storage(module):+ return {name: (p.data_ptr(), p.dtype) for name, p in module.named_parameters()}+++def _stock_fp32(vae):+ ref = copy.deepcopy(vae)+ ref.to(torch.float32)+ return ref+++def test_round_trip_puts_back_the_same_tensors():+ vu = _vu()+ vae = _tiny_vae()+ before = _storage(vae)+ with vu.fp32_keeping_storage(vae):+ assert all(p.dtype == torch.float32 for p in vae.parameters())+ assert _storage(vae) == before+++def test_parts_only_upcast_the_named_submodules():+ vu = _vu()+ vae = _tiny_vae()+ before = _storage(vae)+ with vu.fp32_keeping_storage(vae, [vae.encoder, None]):+ assert next(vae.encoder.parameters()).dtype == torch.float32+ assert next(vae.decoder.parameters()).dtype == torch.float16+ assert _storage(vae) == before+++def test_a_tensor_replaced_meanwhile_is_cast_back_not_overwritten():+ vu = _vu()+ vae = _tiny_vae()+ token = vu.upcast_keeping_storage(vae)+ fresh = torch.zeros_like(vae.decoder.conv_out.weight.data)+ vae.decoder.conv_out.weight.data = fresh+ vu.restore_storage(token)+ w = vae.decoder.conv_out.weight+ assert w.dtype == torch.float16 and bool((w == 0).all())+++def test_pipeline_round_trip_is_bit_identical_and_keeps_storage():+ vu = _vu()+ # diffusers' SDXL decode: vae.to(dtype=fp32), decode, vae.to(dtype=fp16).+ vae = _tiny_vae()+ ref = _stock_fp32(vae)+ z = torch.randn(1, 4, 8, 8)+ before = _storage(vae)+ pipe = types.SimpleNamespace(vae = vae)+ with vu.pipeline_upcasts_keep_storage(pipe):+ assert getattr(vae.__dict__["to"], vu.DTYPE_ONLY_TO_ATTR)+ assert vae.to(dtype = torch.float32) is vae+ assert vae.dtype == torch.float32+ out = vae.decode(z).sample+ vae.to(dtype = torch.float16)+ assert vae.to("cpu") is vae+ assert "to" not in vae.__dict__+ assert _storage(vae) == before+ assert torch.equal(out, ref.decode(z).sample)+++def test_pipeline_raise_mid_upcast_leaves_the_load_dtype():+ vu = _vu()+ vae = _tiny_vae()+ before = _storage(vae)+ with contextlib.suppress(RuntimeError):+ with vu.pipeline_upcasts_keep_storage(types.SimpleNamespace(vae = vae)):+ vae.to(torch.float32)+ raise RuntimeError("decode failed")+ assert _storage(vae) == before++[email protected]("dtype,force_upcast", [(torch.float32, True), (torch.float16, False)])+def test_pipeline_wrapper_is_inert_off_the_upcast_case(dtype, force_upcast):+ vu = _vu()+ vae = _tiny_vae(dtype)+ vae.register_to_config(force_upcast = force_upcast)+ with vu.pipeline_upcasts_keep_storage(types.SimpleNamespace(vae = vae)):+ assert "to" not in vae.__dict__+++def test_fp16_decode_fp32_encode_keeps_encoder_storage():+ vae = _tiny_vae()+ ref = _stock_fp32(vae)+ assert vf.enable_fp16_vae_decode(+ types.SimpleNamespace(vae = vae, unet = type("UNet2DConditionModel", (), {})()), CUDA+ )+ x = torch.randn(1, 3, 32, 32, dtype = torch.float16)+ before = _storage(vae)+ got = vae.encode(x).latent_dist.mean+ assert _storage(vae) == before+ assert torch.equal(got, ref.encode(x.float()).latent_dist.mean)++[email protected](scope = "module")+def tiny_sdxl_path():+ """Resolved at module scope, before the per-test empty hub cache and the outbound-network block apply."""+ import sys+ from pathlib import Path++ tests_dir = str(Path(__file__).resolve().parent)+ if tests_dir not in sys.path:+ sys.path.insert(0, tests_dir)+ from conftest import allow_outbound+ from huggingface_hub import snapshot_download++ repo = "hf-internal-testing/tiny-stable-diffusion-xl-pipe"+ try:+ return snapshot_download(repo, local_files_only = True)+ except Exception: # noqa: BLE001 - not cached yet+ try:+ with allow_outbound():+ return snapshot_download(repo)+ except Exception as exc: # noqa: BLE001+ pytest.skip(f"tiny pipeline not reachable: {exc}")+++def _cuda_or_skip():+ if not torch.cuda.is_available():+ pytest.skip("CUDA only")+ pytest.importorskip("accelerate")+++def test_sdxl_model_offload_vae_stays_pinned_cuda(monkeypatch, tiny_sdxl_path):+ """Stock: the decode's upcast lands before the VAE's first onload, so it is never pinned and every onload copies+ every tensor pageable. Here it is pinned after the first render and stays so; output bit-identical."""+ from core.inference import diffusion_memory as dm++ _cuda_or_skip()+ path = tiny_sdxl_path+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ outs = {}+ for arm in ("stock", "kept"):+ pipe = diffusers.StableDiffusionXLPipeline.from_pretrained(path, torch_dtype = torch.float16)+ pipe.vae.register_to_config(force_upcast = True)+ pipe.enable_model_cpu_offload(device = "cuda:0")+ dm.keep_cpu_weights_on_offload(pipe)+ wrap = (+ _vu().pipeline_upcasts_keep_storage+ if arm == "kept"+ else lambda p: contextlib.nullcontext()+ )+ renders = []+ for _ in range(2):+ with wrap(pipe):+ renders.append(+ pipe(+ "a cat",+ num_inference_steps = 2,+ height = 64,+ width = 64,+ generator = torch.Generator("cpu").manual_seed(0),+ output_type = "pt",+ ).images+ )+ params = list(pipe.vae.parameters())+ assert all(p.dtype == torch.float16 and p.device.type == "cpu" for p in params)+ pinned = sum(p.is_pinned() for p in params)+ if arm == "kept":+ assert pinned == len(params)+ assert pipe.vae.__dict__[dm._KEEP_ATTR].get("pin_layout")+ outs[arm] = renders+ assert torch.equal(outs["stock"][0], outs["kept"][0])+ assert torch.equal(outs["kept"][0], outs["kept"][1])+++def test_nested_load_dtype_casts_still_restore_the_original_tensors():+ # diffusers < 0.37 upcast_vae(): whole VAE to fp32, then these submodules back to the load dtype.+ vu = _vu()+ vae = _tiny_vae()+ before = _storage(vae)+ with vu.pipeline_upcasts_keep_storage(types.SimpleNamespace(vae = vae)):+ vae.to(dtype = torch.float32)+ for module in (vae.post_quant_conv, vae.decoder.conv_in, vae.decoder.mid_block):+ module.to(torch.float16)+ vae.to(dtype = torch.float16)+ assert _storage(vae) == before+++def test_a_changed_same_dtype_replacement_is_left_alone():+ vu = _vu()+ vae = _tiny_vae()+ token = vu.upcast_keeping_storage(vae)+ vae.decoder.conv_out.to(torch.float16)+ with torch.no_grad():+ vae.decoder.conv_out.weight.zero_()+ vu.restore_storage(token)+ assert bool((vae.decoder.conv_out.weight == 0).all())++[email protected]("route", ["helper", "pipeline"])+def test_sequential_offload_replacement_parameters_return_to_the_load_dtype(route):+ # accelerate's leaf hooks register new Parameter objects on every onload / offload.+ pytest.importorskip("accelerate")+ from accelerate import cpu_offload++ vu = _vu()+ vae = _tiny_vae()+ ref = _stock_fp32(vae)+ cpu_offload(vae, execution_device = torch.device("cpu"))+ assert next(vae.parameters()).device.type == "meta"+ x = torch.randn(1, 3, 32, 32, dtype = torch.float16)+ if route == "helper":+ with vu.fp32_keeping_storage(vae):+ got = vae.encode(x.float()).latent_dist.mean+ else:+ with vu.pipeline_upcasts_keep_storage(types.SimpleNamespace(vae = vae)):+ vae.to(dtype = torch.float32)+ got = vae.encode(x.float()).latent_dist.mean+ vae.to(dtype = torch.float16)+ assert {p.dtype for p in vae.parameters()} == {torch.float16}+ assert torch.equal(got, ref.encode(x.float()).latent_dist.mean)