Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: fp32 VAE upcasts keep the offloaded VAE's pinned host weights (SDXL-class force_upcast VAEs) (#14946)

Daniel Han30e76889e225
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 8f56e3cfe33..ea7cbab442d 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -270,6 +270,7 @@ from .diffusion_speed import (
snapshot_backend_flags,
)
from .diffusion_vae_fp16 import enable_fp16_vae_decode
+from .diffusion_vae_upcast import pipeline_upcasts_keep_storage
from .diffusion_vae_tiling import install as install_wide_vae_tiles
from .diffusion_attention import (
apply_attention_backend,
@@ -13412,10 +13413,13 @@ class DiffusionBackend:
pipe, fam, sd_controls, chunk_kwargs, guidance = guidance
):
weighting_used[0] = True
- with sd_controls_mod.variation_latents(
- pipe,
- chunk_variation,
- sd_controls.variation_strength if sd_controls is not None else None,
+ with (
+ sd_controls_mod.variation_latents(
+ pipe,
+ chunk_variation,
+ sd_controls.variation_strength if sd_controls is not None else None,
+ ),
+ pipeline_upcasts_keep_storage(pipe),
):
return pipe(**chunk_kwargs).images
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 01baa29f897..a27622813c1 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -122,6 +122,8 @@ def _resolve_host_memory_reclaimer() -> Optional[Callable[[], None]]:
OFFLOAD_KEEP_CPU_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_KEEP_CPU"
_KEEP_ATTR = "_unsloth_offload_keep_cpu"
+# diffusion_vae_upcast.DTYPE_ONLY_TO_ATTR
+_DTYPE_ONLY_TO_ATTR = "_unsloth_dtype_only_to"
def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:
@@ -757,8 +759,10 @@ def _wrap_cpu_offload_hook(
type(mod).__name__,
)
layout = state.get("pin_layout") if onload and _fast_upload_enabled() else None
- if layout and "to" in mod.__dict__:
- # An instance-level mover is already installed (stable arena, pinned swap): defer to it.
+ prior_to = mod.__dict__.get("to")
+ if layout and prior_to is not None and not getattr(prior_to, _DTYPE_ONLY_TO_ATTR, False):
+ # An instance-level mover is already installed (stable arena, pinned swap): defer to it. A dtype-only
+ # one (diffusion_vae_upcast) passes device moves through, so the pinned chunks still go up first.
layout = None
if not layout:
out = pre_forward(mod, *args, **kwargs)
@@ -778,8 +782,14 @@ def _wrap_cpu_offload_hook(
# The stock pre_forward offloads the previous component, then calls module.to(device) only if the module
# is not there yet. Shadow that one call so the pinned chunks go up first; module.to then moves whatever
# the chunks did not cover (unpinned or replaced weights) exactly as before.
+ def _put_back_to() -> None:
+ if prior_to is None:
+ mod.__dict__.pop("to", None)
+ else:
+ mod.__dict__["to"] = prior_to
+
def _to(*to_args: Any, **to_kwargs: Any) -> Any:
- mod.__dict__.pop("to", None)
+ _put_back_to()
import torch
target = to_args[0] if to_args else to_kwargs.get("device")
@@ -799,7 +809,7 @@ def _wrap_cpu_offload_hook(
try:
out = pre_forward(mod, *args, **kwargs)
finally:
- mod.__dict__.pop("to", None)
+ _put_back_to()
if onload:
for name, p in mod.named_parameters():
if name in host and owner.get(name) is p and p.device.type != "cpu":
diff --git a/studio/backend/core/inference/diffusion_vae_fp16.py b/studio/backend/core/inference/diffusion_vae_fp16.py
index 0706915003e..da0674d3c24 100644
--- a/studio/backend/core/inference/diffusion_vae_fp16.py
+++ b/studio/backend/core/inference/diffusion_vae_fp16.py
@@ -15,6 +15,8 @@ import functools
import types
from typing import Any, Optional
+from .diffusion_vae_upcast import fp32_keeping_storage
+
_SCALE_LOG2 = 8
@@ -116,13 +118,9 @@ def enable_fp16_vae_decode(
slot = types.SimpleNamespace(decode = decode)
def _fp32_call(fn: Any, parts: list, x: Any, *args: Any, **kwargs: Any) -> Any:
- for part in parts:
- part.to(torch.float32)
- try:
+ # Leaves the stored fp16 weights in place, so they stay pinned under model offload.
+ with fp32_keeping_storage(vae, parts):
return fn(x.to(torch.float32), *args, **kwargs)
- finally:
- for part in parts:
- part.to(torch.float16)
@functools.wraps(decode)
def fp16_decode(z: Any, *args: Any, **kwargs: Any) -> Any:
diff --git a/studio/backend/core/inference/diffusion_vae_upcast.py b/studio/backend/core/inference/diffusion_vae_upcast.py
new file mode 100644
index 00000000000..8335fbb932f
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_vae_upcast.py
@@ -0,0 +1,178 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""fp32 VAE runs that leave a 16-bit VAE's stored weights in place."""
+
+from __future__ import annotations
+
+import contextlib
+from typing import Any, Iterable, Optional
+
+# Marks the instance-level ``to`` below; the kept offload hook's pinned onload passes device moves through it.
+DTYPE_ONLY_TO_ATTR = "_unsloth_dtype_only_to"
+_KEEP_ATTR = "_unsloth_offload_keep_cpu"
+
+
+def _first_param(module: Any) -> Any:
+ try:
+ return next(iter(module.parameters()), None)
+ except Exception: # noqa: BLE001 - duck-typed module
+ return None
+
+
+def _onload_through_kept_hook(owner: Any) -> None:
+ """Onload via the kept hook (pinned chunks) so the upcast happens on device, not on the host weights."""
+ hook = getattr(owner, "_hf_hook", None)
+ if not getattr(hook, _KEEP_ATTR, False) or not callable(getattr(hook, "pre_forward", None)):
+ return
+ first = _first_param(owner)
+ if first is None or first.device.type != "cpu":
+ return
+ try:
+ hook.pre_forward(owner)
+ except Exception: # noqa: BLE001 - the forward's own hook onloads it as before
+ pass
+
+
+def _float_tensors(parts: Iterable[Any], dtype: Any) -> tuple[list, list]:
+ """``(module, name)`` of every floating parameter and buffer not already in ``dtype``. Keyed by registration, not
+ object: sequential offload's leaf hooks register a new Parameter on every onload and offload."""
+ params, buffers, seen = [], [], set()
+ for part in parts:
+ for module in part.modules():
+ for name, p in list(getattr(module, "_parameters", {}).items()):
+ if (
+ p is not None
+ and p.is_floating_point()
+ and p.dtype != dtype
+ and id(p) not in seen
+ ):
+ seen.add(id(p))
+ params.append((module, name))
+ for name, b in list(getattr(module, "_buffers", {}).items()):
+ if b is not None and b.is_floating_point() and b.dtype != dtype:
+ buffers.append((module, name))
+ return params, buffers
+
+
+def upcast_keeping_storage(
+ owner: Any,
+ parts: Optional[list] = None,
+ dtype: Any = None,
+) -> list:
+ import torch
+
+ dtype = torch.float32 if dtype is None else dtype
+ parts = [owner] if parts is None else [p for p in parts if p is not None]
+ _onload_through_kept_hook(owner)
+ params, buffers = _float_tensors(parts, dtype)
+ saved = [("p", m, name, m._parameters[name].data) for m, name in params]
+ saved += [("b", m, name, m._buffers[name]) for m, name in buffers]
+ for part in parts:
+ # The class method: an instance-level ``to`` (ours, an offload arena's) only handles device moves.
+ type(part).to(part, dtype = dtype)
+ token: list = []
+ for kind, m, name, orig in saved:
+ cast = (m._parameters if kind == "p" else m._buffers).get(name)
+ token.append((kind, m, name, orig, cast.data_ptr() if cast is not None else None))
+ return token
+
+
+def _restorable(current: Any, orig: Any, cast_ptr: Any) -> bool:
+ """``current`` holds ``orig``'s values (diffusers < 0.37 ``upcast_vae`` casts some submodules back)."""
+ import torch
+
+ if current.device != orig.device or current.shape != orig.shape:
+ return False
+ if current.dtype != orig.dtype:
+ return current.data_ptr() == cast_ptr
+ if current.device.type == "meta" or current.data_ptr() == orig.data_ptr():
+ return current.device.type == "meta"
+ return bool(torch.equal(current, orig))
+
+
+def restore_storage(token: list) -> None:
+ """Put back the original tensors on the current registrations; one that has since moved or been changed is cast
+ back to its load dtype instead, as the stock cast does."""
+ for kind, m, name, orig, cast_ptr in token:
+ try:
+ table = m._parameters if kind == "p" else m._buffers
+ current = table.get(name)
+ if current is None:
+ continue
+ data = current.data if kind == "p" else current
+ if _restorable(data, orig, cast_ptr):
+ if kind == "p":
+ current.data = orig
+ else:
+ table[name] = orig
+ elif data.dtype != orig.dtype:
+ if kind == "p":
+ current.data = data.to(orig.dtype)
+ else:
+ table[name] = data.to(orig.dtype)
+ except Exception: # noqa: BLE001 - best effort; leaves that tensor as the cast made it
+ pass
+ token.clear()
+
+
+def fp32_keeping_storage(owner: Any, parts: Optional[list] = None):
+ token = upcast_keeping_storage(owner, parts)
+ try:
+ yield
+ finally:
+ restore_storage(token)
+
+
+def _is_16bit(dtype: Any) -> bool:
+ import torch
+ return dtype in (torch.float16, torch.bfloat16)
+
+
+def pipeline_upcasts_keep_storage(pipe: Any):
+ """Route diffusers' fp32 / load-dtype round trip through ``upcast_keeping_storage``; other ``to`` calls run as is."""
+ vae = getattr(pipe, "vae", None)
+ first = _first_param(vae) if vae is not None else None
+ config = getattr(vae, "config", None)
+ if (
+ first is None
+ or not _is_16bit(first.dtype)
+ or not getattr(config, "force_upcast", False)
+ or "to" in getattr(vae, "__dict__", {})
+ ):
+ yield
+ return
+ import torch
+
+ load_dtype = first.dtype
+ stock_to = type(vae).to
+ state: dict = {"token": None}
+
+ def to(*args: Any, **kwargs: Any) -> Any:
+ try:
+ device, dtype, _non_blocking, fmt = torch._C._nn._parse_to(*args, **kwargs)
+ except Exception: # noqa: BLE001 - not a tensor-style call: the stock method reports it
+ return stock_to(vae, *args, **kwargs)
+ if device is None and fmt is None and dtype is not None:
+ current = getattr(_first_param(vae), "dtype", None)
+ if dtype == torch.float32 and state["token"] is None and current == load_dtype:
+ state["token"] = upcast_keeping_storage(vae)
+ return vae
+ if dtype == load_dtype and state["token"] is not None:
+ token, state["token"] = state["token"], None
+ restore_storage(token)
+ return vae
+ return stock_to(vae, *args, **kwargs)
+
+ setattr(to, DTYPE_ONLY_TO_ATTR, True)
+ vae.__dict__["to"] = to
+ try:
+ yield
+ finally:
+ if vae.__dict__.get("to") is to:
+ vae.__dict__.pop("to", None)
+ if state["token"] is not None:
+ # A raise between the upcast and the cast back: leave the VAE in its load dtype, as the next call expects.
+ restore_storage(state["token"])
diff --git a/studio/backend/tests/test_diffusion_vae_upcast_storage.py b/studio/backend/tests/test_diffusion_vae_upcast_storage.py
new file mode 100644
index 00000000000..7410f80d085
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_vae_upcast_storage.py
@@ -0,0 +1,257 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An fp32 run of a force_upcast 16-bit VAE keeps its stored (pinned) tensors and matches stock fp32 output."""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+
+from core.inference import diffusion_vae_fp16 as vf # noqa: E402
+
+CUDA = types.SimpleNamespace(device = "cuda")
+
+
+def _vu():
+ # Imported per test, so the behavioural tests below also run (and fail) against a tree without the helper.
+ from core.inference import diffusion_vae_upcast
+ return diffusion_vae_upcast
+
+
+def _tiny_vae(dtype = torch.float16):
+ torch.manual_seed(0)
+ return (
+ diffusers.AutoencoderKL(
+ in_channels = 3,
+ out_channels = 3,
+ down_block_types = ("DownEncoderBlock2D",) * 3,
+ up_block_types = ("UpDecoderBlock2D",) * 3,
+ block_out_channels = (16, 32, 32),
+ layers_per_block = 1,
+ latent_channels = 4,
+ norm_num_groups = 8,
+ force_upcast = True,
+ )
+ .eval()
+ .to(dtype)
+ )
+
+
+def _storage(module):
+ return {name: (p.data_ptr(), p.dtype) for name, p in module.named_parameters()}
+
+
+def _stock_fp32(vae):
+ ref = copy.deepcopy(vae)
+ ref.to(torch.float32)
+ return ref
+
+
+def test_round_trip_puts_back_the_same_tensors():
+ vu = _vu()
+ vae = _tiny_vae()
+ before = _storage(vae)
+ with vu.fp32_keeping_storage(vae):
+ assert all(p.dtype == torch.float32 for p in vae.parameters())
+ assert _storage(vae) == before
+
+
+def test_parts_only_upcast_the_named_submodules():
+ vu = _vu()
+ vae = _tiny_vae()
+ before = _storage(vae)
+ with vu.fp32_keeping_storage(vae, [vae.encoder, None]):
+ assert next(vae.encoder.parameters()).dtype == torch.float32
+ assert next(vae.decoder.parameters()).dtype == torch.float16
+ assert _storage(vae) == before
+
+
+def test_a_tensor_replaced_meanwhile_is_cast_back_not_overwritten():
+ vu = _vu()
+ vae = _tiny_vae()
+ token = vu.upcast_keeping_storage(vae)
+ fresh = torch.zeros_like(vae.decoder.conv_out.weight.data)
+ vae.decoder.conv_out.weight.data = fresh
+ vu.restore_storage(token)
+ w = vae.decoder.conv_out.weight
+ assert w.dtype == torch.float16 and bool((w == 0).all())
+
+
+def test_pipeline_round_trip_is_bit_identical_and_keeps_storage():
+ vu = _vu()
+ # diffusers' SDXL decode: vae.to(dtype=fp32), decode, vae.to(dtype=fp16).
+ vae = _tiny_vae()
+ ref = _stock_fp32(vae)
+ z = torch.randn(1, 4, 8, 8)
+ before = _storage(vae)
+ pipe = types.SimpleNamespace(vae = vae)
+ with vu.pipeline_upcasts_keep_storage(pipe):
+ assert getattr(vae.__dict__["to"], vu.DTYPE_ONLY_TO_ATTR)
+ assert vae.to(dtype = torch.float32) is vae
+ assert vae.dtype == torch.float32
+ out = vae.decode(z).sample
+ vae.to(dtype = torch.float16)
+ assert vae.to("cpu") is vae
+ assert "to" not in vae.__dict__
+ assert _storage(vae) == before
+ assert torch.equal(out, ref.decode(z).sample)
+
+
+def test_pipeline_raise_mid_upcast_leaves_the_load_dtype():
+ vu = _vu()
+ vae = _tiny_vae()
+ before = _storage(vae)
+ with contextlib.suppress(RuntimeError):
+ with vu.pipeline_upcasts_keep_storage(types.SimpleNamespace(vae = vae)):
+ vae.to(torch.float32)
+ raise RuntimeError("decode failed")
+ assert _storage(vae) == before
+
+
[email protected]("dtype,force_upcast", [(torch.float32, True), (torch.float16, False)])
+def test_pipeline_wrapper_is_inert_off_the_upcast_case(dtype, force_upcast):
+ vu = _vu()
+ vae = _tiny_vae(dtype)
+ vae.register_to_config(force_upcast = force_upcast)
+ with vu.pipeline_upcasts_keep_storage(types.SimpleNamespace(vae = vae)):
+ assert "to" not in vae.__dict__
+
+
+def test_fp16_decode_fp32_encode_keeps_encoder_storage():
+ vae = _tiny_vae()
+ ref = _stock_fp32(vae)
+ assert vf.enable_fp16_vae_decode(
+ types.SimpleNamespace(vae = vae, unet = type("UNet2DConditionModel", (), {})()), CUDA
+ )
+ x = torch.randn(1, 3, 32, 32, dtype = torch.float16)
+ before = _storage(vae)
+ got = vae.encode(x).latent_dist.mean
+ assert _storage(vae) == before
+ assert torch.equal(got, ref.encode(x.float()).latent_dist.mean)
+
+
[email protected](scope = "module")
+def tiny_sdxl_path():
+ """Resolved at module scope, before the per-test empty hub cache and the outbound-network block apply."""
+ import sys
+ from pathlib import Path
+
+ tests_dir = str(Path(__file__).resolve().parent)
+ if tests_dir not in sys.path:
+ sys.path.insert(0, tests_dir)
+ from conftest import allow_outbound
+ from huggingface_hub import snapshot_download
+
+ repo = "hf-internal-testing/tiny-stable-diffusion-xl-pipe"
+ try:
+ return snapshot_download(repo, local_files_only = True)
+ except Exception: # noqa: BLE001 - not cached yet
+ try:
+ with allow_outbound():
+ return snapshot_download(repo)
+ except Exception as exc: # noqa: BLE001
+ pytest.skip(f"tiny pipeline not reachable: {exc}")
+
+
+def _cuda_or_skip():
+ if not torch.cuda.is_available():
+ pytest.skip("CUDA only")
+ pytest.importorskip("accelerate")
+
+
+def test_sdxl_model_offload_vae_stays_pinned_cuda(monkeypatch, tiny_sdxl_path):
+ """Stock: the decode's upcast lands before the VAE's first onload, so it is never pinned and every onload copies
+ every tensor pageable. Here it is pinned after the first render and stays so; output bit-identical."""
+ from core.inference import diffusion_memory as dm
+
+ _cuda_or_skip()
+ path = tiny_sdxl_path
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ outs = {}
+ for arm in ("stock", "kept"):
+ pipe = diffusers.StableDiffusionXLPipeline.from_pretrained(path, torch_dtype = torch.float16)
+ pipe.vae.register_to_config(force_upcast = True)
+ pipe.enable_model_cpu_offload(device = "cuda:0")
+ dm.keep_cpu_weights_on_offload(pipe)
+ wrap = (
+ _vu().pipeline_upcasts_keep_storage
+ if arm == "kept"
+ else lambda p: contextlib.nullcontext()
+ )
+ renders = []
+ for _ in range(2):
+ with wrap(pipe):
+ renders.append(
+ pipe(
+ "a cat",
+ num_inference_steps = 2,
+ height = 64,
+ width = 64,
+ generator = torch.Generator("cpu").manual_seed(0),
+ output_type = "pt",
+ ).images
+ )
+ params = list(pipe.vae.parameters())
+ assert all(p.dtype == torch.float16 and p.device.type == "cpu" for p in params)
+ pinned = sum(p.is_pinned() for p in params)
+ if arm == "kept":
+ assert pinned == len(params)
+ assert pipe.vae.__dict__[dm._KEEP_ATTR].get("pin_layout")
+ outs[arm] = renders
+ assert torch.equal(outs["stock"][0], outs["kept"][0])
+ assert torch.equal(outs["kept"][0], outs["kept"][1])
+
+
+def test_nested_load_dtype_casts_still_restore_the_original_tensors():
+ # diffusers < 0.37 upcast_vae(): whole VAE to fp32, then these submodules back to the load dtype.
+ vu = _vu()
+ vae = _tiny_vae()
+ before = _storage(vae)
+ with vu.pipeline_upcasts_keep_storage(types.SimpleNamespace(vae = vae)):
+ vae.to(dtype = torch.float32)
+ for module in (vae.post_quant_conv, vae.decoder.conv_in, vae.decoder.mid_block):
+ module.to(torch.float16)
+ vae.to(dtype = torch.float16)
+ assert _storage(vae) == before
+
+
+def test_a_changed_same_dtype_replacement_is_left_alone():
+ vu = _vu()
+ vae = _tiny_vae()
+ token = vu.upcast_keeping_storage(vae)
+ vae.decoder.conv_out.to(torch.float16)
+ with torch.no_grad():
+ vae.decoder.conv_out.weight.zero_()
+ vu.restore_storage(token)
+ assert bool((vae.decoder.conv_out.weight == 0).all())
+
+
[email protected]("route", ["helper", "pipeline"])
+def test_sequential_offload_replacement_parameters_return_to_the_load_dtype(route):
+ # accelerate's leaf hooks register new Parameter objects on every onload / offload.
+ pytest.importorskip("accelerate")
+ from accelerate import cpu_offload
+
+ vu = _vu()
+ vae = _tiny_vae()
+ ref = _stock_fp32(vae)
+ cpu_offload(vae, execution_device = torch.device("cpu"))
+ assert next(vae.parameters()).device.type == "meta"
+ x = torch.randn(1, 3, 32, 32, dtype = torch.float16)
+ if route == "helper":
+ with vu.fp32_keeping_storage(vae):
+ got = vae.encode(x.float()).latent_dist.mean
+ else:
+ with vu.pipeline_upcasts_keep_storage(types.SimpleNamespace(vae = vae)):
+ vae.to(dtype = torch.float32)
+ got = vae.encode(x.float()).latent_dist.mean
+ vae.to(dtype = torch.float16)
+ assert {p.dtype for p in vae.parameters()} == {torch.float16}
+ assert torch.equal(got, ref.encode(x.float()).latent_dist.mean)