Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Recover VAE storage after precision-conversion OOMs

Daniel Han1d81cbe7e34e
diff --git a/studio/backend/core/inference/diffusion_vae_upcast.py b/studio/backend/core/inference/diffusion_vae_upcast.py
index 44b37ce4df9..48e27a8f9a7 100644
--- a/studio/backend/core/inference/diffusion_vae_upcast.py
+++ b/studio/backend/core/inference/diffusion_vae_upcast.py
@@ -1,14 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""fp32 VAE runs that leave a 16-bit VAE's stored weights in place."""
-
from __future__ import annotations
import contextlib
from typing import Any, Iterable, Optional
-# Marks the instance-level ``to`` below; the kept offload hook's pinned onload passes device moves through it.
DTYPE_ONLY_TO_ATTR = "_unsloth_dtype_only_to"
_KEEP_ATTR = "_unsloth_offload_keep_cpu"
@@ -21,7 +18,6 @@ def _first_param(module: Any) -> Any:
def _onload_through_kept_hook(owner: Any) -> None:
- """Onload via the kept hook (pinned chunks) so the upcast happens on device, not on the host weights."""
hook = getattr(owner, "_hf_hook", None)
if not getattr(hook, _KEEP_ATTR, False) or not callable(getattr(hook, "pre_forward", None)):
return
@@ -35,8 +31,7 @@ def _onload_through_kept_hook(owner: Any) -> None:
def _float_tensors(parts: Iterable[Any], dtype: Any) -> tuple[list, list]:
- """``(module, name)`` of every floating parameter and buffer not already in ``dtype``. Keyed by registration, not
- object: sequential offload's leaf hooks register a new Parameter on every onload and offload."""
+ """Track registrations: sequential offload replaces Parameter objects."""
params, buffers, seen = [], [], set()
for part in parts:
for module in part.modules():
@@ -56,7 +51,6 @@ def _float_tensors(parts: Iterable[Any], dtype: Any) -> tuple[list, list]:
def _kept_state(owner: Any) -> Optional[dict]:
- """The kept offload hook's state on ``owner`` (diffusion_memory._wrap_cpu_offload_hook), else None."""
if not getattr(getattr(owner, "_hf_hook", None), _KEEP_ATTR, False):
return None
state = getattr(owner, "__dict__", {}).get(_KEEP_ATTR)
@@ -71,7 +65,6 @@ def _names(owner: Any) -> dict:
def _host_backed(state: Optional[dict], kind: str, full: Optional[str], current: Any) -> Any:
- """The pinned host tensor ``current`` is an untouched onload of (the hook would re-point it on offload), else None."""
if state is None or full is None or current.device.type in ("cpu", "meta"):
return None
import torch
@@ -111,7 +104,6 @@ def _version(t: Any) -> Optional[int]:
def _mark_onloaded(
state: Optional[dict], kind: str, full: Optional[str], m: Any, name: str
) -> None:
- """Mark the cast-back tensor as the hook's untouched onload, so offload re-points it at the pinned host tensor."""
if state is None or full is None:
return
table = state.get("version" if kind == "p" else "buffer_version")
@@ -145,8 +137,6 @@ def upcast_keeping_storage(
parts: Optional[list] = None,
dtype: Any = None,
) -> list:
- """Cast ``parts`` to ``dtype`` one tensor at a time (never both copies on device); host tensors are kept for
- ``restore_storage``. Kept-hook onloads are re-pointed at their pinned host tensor first, freeing the shared chunk."""
import torch
dtype = torch.float32 if dtype is None else dtype
@@ -168,16 +158,18 @@ def upcast_keeping_storage(
how, ref = "orig", data
else:
how, ref = "dev", (data.device, data.dtype, full, state, None)
- # [kind, module, name, how, ref, cast data_ptr, cast _version]; the cast fields stay None until it lands.
+ # [kind, module, name, storage source, original, cast pointer, cast version]
records.append([kind, m, name, how, ref, None, None])
del current, data
token: list = []
try:
- # Host-backed weights first, all of them: they share one device chunk, which frees only once none points into it.
+ # Every host-backed view must leave the shared device chunk before it can be freed.
for rec in records:
if rec[3] == "host":
token.append(rec)
_set(rec[0], rec[1], rec[2], rec[4][4])
+ if rec[0] == "b":
+ rec[4][3]["buffer_version"][rec[4][2]] = None
for rec in records:
kind, m, name, how, ref = rec[:5]
if how != "host":
@@ -188,18 +180,16 @@ def upcast_keeping_storage(
del data
_set(kind, m, name, cast)
del cast
- current = _get(kind, m, name)
+ current = (m._parameters if kind == "p" else m._buffers)[name]
rec[5], rec[6] = current.data_ptr(), _version(current)
del current
except BaseException:
- # A failed upcast (OOM) puts every tensor back in its original dtype before the error propagates.
restore_storage(token)
raise
return token
def _restorable(current: Any, orig: Any, cast_ptr: Any) -> bool:
- """``current`` holds ``orig``'s values (diffusers < 0.37 ``upcast_vae`` casts some submodules back)."""
import torch
if current.device != orig.device or current.shape != orig.shape:
@@ -227,33 +217,77 @@ def _restore_one(rec: list) -> None:
device, load_dtype, full, state, kept = ref
if how == "host" and rec[5] is None:
if data.data_ptr() == kept.data_ptr():
- # Never cast: back on the device, as the hook's onload of the pinned host tensor.
_set(kind, m, name, kept.to(device, non_blocking = kept.is_pinned()))
_mark_onloaded(state, kind, full, m, name)
return
- untouched = data.data_ptr() == cast_ptr and _version(data) == cast_version
+ current = (m._parameters if kind == "p" else m._buffers)[name]
+ untouched = data.data_ptr() == cast_ptr and _version(current) == cast_version
if cast_ptr is None:
- # Never cast (a "dev" tensor the failed upcast had not reached): still the original.
return
if data.dtype != load_dtype:
- _set(kind, m, name, data.to(load_dtype))
+ try:
+ cast = data.to(load_dtype)
+ except RuntimeError as exc:
+ from .diffusion_vae_oom import is_oom
+ if is_oom(exc) and untouched and cast_version is not None and how == "host":
+ _set(kind, m, name, kept)
+ rec[5], rec[6] = None, None
+ raise
+ _set(kind, m, name, cast)
if untouched and how == "host":
- # fp16 / bf16 -> fp32 -> back is exact, so this is the pinned host tensor's onload again.
_mark_onloaded(state, kind, full, m, name)
elif how == "host" and data.device.type != "cpu":
- # Cast back meanwhile by a submodule ``to`` (diffusers < 0.37 ``upcast_vae``): still the host tensor's values?
if data.shape == kept.shape and bool(torch.equal(data, kept.to(data.device))):
_mark_onloaded(state, kind, full, m, name)
+def _release_restore_memory(token: list, exc: Optional[BaseException] = None) -> None:
+ import torch
+ import traceback
+
+ errors, seen = [exc], set()
+ while errors:
+ error = errors.pop()
+ if error is None or id(error) in seen:
+ continue
+ seen.add(id(error))
+ traceback.clear_frames(error.__traceback__)
+ errors.extend((error.__cause__, error.__context__))
+ errors.extend(getattr(error, "exceptions", ()))
+ backends = dict.fromkeys(rec[4][0].type for rec in token if rec[3] in ("dev", "host"))
+ for backend in backends:
+ clear = getattr(getattr(torch, backend, None), "empty_cache", None)
+ if callable(clear):
+ try:
+ clear()
+ except Exception: # noqa: BLE001
+ pass
+
+
def restore_storage(token: list) -> None:
- """Put back the original dtype: a kept host tensor itself, a device tensor by an exact cast back; one since moved
- or changed is cast back to its load dtype, as the stock cast does."""
- for rec in token:
- try:
- _restore_one(rec)
- except Exception: # noqa: BLE001 - best effort; leaves that tensor as the cast made it
- pass
+ import sys
+
+ pending = list(reversed(token))
+ exc = sys.exc_info()[1]
+ if exc is not None:
+ from .diffusion_vae_oom import is_oom
+ if is_oom(exc):
+ _release_restore_memory(pending, exc)
+ while pending:
+ failed = []
+ released = False
+ for rec in pending:
+ cast_ptr = rec[5]
+ try:
+ _restore_one(rec)
+ except Exception: # noqa: BLE001 - retried below; a later cast back can free the room it needs
+ failed.append(rec)
+ released |= cast_ptr is not None and rec[5] is None
+ if released:
+ _release_restore_memory(failed)
+ elif len(failed) == len(pending):
+ break
+ pending = failed
token.clear()
@@ -273,7 +307,6 @@ def _is_16bit(dtype: Any) -> bool:
@contextlib.contextmanager
def pipeline_upcasts_keep_storage(pipe: Any):
- """Route diffusers' fp32 / load-dtype round trip through ``upcast_keeping_storage``; other ``to`` calls run as is."""
vae = getattr(pipe, "vae", None)
first = _first_param(vae) if vae is not None else None
config = getattr(vae, "config", None)
@@ -315,5 +348,4 @@ def pipeline_upcasts_keep_storage(pipe: Any):
if vae.__dict__.get("to") is to:
vae.__dict__.pop("to", None)
if state["token"] is not None:
- # A raise between the upcast and the cast back: leave the VAE in its load dtype, as the next call expects.
restore_storage(state["token"])
diff --git a/studio/backend/tests/test_diffusion_vae_rollback_oom.py b/studio/backend/tests/test_diffusion_vae_rollback_oom.py
new file mode 100644
index 00000000000..a228e10b38f
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_vae_rollback_oom.py
@@ -0,0 +1,266 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+
+from __future__ import annotations
+
+import traceback
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_vae_upcast as vu # noqa: E402
+
+
+class _Budget:
+ def __init__(
+ self,
+ cap: int,
+ error_type = torch.OutOfMemoryError,
+ ):
+ self.cap, self.used = cap, 0
+ self.ooms = 0
+ self.error_type = error_type
+
+ def alloc(self, n: int) -> None:
+ if self.used + n > self.cap:
+ self.ooms += 1
+ raise self.error_type("out of memory")
+ self.used += n
+
+
+class _Tensor:
+ def __init__(
+ self,
+ budget: _Budget,
+ n: int,
+ dtype,
+ device = "cuda:0",
+ value = 1,
+ ):
+ self.budget, self.n, self.dtype = budget, n, dtype
+ self.device = torch.device(device)
+ self.size = (
+ n * (2 if dtype in (torch.float16, torch.bfloat16) else 4)
+ if self.device.type != "cpu"
+ else 0
+ )
+ self.shape = (n,)
+ self._version, self.value = 0, value
+ self.live = False
+ if self.size:
+ budget.alloc(self.size)
+ self.live = True
+
+ def __del__(self):
+ if self.live:
+ self.budget.used -= self.size
+
+ def is_pinned(self):
+ return False
+
+ def is_floating_point(self):
+ return True
+
+ def is_inference(self):
+ return False
+
+ def data_ptr(self):
+ return id(self)
+
+ def to(
+ self,
+ *args,
+ dtype = None,
+ device = None,
+ **kwargs,
+ ):
+ dtype = dtype or next((a for a in args if isinstance(a, torch.dtype)), self.dtype)
+ device = torch.device(
+ device or next((a for a in args if not isinstance(a, torch.dtype)), self.device)
+ )
+ return (
+ self
+ if dtype == self.dtype and device == self.device
+ else _Tensor(self.budget, self.n, dtype, device, self.value)
+ )
+
+
+class _Param:
+ def __init__(self, t):
+ self.data = t
+ self._version = 0
+
+ @property
+ def dtype(self):
+ return self.data.dtype
+
+ def is_floating_point(self):
+ return True
+
+ def is_inference(self):
+ return False
+
+ def data_ptr(self):
+ return self.data.data_ptr()
+
+ @property
+ def device(self):
+ return self.data.device
+
+
+class _Model:
+ def __init__(
+ self,
+ budget: _Budget,
+ dtype = torch.float16,
+ kind = "p",
+ ):
+ self._buffers = {}
+ self._parameters = {
+ name: _Param(_Tensor(budget, n, dtype))
+ for name, n in [("large0", 4), ("bias", 1), ("large1", 4)]
+ }
+ if kind == "b":
+ self._buffers = {name: p.data for name, p in self._parameters.items()}
+ self._parameters = {}
+
+ def modules(self):
+ return [self]
+
+ def named_modules(self):
+ return [("", self)]
+
+ def parameters(self):
+ return iter(self._parameters.values())
+
+
+def _kept_model(
+ monkeypatch,
+ budget,
+ dtype = torch.float16,
+ kind = "p",
+):
+ model = _Model(budget, dtype, kind)
+ table = model._parameters if kind == "p" else model._buffers
+ hosts = {
+ name: _Tensor(budget, t.data.n if kind == "p" else t.n, dtype, "cpu")
+ for name, t in table.items()
+ }
+ state = {
+ "host" if kind == "p" else "buffer_host": hosts,
+ "version" if kind == "p" else "buffer_version": {
+ name: (t, 0, t.data_ptr()) for name, t in table.items()
+ },
+ }
+ model._hf_hook = types.SimpleNamespace(_unsloth_offload_keep_cpu = True)
+ model._unsloth_offload_keep_cpu = state
+ monkeypatch.setattr(vu, "_host_backed", lambda state, kind, full, current: hosts[full])
+ return model, state, table
+
+
+def test_rollback_retries_a_cast_back_that_ran_out_of_room():
+ # Large0's cast-back needs 8 free units; only the bias cast-back makes that room.
+ budget = _Budget(34)
+ model = _Model(budget)
+ with pytest.raises(torch.OutOfMemoryError):
+ vu.upcast_keeping_storage(model)
+ dtypes = {name: p.dtype for name, p in model._parameters.items()}
+ assert dtypes == {name: torch.float16 for name in dtypes}
+ assert budget.used == 18
+ assert budget.ooms == 1
+
+
[email protected]("cap", range(18, 36))
[email protected]("dtype", [torch.float16, torch.bfloat16])
[email protected]("kind", ["p", "b"])
+def test_kept_host_rollback_restores_tight_budgets(monkeypatch, cap, dtype, kind):
+ budget = _Budget(cap)
+ model, state, table = _kept_model(monkeypatch, budget, dtype, kind)
+ with pytest.raises(torch.OutOfMemoryError):
+ vu.upcast_keeping_storage(model)
+ assert all(t.device.type == "cuda" and t.dtype == dtype for t in table.values())
+ assert budget.used == 18
+ versions = state["version" if kind == "p" else "buffer_version"]
+ assert all(versions[name][2] == t.data_ptr() for name, t in table.items())
+
+
[email protected]("chained", [False, True])
[email protected]("error_type", [torch.OutOfMemoryError, RuntimeError])
+def test_forward_oom_frees_dead_frame_activations_before_restoring(chained, error_type):
+ budget = _Budget(54)
+ model = _Model(budget)
+ error = error_type("MPS backend out of memory")
+
+ def inner():
+ _activation = _Tensor(budget, 4, torch.float32)
+ raise error_type("out of memory") if chained else error
+
+ def forward():
+ try:
+ inner()
+ except RuntimeError as cause:
+ if chained:
+ raise error from cause
+ raise
+
+ with pytest.raises(error_type) as caught:
+ with vu.fp32_keeping_storage(model):
+ forward()
+ assert caught.value is error
+ assert "forward" in [f.name for f in traceback.extract_tb(error.__traceback__)]
+ assert all(p.dtype == torch.float16 for p in model._parameters.values())
+ assert budget.used == 18
+
+
+def test_kept_host_rollback_handles_backend_runtime_oom(monkeypatch):
+ budget = _Budget(20, RuntimeError)
+ model, state, table = _kept_model(monkeypatch, budget)
+ with pytest.raises(RuntimeError, match = "out of memory"):
+ vu.upcast_keeping_storage(model)
+ assert all(t.device.type == "cuda" and t.dtype == torch.float16 for t in table.values())
+ assert budget.used == 18
+
+
+def test_backend_cache_failure_does_not_replace_the_original_oom(monkeypatch):
+ def fail():
+ raise RuntimeError("cache unavailable")
+
+ monkeypatch.setattr(torch.cuda, "empty_cache", fail)
+ budget = _Budget(34)
+ model = _Model(budget)
+ with pytest.raises(torch.OutOfMemoryError, match = "out of memory"):
+ vu.upcast_keeping_storage(model)
+ assert all(p.dtype == torch.float16 for p in model._parameters.values())
+
+
+def test_host_fallback_preserves_inplace_parameter_changes(monkeypatch):
+ budget = _Budget(100)
+ model, state, table = _kept_model(monkeypatch, budget)
+ token = vu.upcast_keeping_storage(model)
+ table["large0"].data.value = 7
+ table["large0"]._version += 1
+ budget.cap = 20
+ vu.restore_storage(token)
+ assert table["large0"].data.value == 7
+ assert not token
+
+
+def test_no_progress_stops_and_preserves_non_oom_tracebacks(monkeypatch):
+ attempts = []
+
+ def fail(rec):
+ attempts.append(rec)
+ raise ValueError("persistent")
+
+ monkeypatch.setattr(vu, "_restore_one", fail)
+ token = [[None] * 7 for _ in range(3)]
+ try:
+ raise ValueError("caller")
+ except ValueError as error:
+ tb = error.__traceback__
+ vu.restore_storage(token)
+ assert error.__traceback__ is tb
+ assert len(attempts) == 3
+ assert not token