Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Recover VAE storage after precision-conversion OOMs
Daniel Han1d81cbe7e34e
diff --git a/studio/backend/core/inference/diffusion_vae_upcast.py b/studio/backend/core/inference/diffusion_vae_upcast.pyindex 44b37ce4df9..48e27a8f9a7 100644--- a/studio/backend/core/inference/diffusion_vae_upcast.py+++ b/studio/backend/core/inference/diffusion_vae_upcast.py@@ -1,14 +1,11 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""fp32 VAE runs that leave a 16-bit VAE's stored weights in place."""-from __future__ import annotationsimport contextlibfrom typing import Any, Iterable, Optional-# Marks the instance-level ``to`` below; the kept offload hook's pinned onload passes device moves through it.DTYPE_ONLY_TO_ATTR = "_unsloth_dtype_only_to"_KEEP_ATTR = "_unsloth_offload_keep_cpu"@@ -21,7 +18,6 @@ def _first_param(module: Any) -> Any:def _onload_through_kept_hook(owner: Any) -> None:- """Onload via the kept hook (pinned chunks) so the upcast happens on device, not on the host weights."""hook = getattr(owner, "_hf_hook", None)if not getattr(hook, _KEEP_ATTR, False) or not callable(getattr(hook, "pre_forward", None)):return@@ -35,8 +31,7 @@ def _onload_through_kept_hook(owner: Any) -> None:def _float_tensors(parts: Iterable[Any], dtype: Any) -> tuple[list, list]:- """``(module, name)`` of every floating parameter and buffer not already in ``dtype``. Keyed by registration, not- object: sequential offload's leaf hooks register a new Parameter on every onload and offload."""+ """Track registrations: sequential offload replaces Parameter objects."""params, buffers, seen = [], [], set()for part in parts:for module in part.modules():@@ -56,7 +51,6 @@ def _float_tensors(parts: Iterable[Any], dtype: Any) -> tuple[list, list]:def _kept_state(owner: Any) -> Optional[dict]:- """The kept offload hook's state on ``owner`` (diffusion_memory._wrap_cpu_offload_hook), else None."""if not getattr(getattr(owner, "_hf_hook", None), _KEEP_ATTR, False):return Nonestate = getattr(owner, "__dict__", {}).get(_KEEP_ATTR)@@ -71,7 +65,6 @@ def _names(owner: Any) -> dict:def _host_backed(state: Optional[dict], kind: str, full: Optional[str], current: Any) -> Any:- """The pinned host tensor ``current`` is an untouched onload of (the hook would re-point it on offload), else None."""if state is None or full is None or current.device.type in ("cpu", "meta"):return Noneimport torch@@ -111,7 +104,6 @@ def _version(t: Any) -> Optional[int]:def _mark_onloaded(state: Optional[dict], kind: str, full: Optional[str], m: Any, name: str) -> None:- """Mark the cast-back tensor as the hook's untouched onload, so offload re-points it at the pinned host tensor."""if state is None or full is None:returntable = state.get("version" if kind == "p" else "buffer_version")@@ -145,8 +137,6 @@ def upcast_keeping_storage(parts: Optional[list] = None,dtype: Any = None,) -> list:- """Cast ``parts`` to ``dtype`` one tensor at a time (never both copies on device); host tensors are kept for- ``restore_storage``. Kept-hook onloads are re-pointed at their pinned host tensor first, freeing the shared chunk."""import torchdtype = torch.float32 if dtype is None else dtype@@ -168,16 +158,18 @@ def upcast_keeping_storage(how, ref = "orig", dataelse:how, ref = "dev", (data.device, data.dtype, full, state, None)- # [kind, module, name, how, ref, cast data_ptr, cast _version]; the cast fields stay None until it lands.+ # [kind, module, name, storage source, original, cast pointer, cast version]records.append([kind, m, name, how, ref, None, None])del current, datatoken: list = []try:- # Host-backed weights first, all of them: they share one device chunk, which frees only once none points into it.+ # Every host-backed view must leave the shared device chunk before it can be freed.for rec in records:if rec[3] == "host":token.append(rec)_set(rec[0], rec[1], rec[2], rec[4][4])+ if rec[0] == "b":+ rec[4][3]["buffer_version"][rec[4][2]] = Nonefor rec in records:kind, m, name, how, ref = rec[:5]if how != "host":@@ -188,18 +180,16 @@ def upcast_keeping_storage(del data_set(kind, m, name, cast)del cast- current = _get(kind, m, name)+ current = (m._parameters if kind == "p" else m._buffers)[name]rec[5], rec[6] = current.data_ptr(), _version(current)del currentexcept BaseException:- # A failed upcast (OOM) puts every tensor back in its original dtype before the error propagates.restore_storage(token)raisereturn tokendef _restorable(current: Any, orig: Any, cast_ptr: Any) -> bool:- """``current`` holds ``orig``'s values (diffusers < 0.37 ``upcast_vae`` casts some submodules back)."""import torchif current.device != orig.device or current.shape != orig.shape:@@ -227,33 +217,77 @@ def _restore_one(rec: list) -> None:device, load_dtype, full, state, kept = refif how == "host" and rec[5] is None:if data.data_ptr() == kept.data_ptr():- # Never cast: back on the device, as the hook's onload of the pinned host tensor._set(kind, m, name, kept.to(device, non_blocking = kept.is_pinned()))_mark_onloaded(state, kind, full, m, name)return- untouched = data.data_ptr() == cast_ptr and _version(data) == cast_version+ current = (m._parameters if kind == "p" else m._buffers)[name]+ untouched = data.data_ptr() == cast_ptr and _version(current) == cast_versionif cast_ptr is None:- # Never cast (a "dev" tensor the failed upcast had not reached): still the original.returnif data.dtype != load_dtype:- _set(kind, m, name, data.to(load_dtype))+ try:+ cast = data.to(load_dtype)+ except RuntimeError as exc:+ from .diffusion_vae_oom import is_oom+ if is_oom(exc) and untouched and cast_version is not None and how == "host":+ _set(kind, m, name, kept)+ rec[5], rec[6] = None, None+ raise+ _set(kind, m, name, cast)if untouched and how == "host":- # fp16 / bf16 -> fp32 -> back is exact, so this is the pinned host tensor's onload again._mark_onloaded(state, kind, full, m, name)elif how == "host" and data.device.type != "cpu":- # Cast back meanwhile by a submodule ``to`` (diffusers < 0.37 ``upcast_vae``): still the host tensor's values?if data.shape == kept.shape and bool(torch.equal(data, kept.to(data.device))):_mark_onloaded(state, kind, full, m, name)+def _release_restore_memory(token: list, exc: Optional[BaseException] = None) -> None:+ import torch+ import traceback++ errors, seen = [exc], set()+ while errors:+ error = errors.pop()+ if error is None or id(error) in seen:+ continue+ seen.add(id(error))+ traceback.clear_frames(error.__traceback__)+ errors.extend((error.__cause__, error.__context__))+ errors.extend(getattr(error, "exceptions", ()))+ backends = dict.fromkeys(rec[4][0].type for rec in token if rec[3] in ("dev", "host"))+ for backend in backends:+ clear = getattr(getattr(torch, backend, None), "empty_cache", None)+ if callable(clear):+ try:+ clear()+ except Exception: # noqa: BLE001+ pass++def restore_storage(token: list) -> None:- """Put back the original dtype: a kept host tensor itself, a device tensor by an exact cast back; one since moved- or changed is cast back to its load dtype, as the stock cast does."""- for rec in token:- try:- _restore_one(rec)- except Exception: # noqa: BLE001 - best effort; leaves that tensor as the cast made it- pass+ import sys++ pending = list(reversed(token))+ exc = sys.exc_info()[1]+ if exc is not None:+ from .diffusion_vae_oom import is_oom+ if is_oom(exc):+ _release_restore_memory(pending, exc)+ while pending:+ failed = []+ released = False+ for rec in pending:+ cast_ptr = rec[5]+ try:+ _restore_one(rec)+ except Exception: # noqa: BLE001 - retried below; a later cast back can free the room it needs+ failed.append(rec)+ released |= cast_ptr is not None and rec[5] is None+ if released:+ _release_restore_memory(failed)+ elif len(failed) == len(pending):+ break+ pending = failedtoken.clear()@@ -273,7 +307,6 @@ def _is_16bit(dtype: Any) -> bool:@contextlib.contextmanagerdef pipeline_upcasts_keep_storage(pipe: Any):- """Route diffusers' fp32 / load-dtype round trip through ``upcast_keeping_storage``; other ``to`` calls run as is."""vae = getattr(pipe, "vae", None)first = _first_param(vae) if vae is not None else Noneconfig = getattr(vae, "config", None)@@ -315,5 +348,4 @@ def pipeline_upcasts_keep_storage(pipe: Any):if vae.__dict__.get("to") is to:vae.__dict__.pop("to", None)if state["token"] is not None:- # A raise between the upcast and the cast back: leave the VAE in its load dtype, as the next call expects.restore_storage(state["token"])diff --git a/studio/backend/tests/test_diffusion_vae_rollback_oom.py b/studio/backend/tests/test_diffusion_vae_rollback_oom.pynew file mode 100644index 00000000000..a228e10b38f--- /dev/null+++ b/studio/backend/tests/test_diffusion_vae_rollback_oom.py@@ -0,0 +1,266 @@+# SPDX-License-Identifier: AGPL-3.0-or-later+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.++from __future__ import annotations++import traceback+import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_vae_upcast as vu # noqa: E402+++class _Budget:+ def __init__(+ self,+ cap: int,+ error_type = torch.OutOfMemoryError,+ ):+ self.cap, self.used = cap, 0+ self.ooms = 0+ self.error_type = error_type++ def alloc(self, n: int) -> None:+ if self.used + n > self.cap:+ self.ooms += 1+ raise self.error_type("out of memory")+ self.used += n+++class _Tensor:+ def __init__(+ self,+ budget: _Budget,+ n: int,+ dtype,+ device = "cuda:0",+ value = 1,+ ):+ self.budget, self.n, self.dtype = budget, n, dtype+ self.device = torch.device(device)+ self.size = (+ n * (2 if dtype in (torch.float16, torch.bfloat16) else 4)+ if self.device.type != "cpu"+ else 0+ )+ self.shape = (n,)+ self._version, self.value = 0, value+ self.live = False+ if self.size:+ budget.alloc(self.size)+ self.live = True++ def __del__(self):+ if self.live:+ self.budget.used -= self.size++ def is_pinned(self):+ return False++ def is_floating_point(self):+ return True++ def is_inference(self):+ return False++ def data_ptr(self):+ return id(self)++ def to(+ self,+ *args,+ dtype = None,+ device = None,+ **kwargs,+ ):+ dtype = dtype or next((a for a in args if isinstance(a, torch.dtype)), self.dtype)+ device = torch.device(+ device or next((a for a in args if not isinstance(a, torch.dtype)), self.device)+ )+ return (+ self+ if dtype == self.dtype and device == self.device+ else _Tensor(self.budget, self.n, dtype, device, self.value)+ )+++class _Param:+ def __init__(self, t):+ self.data = t+ self._version = 0++ @property+ def dtype(self):+ return self.data.dtype++ def is_floating_point(self):+ return True++ def is_inference(self):+ return False++ def data_ptr(self):+ return self.data.data_ptr()++ @property+ def device(self):+ return self.data.device+++class _Model:+ def __init__(+ self,+ budget: _Budget,+ dtype = torch.float16,+ kind = "p",+ ):+ self._buffers = {}+ self._parameters = {+ name: _Param(_Tensor(budget, n, dtype))+ for name, n in [("large0", 4), ("bias", 1), ("large1", 4)]+ }+ if kind == "b":+ self._buffers = {name: p.data for name, p in self._parameters.items()}+ self._parameters = {}++ def modules(self):+ return [self]++ def named_modules(self):+ return [("", self)]++ def parameters(self):+ return iter(self._parameters.values())+++def _kept_model(+ monkeypatch,+ budget,+ dtype = torch.float16,+ kind = "p",+):+ model = _Model(budget, dtype, kind)+ table = model._parameters if kind == "p" else model._buffers+ hosts = {+ name: _Tensor(budget, t.data.n if kind == "p" else t.n, dtype, "cpu")+ for name, t in table.items()+ }+ state = {+ "host" if kind == "p" else "buffer_host": hosts,+ "version" if kind == "p" else "buffer_version": {+ name: (t, 0, t.data_ptr()) for name, t in table.items()+ },+ }+ model._hf_hook = types.SimpleNamespace(_unsloth_offload_keep_cpu = True)+ model._unsloth_offload_keep_cpu = state+ monkeypatch.setattr(vu, "_host_backed", lambda state, kind, full, current: hosts[full])+ return model, state, table+++def test_rollback_retries_a_cast_back_that_ran_out_of_room():+ # Large0's cast-back needs 8 free units; only the bias cast-back makes that room.+ budget = _Budget(34)+ model = _Model(budget)+ with pytest.raises(torch.OutOfMemoryError):+ vu.upcast_keeping_storage(model)+ dtypes = {name: p.dtype for name, p in model._parameters.items()}+ assert dtypes == {name: torch.float16 for name in dtypes}+ assert budget.used == 18+ assert budget.ooms == 1++[email protected]("cap", range(18, 36))[email protected]("dtype", [torch.float16, torch.bfloat16])[email protected]("kind", ["p", "b"])+def test_kept_host_rollback_restores_tight_budgets(monkeypatch, cap, dtype, kind):+ budget = _Budget(cap)+ model, state, table = _kept_model(monkeypatch, budget, dtype, kind)+ with pytest.raises(torch.OutOfMemoryError):+ vu.upcast_keeping_storage(model)+ assert all(t.device.type == "cuda" and t.dtype == dtype for t in table.values())+ assert budget.used == 18+ versions = state["version" if kind == "p" else "buffer_version"]+ assert all(versions[name][2] == t.data_ptr() for name, t in table.items())++[email protected]("chained", [False, True])[email protected]("error_type", [torch.OutOfMemoryError, RuntimeError])+def test_forward_oom_frees_dead_frame_activations_before_restoring(chained, error_type):+ budget = _Budget(54)+ model = _Model(budget)+ error = error_type("MPS backend out of memory")++ def inner():+ _activation = _Tensor(budget, 4, torch.float32)+ raise error_type("out of memory") if chained else error++ def forward():+ try:+ inner()+ except RuntimeError as cause:+ if chained:+ raise error from cause+ raise++ with pytest.raises(error_type) as caught:+ with vu.fp32_keeping_storage(model):+ forward()+ assert caught.value is error+ assert "forward" in [f.name for f in traceback.extract_tb(error.__traceback__)]+ assert all(p.dtype == torch.float16 for p in model._parameters.values())+ assert budget.used == 18+++def test_kept_host_rollback_handles_backend_runtime_oom(monkeypatch):+ budget = _Budget(20, RuntimeError)+ model, state, table = _kept_model(monkeypatch, budget)+ with pytest.raises(RuntimeError, match = "out of memory"):+ vu.upcast_keeping_storage(model)+ assert all(t.device.type == "cuda" and t.dtype == torch.float16 for t in table.values())+ assert budget.used == 18+++def test_backend_cache_failure_does_not_replace_the_original_oom(monkeypatch):+ def fail():+ raise RuntimeError("cache unavailable")++ monkeypatch.setattr(torch.cuda, "empty_cache", fail)+ budget = _Budget(34)+ model = _Model(budget)+ with pytest.raises(torch.OutOfMemoryError, match = "out of memory"):+ vu.upcast_keeping_storage(model)+ assert all(p.dtype == torch.float16 for p in model._parameters.values())+++def test_host_fallback_preserves_inplace_parameter_changes(monkeypatch):+ budget = _Budget(100)+ model, state, table = _kept_model(monkeypatch, budget)+ token = vu.upcast_keeping_storage(model)+ table["large0"].data.value = 7+ table["large0"]._version += 1+ budget.cap = 20+ vu.restore_storage(token)+ assert table["large0"].data.value == 7+ assert not token+++def test_no_progress_stops_and_preserves_non_oom_tracebacks(monkeypatch):+ attempts = []++ def fail(rec):+ attempts.append(rec)+ raise ValueError("persistent")++ monkeypatch.setattr(vu, "_restore_one", fail)+ token = [[None] * 7 for _ in range(3)]+ try:+ raise ValueError("caller")+ except ValueError as error:+ tb = error.__traceback__+ vu.restore_storage(token)+ assert error.__traceback__ is tb+ assert len(attempts) == 3+ assert not token