Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: free the failed group offload setup before the OOM retry (#14821)

Daniel Hanfe80287df082
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index f8e9ef48d43..e1cefbfe711 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -4390,6 +4390,7 @@ def apply_memory_plan(
applied = _apply_group_offload(pipe, placement, logger, **group_kwargs)
failure = getattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, None)
if not applied and failure is not None and _is_oom(failure):
+ _set_group_offload_failure(pipe, None) # the retry records its own failure, if any
lowered = _retry_group_offload_lower(pipe, plan, placement, logger, failure)
if lowered is not None:
applied = True
@@ -4483,8 +4484,20 @@ def apply_memory_plan(
_GROUP_OFFLOAD_FAILURE_ATTR = "_unsloth_group_offload_failure"
+class _GroupOffloadFailure(Exception):
+ """Type and message of a failed group offload setup. The live exception is not kept: its traceback holds the
+ failed frames, whose tensors would stay allocated through the very retry that needs the room."""
+
+ def __init__(self, exc: BaseException) -> None:
+ super().__init__(f"{type(exc).__name__}: {exc}")
+ self.type_name = type(exc).__name__
+ self.oom = _is_oom(exc)
+
+
def _set_group_offload_failure(pipe: Any, exc: Optional[BaseException]) -> None:
"""What made ``_apply_group_offload`` return False (it absorbs the exception so the caller can fall back)."""
+ if exc is not None and not isinstance(exc, _GroupOffloadFailure):
+ exc = _GroupOffloadFailure(exc)
try:
setattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, exc)
except Exception: # noqa: BLE001
@@ -4492,6 +4505,8 @@ def _set_group_offload_failure(pipe: Any, exc: Optional[BaseException]) -> None:
def _is_oom(exc: BaseException) -> bool:
+ if isinstance(exc, _GroupOffloadFailure):
+ return exc.oom
from .diffusion_batched import is_oom_error
return is_oom_error(exc)
diff --git a/studio/backend/tests/test_diffusion_learned_peaks.py b/studio/backend/tests/test_diffusion_learned_peaks.py
index 6abfb154ea1..6e8b4ef8a36 100644
--- a/studio/backend/tests/test_diffusion_learned_peaks.py
+++ b/studio/backend/tests/test_diffusion_learned_peaks.py
@@ -12,6 +12,7 @@ the load peaked at 4.6 GiB. CPU-only; loaded sizes and the torchao check are stu
from __future__ import annotations
import json
+import sys
import types
import pytest
@@ -232,7 +233,14 @@ def test_request_guard_of_an_unseeded_family_reads_its_learned_peak(klein_pipe):
# ---------------------------------------------------------------------------------------------------- video
+def _pin_device_name(monkeypatch):
+ """The fake torch these tests install has no get_device_name, so the record would key on None while the lookup,
+ on real torch, keys on the visible card's name: pin both."""
+ monkeypatch.setattr(mp, "device_name", lambda device: "NVIDIA L4")
+
+
def test_video_request_peaks_persist_across_processes(store, monkeypatch):
+ _pin_device_name(monkeypatch)
monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)
monkeypatch.setattr(vsr, "_PEAKS", {})
key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")
@@ -250,6 +258,7 @@ def test_video_request_peaks_persist_across_processes(store, monkeypatch):
monkeypatch.setitem(__import__("sys").modules, "torch", _Torch)
assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) == 3000
monkeypatch.undo()
+ _pin_device_name(monkeypatch)
monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(store))
monkeypatch.setattr(
mp, "device_name", lambda dev: "NVIDIA L4"
@@ -1363,6 +1372,75 @@ def test_oom_inside_the_first_offload_apply_unwinds_its_hooks(monkeypatch):
assert dm._is_oom(getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR))
+def test_the_setup_oom_retry_frees_what_the_failed_setup_allocated(monkeypatch):
+ """The failed setup's traceback holds its frames, so the stored failure is only type + message: a 4 MiB
+ allocation made before the OOM is gone by the time the retry installs again, and after it."""
+ import weakref
+
+ torch = pytest.importorskip("torch")
+ scratch_refs = []
+ alive_at_retry = []
+
+ def installer(module, **kwargs):
+ if not scratch_refs:
+ scratch = torch.empty(1 << 20, dtype = torch.float32) # 4 MiB held by the failing frame
+ scratch_refs.append(weakref.ref(scratch))
+ raise torch.OutOfMemoryError("CUDA out of memory: injected group setup failure")
+ alive_at_retry.append(scratch_refs[0]() is not None)
+
+ hooks = types.ModuleType("diffusers.hooks")
+ hooks.apply_group_offloading = installer
+ hooks.group_offloading = types.SimpleNamespace(_is_torchao_tensor = lambda _: False)
+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
+ monkeypatch.setattr(dm, "install_group_offload_buffer_restore", lambda: None)
+ monkeypatch.setattr(dm, "install_group_offload_hooks_eager", lambda: None)
+ monkeypatch.setattr(dm, "_has_offload_groups", lambda module: False)
+ monkeypatch.setattr(dm, "_torchao_group_offload_kwargs", lambda module, kwargs, pinned: kwargs)
+ monkeypatch.setattr(dm, "keep_cpu_weights_on_offload", lambda pipe, logger: 0)
+ dit = torch.nn.Linear(2, 2)
+ pipe = types.SimpleNamespace(
+ transformer = dit,
+ components = {"transformer": dit},
+ enable_model_cpu_offload = lambda **kwargs: None,
+ )
+ plan = dm.MemoryPlan(
+ requested_mode = "auto",
+ offload_policy = dm.OFFLOAD_GROUP,
+ vae_tiling = False,
+ vae_slicing = False,
+ device_memory = _memory(16000, 16000),
+ estimates = {},
+ reasons = (),
+ stream_text_encoders = True,
+ stream_transformer = True,
+ )
+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "cpu")
+ assert policy == dm.OFFLOAD_GROUP and dm.applied_plan(pipe, plan).estimates["setup_oom_retry"]
+ assert alive_at_retry == [False]
+ assert scratch_refs[0]() is None
+ assert getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR, None) is None # cleared once the retry took over
+
+
+def test_a_stored_setup_failure_keeps_its_kind_but_not_its_traceback():
+ def nested():
+ try:
+ raise _OOM("CUDA out of memory")
+ except _OOM as oom:
+ raise RuntimeError("compile failed") from oom
+
+ pipe = types.SimpleNamespace()
+ try:
+ nested()
+ except RuntimeError as exc:
+ dm._set_group_offload_failure(pipe, exc)
+ stored = getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR)
+ assert dm._is_oom(stored) # the OOM in the cause chain still classifies
+ assert stored.__traceback__ is None and stored.__cause__ is None and stored.__context__ is None
+ assert str(stored) == "RuntimeError: compile failed" and stored.type_name == "RuntimeError"
+ dm._set_group_offload_failure(pipe, ValueError("no block list"))
+ assert not dm._is_oom(getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR))
+
+
def test_video_window_stays_open_until_the_high_water_mark_is_read(store, monkeypatch):
monkeypatch.setattr(vsr, "_PEAKS", {})
work = 1280 * 704 * 49
@@ -1522,6 +1600,7 @@ def test_compile_fallback_during_the_render_records_nothing(store, monkeypatch):
def test_failed_capture_video_peak_never_reaches_the_keyed_or_persisted_peaks(store, monkeypatch):
+ _pin_device_name(monkeypatch)
monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)
monkeypatch.setattr(vsr, "_PEAKS", {})
key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")
@@ -1865,6 +1944,7 @@ def test_resident_text_encoders_add_their_streaming_window(monkeypatch):
def test_video_peaks_are_persisted_on_the_streamed_baseline(store, monkeypatch):
+ _pin_device_name(monkeypatch)
monkeypatch.setattr(vsr, "_PEAKS", {})
key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")
work = 1280 * 704 * 49