Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: free the failed group offload setup before the OOM retry (#14821)
Daniel Hanfe80287df082
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex f8e9ef48d43..e1cefbfe711 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -4390,6 +4390,7 @@ def apply_memory_plan(applied = _apply_group_offload(pipe, placement, logger, **group_kwargs)failure = getattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, None)if not applied and failure is not None and _is_oom(failure):+ _set_group_offload_failure(pipe, None) # the retry records its own failure, if anylowered = _retry_group_offload_lower(pipe, plan, placement, logger, failure)if lowered is not None:applied = True@@ -4483,8 +4484,20 @@ def apply_memory_plan(_GROUP_OFFLOAD_FAILURE_ATTR = "_unsloth_group_offload_failure"+class _GroupOffloadFailure(Exception):+ """Type and message of a failed group offload setup. The live exception is not kept: its traceback holds the+ failed frames, whose tensors would stay allocated through the very retry that needs the room."""++ def __init__(self, exc: BaseException) -> None:+ super().__init__(f"{type(exc).__name__}: {exc}")+ self.type_name = type(exc).__name__+ self.oom = _is_oom(exc)++def _set_group_offload_failure(pipe: Any, exc: Optional[BaseException]) -> None:"""What made ``_apply_group_offload`` return False (it absorbs the exception so the caller can fall back)."""+ if exc is not None and not isinstance(exc, _GroupOffloadFailure):+ exc = _GroupOffloadFailure(exc)try:setattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, exc)except Exception: # noqa: BLE001@@ -4492,6 +4505,8 @@ def _set_group_offload_failure(pipe: Any, exc: Optional[BaseException]) -> None:def _is_oom(exc: BaseException) -> bool:+ if isinstance(exc, _GroupOffloadFailure):+ return exc.oomfrom .diffusion_batched import is_oom_errorreturn is_oom_error(exc)diff --git a/studio/backend/tests/test_diffusion_learned_peaks.py b/studio/backend/tests/test_diffusion_learned_peaks.pyindex 6abfb154ea1..6e8b4ef8a36 100644--- a/studio/backend/tests/test_diffusion_learned_peaks.py+++ b/studio/backend/tests/test_diffusion_learned_peaks.py@@ -12,6 +12,7 @@ the load peaked at 4.6 GiB. CPU-only; loaded sizes and the torchao check are stufrom __future__ import annotationsimport json+import sysimport typesimport pytest@@ -232,7 +233,14 @@ def test_request_guard_of_an_unseeded_family_reads_its_learned_peak(klein_pipe):# ---------------------------------------------------------------------------------------------------- video+def _pin_device_name(monkeypatch):+ """The fake torch these tests install has no get_device_name, so the record would key on None while the lookup,+ on real torch, keys on the visible card's name: pin both."""+ monkeypatch.setattr(mp, "device_name", lambda device: "NVIDIA L4")++def test_video_request_peaks_persist_across_processes(store, monkeypatch):+ _pin_device_name(monkeypatch)monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)monkeypatch.setattr(vsr, "_PEAKS", {})key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")@@ -250,6 +258,7 @@ def test_video_request_peaks_persist_across_processes(store, monkeypatch):monkeypatch.setitem(__import__("sys").modules, "torch", _Torch)assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) == 3000monkeypatch.undo()+ _pin_device_name(monkeypatch)monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(store))monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4"@@ -1363,6 +1372,75 @@ def test_oom_inside_the_first_offload_apply_unwinds_its_hooks(monkeypatch):assert dm._is_oom(getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR))+def test_the_setup_oom_retry_frees_what_the_failed_setup_allocated(monkeypatch):+ """The failed setup's traceback holds its frames, so the stored failure is only type + message: a 4 MiB+ allocation made before the OOM is gone by the time the retry installs again, and after it."""+ import weakref++ torch = pytest.importorskip("torch")+ scratch_refs = []+ alive_at_retry = []++ def installer(module, **kwargs):+ if not scratch_refs:+ scratch = torch.empty(1 << 20, dtype = torch.float32) # 4 MiB held by the failing frame+ scratch_refs.append(weakref.ref(scratch))+ raise torch.OutOfMemoryError("CUDA out of memory: injected group setup failure")+ alive_at_retry.append(scratch_refs[0]() is not None)++ hooks = types.ModuleType("diffusers.hooks")+ hooks.apply_group_offloading = installer+ hooks.group_offloading = types.SimpleNamespace(_is_torchao_tensor = lambda _: False)+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)+ monkeypatch.setattr(dm, "install_group_offload_buffer_restore", lambda: None)+ monkeypatch.setattr(dm, "install_group_offload_hooks_eager", lambda: None)+ monkeypatch.setattr(dm, "_has_offload_groups", lambda module: False)+ monkeypatch.setattr(dm, "_torchao_group_offload_kwargs", lambda module, kwargs, pinned: kwargs)+ monkeypatch.setattr(dm, "keep_cpu_weights_on_offload", lambda pipe, logger: 0)+ dit = torch.nn.Linear(2, 2)+ pipe = types.SimpleNamespace(+ transformer = dit,+ components = {"transformer": dit},+ enable_model_cpu_offload = lambda **kwargs: None,+ )+ plan = dm.MemoryPlan(+ requested_mode = "auto",+ offload_policy = dm.OFFLOAD_GROUP,+ vae_tiling = False,+ vae_slicing = False,+ device_memory = _memory(16000, 16000),+ estimates = {},+ reasons = (),+ stream_text_encoders = True,+ stream_transformer = True,+ )+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "cpu")+ assert policy == dm.OFFLOAD_GROUP and dm.applied_plan(pipe, plan).estimates["setup_oom_retry"]+ assert alive_at_retry == [False]+ assert scratch_refs[0]() is None+ assert getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR, None) is None # cleared once the retry took over+++def test_a_stored_setup_failure_keeps_its_kind_but_not_its_traceback():+ def nested():+ try:+ raise _OOM("CUDA out of memory")+ except _OOM as oom:+ raise RuntimeError("compile failed") from oom++ pipe = types.SimpleNamespace()+ try:+ nested()+ except RuntimeError as exc:+ dm._set_group_offload_failure(pipe, exc)+ stored = getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR)+ assert dm._is_oom(stored) # the OOM in the cause chain still classifies+ assert stored.__traceback__ is None and stored.__cause__ is None and stored.__context__ is None+ assert str(stored) == "RuntimeError: compile failed" and stored.type_name == "RuntimeError"+ dm._set_group_offload_failure(pipe, ValueError("no block list"))+ assert not dm._is_oom(getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR))++def test_video_window_stays_open_until_the_high_water_mark_is_read(store, monkeypatch):monkeypatch.setattr(vsr, "_PEAKS", {})work = 1280 * 704 * 49@@ -1522,6 +1600,7 @@ def test_compile_fallback_during_the_render_records_nothing(store, monkeypatch):def test_failed_capture_video_peak_never_reaches_the_keyed_or_persisted_peaks(store, monkeypatch):+ _pin_device_name(monkeypatch)monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)monkeypatch.setattr(vsr, "_PEAKS", {})key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")@@ -1865,6 +1944,7 @@ def test_resident_text_encoders_add_their_streaming_window(monkeypatch):def test_video_peaks_are_persisted_on_the_streamed_baseline(store, monkeypatch):+ _pin_device_name(monkeypatch)monkeypatch.setattr(vsr, "_PEAKS", {})key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")work = 1280 * 704 * 49