Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run Qwen-Image-2.1 edits on 16 GB cards instead of refusing them (#12752)

oobaboogadfc56d98d08a
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index a3e68e42f33..b578b8fcd0f 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -148,6 +148,8 @@ from .diffusion_memory import (
refine_memory_plan_for_components,
refine_plan_from_loaded_weights,
release_resident_groups,
+ resident_group_mib,
+ hook_resident_denoiser,
measured_request_extra_mib,
settled_snapshot_device_memory,
snapshot_device_memory,
@@ -9967,6 +9969,11 @@ class DiffusionBackend:
condition_pixels = guard_condition_pixels,
)
if extra_mib > 0:
+ releasable_mib = resident_group_mib(state.pipe)
+ if guard_condition_pixels > 0 and extra_mib > releasable_mib:
+ # The guard credits a conditioned request nothing the load reserved: hook the resident
+ # transformer so this request can stream part of it.
+ hook_resident_denoiser(state.pipe, guard_target.torch_device, logger)
restore_resident = release_resident_groups(state.pipe, extra_mib, logger)
guard_kwargs = dict(
# NOT the settled snapshot the load uses: that one calls empty_cache(), which is right once
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index c0f15af8689..03cc1b63853 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -2802,6 +2802,144 @@ def release_resident_groups(
return None
+def resident_group_mib(pipe: Any) -> int:
+ """MiB an oversized request could release: every resident offload group of the pipeline's components."""
+ total = 0
+ for module in (getattr(pipe, "components", {}) or {}).values():
+ if not getattr(module, "_unsloth_resident_room", None):
+ continue
+ for group in _offload_groups(module) or []:
+ if getattr(group, "_unsloth_resident", False):
+ total += int(getattr(group, "_unsloth_resident_bytes", 0))
+ return total >> 20
+
+
+def hook_resident_denoiser(
+ pipe: Any,
+ device: Any,
+ logger: Any = None,
+) -> bool:
+ """Give a hookless resident torchao transformer block-level offload hooks, every group kept resident, so
+ release_resident_groups can stream part of it. On demand, not at load: hooks at load recompile every load's second
+ render. True when installed."""
+ transformer = getattr(pipe, "transformer", None)
+ if (
+ transformer is None
+ or _offload_groups(transformer)
+ or not _pipe_denoisers_hold_torchao(pipe)
+ ):
+ return False
+ import torch
+
+ onload = torch.device(device)
+ if onload.type != "cuda":
+ return False
+ dit_mib = _module_host_mib(transformer)
+ # the load's _torchao_stream_pinnable rule: unpinned, the apply copies the whole transformer to pageable host RAM
+ if not _streamed_pin_plan(dit_mib, 0)[0]:
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: the resident transformer keeps no offload hooks (%d MiB cannot be pinned)",
+ dit_mib,
+ )
+ return False
+ torch_hooks = _module_forward_hook_ids(transformer)
+ try:
+ import inspect
+
+ from diffusers.hooks import apply_group_offloading
+
+ install_group_offload_buffer_restore()
+ install_group_offload_hooks_eager()
+ gkwargs: dict[str, Any] = {
+ "onload_device": onload,
+ "offload_device": torch.device("cpu"),
+ "offload_type": "block_level",
+ "num_blocks_per_group": DEFAULT_GROUP_BLOCKS,
+ "use_stream": True,
+ }
+ params = inspect.signature(apply_group_offloading).parameters
+ for name in ("non_blocking", "record_stream"):
+ if name in params:
+ gkwargs[name] = True
+ if "low_cpu_mem_usage" in params:
+ gkwargs["low_cpu_mem_usage"] = False
+ pinned_mib = [0]
+ apply_group_offloading(
+ transformer, **_torchao_group_offload_kwargs(transformer, gkwargs, pinned_mib)
+ )
+ if not _pin_top_level_group(transformer, logger, pinned_mib):
+ _skip_top_level_copy_back(transformer, logger)
+ if gkwargs.get("record_stream"):
+ install_group_prefetch(transformer, onload, logger)
+ kept = _keep_groups_resident(transformer, dit_mib + 1, onload, logger)
+ # the helper reports its own failure as 0 MiB kept: anything still streamed is a failed install
+ if not all(getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)):
+ raise RuntimeError("not every offload group stayed resident")
+ except Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fit
+ # the apply already moved the weights to their host copies: back to the card, as before the call
+ _remove_group_offload_hooks(transformer)
+ _drop_module_forward_hooks_since(transformer, torch_hooks)
+ from .diffusion_offload_prefetch import PREFETCHER_ATTR
+
+ for attr in (PREFETCHER_ATTR, "_unsloth_stream_state"):
+ transformer.__dict__.pop(attr, None)
+ _return_module_tensors(transformer, onload)
+ if logger is not None:
+ logger.warning("diffusion.memory: resident transformer left without hooks (%s)", exc)
+ return False
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: the resident transformer takes offload hooks (%d MiB kept resident) so this request "
+ "can stream part of it",
+ kept,
+ )
+ return True
+
+
+_FORWARD_HOOK_DICTS = (
+ "_forward_pre_hooks",
+ "_forward_pre_hooks_with_kwargs",
+ "_forward_hooks",
+ "_forward_hooks_with_kwargs",
+ "_forward_hooks_always_called",
+)
+
+
+def _module_forward_hook_ids(module: Any) -> dict[str, set]:
+ return {name: set(getattr(module, name, None) or ()) for name in _FORWARD_HOOK_DICTS}
+
+
+def _drop_module_forward_hooks_since(module: Any, before: dict[str, set]) -> None:
+ """Remove the torch forward hooks registered on ``module`` after ``before`` was taken (the group prefetcher's)."""
+ for name, ids in before.items():
+ hooks = getattr(module, name, None)
+ if hooks is not None:
+ for key in set(hooks) - ids:
+ hooks.pop(key, None)
+
+
+def _return_module_tensors(module: Any, onload: Any) -> None:
+ """Move ``module``'s weights back to ``onload`` after its hooks are gone. A torchao weight on its host copy can
+ still report the card (``Module.to`` skips or fails on it), so its inner tensors move under a new wrapper."""
+ import torch
+ from diffusers.hooks import group_offloading as go
+
+ is_torchao = getattr(go, "_is_torchao_tensor", lambda t: False)
+ for t in [*module.parameters(), *module.buffers()]:
+ if _placed_on(t, onload.type, is_torchao):
+ continue
+ if is_torchao(t):
+ names, ctx = t.__tensor_flatten__()
+ moved = type(t).__tensor_unflatten__(
+ {name: getattr(t, name).to(onload) for name in names}, ctx, t.size(), t.stride()
+ )
+ go._swap_torchao_tensor(t, moved)
+ else:
+ t.data = t.data.to(onload)
+ torch.cuda.synchronize(onload)
+
+
def _is_text_encoder_module(pipe: Any, module: Any) -> bool:
for name, component in (getattr(pipe, "components", {}) or {}).items():
if component is module:
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
index 206fb8dc43c..bf52c75023a 100644
--- a/studio/backend/tests/test_diffusion_measured_placement.py
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -513,6 +513,130 @@ def test_partial_release_frees_only_what_the_request_needs(monkeypatch):
assert all(next(b.parameters()).device.type == "cuda" for b in net.blocks)
+def _resident_hooked(torch, net):
+ from diffusers.hooks import apply_group_offloading
+
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ record_stream = True,
+ non_blocking = True,
+ )
+ assert dm._keep_groups_resident(net, 1024, "cuda") > 0
+ return net
+
+
+def _hookless_int8_transformer(monkeypatch):
+ """A resident int8 torchao transformer with no offload hooks (the 16 GB tier), its input and reference output."""
+ torch, _ = _cuda_offload_model()
+ pytest.importorskip("torchao")
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)
+ cfg = Int8DynamicActivationInt8WeightConfig(set_inductor_config = False)
+ if not hasattr(cfg, "version"):
+ pytest.skip("torchao predates versioned configs")
+ cfg.version = 2
+ torch.manual_seed(0)
+ net = (
+ torch.nn.Sequential(
+ torch.nn.Linear(256, 512),
+ torch.nn.Sequential(*[torch.nn.Linear(512, 512) for _ in range(3)]),
+ )
+ .cuda()
+ .to(torch.bfloat16)
+ )
+ quantize_(net[1], cfg)
+ net.requires_grad_(False)
+ x = torch.randn(64, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = net(x)
+ return torch, net, x, ref
+
+
+def test_a_hookless_resident_transformer_takes_hooks_on_demand(monkeypatch):
+ """The 16 GB tier keeps the int8 transformer resident without offload hooks; an oversized request (an edit's
+ reference) gives it hooks, every group resident, so release_resident_groups can stream part of it."""
+ torch, net, x, ref = _hookless_int8_transformer(monkeypatch)
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ assert dm.resident_group_mib(pipe) == 0
+ assert dm.hook_resident_denoiser(pipe, "cuda")
+ assert dm.resident_group_mib(pipe) > 0
+ assert not dm.hook_resident_denoiser(pipe, "cuda") # already hooked
+ with torch.no_grad():
+ assert torch.equal(net(x), ref)
+ restore = dm.release_resident_groups(pipe, 1)
+ assert restore is not None
+ for _ in range(2):
+ assert torch.equal(net(x), ref)
+ restore()
+ assert torch.equal(net(x), ref)
+
+
+def test_an_unpinnable_resident_transformer_keeps_no_hooks(monkeypatch):
+ """Without room to pin it the apply would copy the whole transformer to pageable host RAM and stream it without
+ a copy stream, which the load refuses for torchao too; the request is refused as before instead."""
+ torch, net, x, ref = _hookless_int8_transformer(monkeypatch)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", raising = False)
+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: 0)
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ assert not dm.hook_resident_denoiser(pipe, "cuda")
+ assert not dm._offload_groups(net)
+ assert dm.resident_group_mib(pipe) == 0
+ assert all(p.device.type == "cuda" for p in net.parameters())
+ with torch.no_grad():
+ assert torch.equal(net(x), ref)
+
+
[email protected]("failure", ["raises", "swallowed"])
+def test_a_failed_hook_install_leaves_the_transformer_resident(monkeypatch, failure):
+ """The apply moves the groups' weights to their host copies; a failure after it puts them back on the card,
+ whether the residency step raises or (as _keep_groups_resident does) reports it as 0 MiB kept."""
+ torch, net, x, ref = _hookless_int8_transformer(monkeypatch)
+ from core.inference.diffusion_offload_prefetch import module_prefetcher
+
+ keep = dm._keep_groups_resident
+
+ def _fail(*args, **kwargs):
+ assert module_prefetcher(net) is not None # fails after the prefetcher's install
+ if failure == "raises":
+ raise RuntimeError("injected")
+
+ def _boom(*_a, **_k):
+ raise RuntimeError("injected")
+
+ with monkeypatch.context() as m:
+ m.setattr(dm, "_storage_nbytes", _boom)
+ assert keep(*args, **kwargs) == 0 # the real helper swallows it
+ return 0
+
+ monkeypatch.setattr(dm, "_keep_groups_resident", _fail)
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ assert not dm.hook_resident_denoiser(pipe, "cuda")
+ assert not dm._offload_groups(net)
+ assert module_prefetcher(net) is None
+ assert not net._forward_pre_hooks and not net._forward_hooks
+ with torch.no_grad():
+ for _ in range(2):
+ assert torch.equal(net(x), ref)
+
+
+def test_generate_hooks_the_resident_transformer_only_when_the_release_falls_short():
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ at = src.index("releasable_mib = resident_group_mib(state.pipe)")
+ assert "if guard_condition_pixels > 0 and extra_mib > releasable_mib:" in src[at : at + 200]
+ assert "hook_resident_denoiser(state.pipe," in src[at : at + 600]
+ assert (
+ src.index("restore_resident = release_resident_groups(state.pipe, extra_mib, logger)") > at
+ )
+
+
def test_measured_request_extra(monkeypatch):
monkeypatch.delenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", raising = False)
pipe = types.SimpleNamespace(_unsloth_measured_reserve = (2304, "qwen-image-2.1", "default"))