Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run Qwen-Image-2.1 edits on 16 GB cards instead of refusing them (#12752)
oobaboogadfc56d98d08a
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex a3e68e42f33..b578b8fcd0f 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -148,6 +148,8 @@ from .diffusion_memory import (refine_memory_plan_for_components,refine_plan_from_loaded_weights,release_resident_groups,+ resident_group_mib,+ hook_resident_denoiser,measured_request_extra_mib,settled_snapshot_device_memory,snapshot_device_memory,@@ -9967,6 +9969,11 @@ class DiffusionBackend:condition_pixels = guard_condition_pixels,)if extra_mib > 0:+ releasable_mib = resident_group_mib(state.pipe)+ if guard_condition_pixels > 0 and extra_mib > releasable_mib:+ # The guard credits a conditioned request nothing the load reserved: hook the resident+ # transformer so this request can stream part of it.+ hook_resident_denoiser(state.pipe, guard_target.torch_device, logger)restore_resident = release_resident_groups(state.pipe, extra_mib, logger)guard_kwargs = dict(# NOT the settled snapshot the load uses: that one calls empty_cache(), which is right oncediff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex c0f15af8689..03cc1b63853 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -2802,6 +2802,144 @@ def release_resident_groups(return None+def resident_group_mib(pipe: Any) -> int:+ """MiB an oversized request could release: every resident offload group of the pipeline's components."""+ total = 0+ for module in (getattr(pipe, "components", {}) or {}).values():+ if not getattr(module, "_unsloth_resident_room", None):+ continue+ for group in _offload_groups(module) or []:+ if getattr(group, "_unsloth_resident", False):+ total += int(getattr(group, "_unsloth_resident_bytes", 0))+ return total >> 20+++def hook_resident_denoiser(+ pipe: Any,+ device: Any,+ logger: Any = None,+) -> bool:+ """Give a hookless resident torchao transformer block-level offload hooks, every group kept resident, so+ release_resident_groups can stream part of it. On demand, not at load: hooks at load recompile every load's second+ render. True when installed."""+ transformer = getattr(pipe, "transformer", None)+ if (+ transformer is None+ or _offload_groups(transformer)+ or not _pipe_denoisers_hold_torchao(pipe)+ ):+ return False+ import torch++ onload = torch.device(device)+ if onload.type != "cuda":+ return False+ dit_mib = _module_host_mib(transformer)+ # the load's _torchao_stream_pinnable rule: unpinned, the apply copies the whole transformer to pageable host RAM+ if not _streamed_pin_plan(dit_mib, 0)[0]:+ if logger is not None:+ logger.info(+ "diffusion.memory: the resident transformer keeps no offload hooks (%d MiB cannot be pinned)",+ dit_mib,+ )+ return False+ torch_hooks = _module_forward_hook_ids(transformer)+ try:+ import inspect++ from diffusers.hooks import apply_group_offloading++ install_group_offload_buffer_restore()+ install_group_offload_hooks_eager()+ gkwargs: dict[str, Any] = {+ "onload_device": onload,+ "offload_device": torch.device("cpu"),+ "offload_type": "block_level",+ "num_blocks_per_group": DEFAULT_GROUP_BLOCKS,+ "use_stream": True,+ }+ params = inspect.signature(apply_group_offloading).parameters+ for name in ("non_blocking", "record_stream"):+ if name in params:+ gkwargs[name] = True+ if "low_cpu_mem_usage" in params:+ gkwargs["low_cpu_mem_usage"] = False+ pinned_mib = [0]+ apply_group_offloading(+ transformer, **_torchao_group_offload_kwargs(transformer, gkwargs, pinned_mib)+ )+ if not _pin_top_level_group(transformer, logger, pinned_mib):+ _skip_top_level_copy_back(transformer, logger)+ if gkwargs.get("record_stream"):+ install_group_prefetch(transformer, onload, logger)+ kept = _keep_groups_resident(transformer, dit_mib + 1, onload, logger)+ # the helper reports its own failure as 0 MiB kept: anything still streamed is a failed install+ if not all(getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)):+ raise RuntimeError("not every offload group stayed resident")+ except Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fit+ # the apply already moved the weights to their host copies: back to the card, as before the call+ _remove_group_offload_hooks(transformer)+ _drop_module_forward_hooks_since(transformer, torch_hooks)+ from .diffusion_offload_prefetch import PREFETCHER_ATTR++ for attr in (PREFETCHER_ATTR, "_unsloth_stream_state"):+ transformer.__dict__.pop(attr, None)+ _return_module_tensors(transformer, onload)+ if logger is not None:+ logger.warning("diffusion.memory: resident transformer left without hooks (%s)", exc)+ return False+ if logger is not None:+ logger.info(+ "diffusion.memory: the resident transformer takes offload hooks (%d MiB kept resident) so this request "+ "can stream part of it",+ kept,+ )+ return True+++_FORWARD_HOOK_DICTS = (+ "_forward_pre_hooks",+ "_forward_pre_hooks_with_kwargs",+ "_forward_hooks",+ "_forward_hooks_with_kwargs",+ "_forward_hooks_always_called",+)+++def _module_forward_hook_ids(module: Any) -> dict[str, set]:+ return {name: set(getattr(module, name, None) or ()) for name in _FORWARD_HOOK_DICTS}+++def _drop_module_forward_hooks_since(module: Any, before: dict[str, set]) -> None:+ """Remove the torch forward hooks registered on ``module`` after ``before`` was taken (the group prefetcher's)."""+ for name, ids in before.items():+ hooks = getattr(module, name, None)+ if hooks is not None:+ for key in set(hooks) - ids:+ hooks.pop(key, None)+++def _return_module_tensors(module: Any, onload: Any) -> None:+ """Move ``module``'s weights back to ``onload`` after its hooks are gone. A torchao weight on its host copy can+ still report the card (``Module.to`` skips or fails on it), so its inner tensors move under a new wrapper."""+ import torch+ from diffusers.hooks import group_offloading as go++ is_torchao = getattr(go, "_is_torchao_tensor", lambda t: False)+ for t in [*module.parameters(), *module.buffers()]:+ if _placed_on(t, onload.type, is_torchao):+ continue+ if is_torchao(t):+ names, ctx = t.__tensor_flatten__()+ moved = type(t).__tensor_unflatten__(+ {name: getattr(t, name).to(onload) for name in names}, ctx, t.size(), t.stride()+ )+ go._swap_torchao_tensor(t, moved)+ else:+ t.data = t.data.to(onload)+ torch.cuda.synchronize(onload)++def _is_text_encoder_module(pipe: Any, module: Any) -> bool:for name, component in (getattr(pipe, "components", {}) or {}).items():if component is module:diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pyindex 206fb8dc43c..bf52c75023a 100644--- a/studio/backend/tests/test_diffusion_measured_placement.py+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -513,6 +513,130 @@ def test_partial_release_frees_only_what_the_request_needs(monkeypatch):assert all(next(b.parameters()).device.type == "cuda" for b in net.blocks)+def _resident_hooked(torch, net):+ from diffusers.hooks import apply_group_offloading++ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ record_stream = True,+ non_blocking = True,+ )+ assert dm._keep_groups_resident(net, 1024, "cuda") > 0+ return net+++def _hookless_int8_transformer(monkeypatch):+ """A resident int8 torchao transformer with no offload hooks (the 16 GB tier), its input and reference output."""+ torch, _ = _cuda_offload_model()+ pytest.importorskip("torchao")+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)+ cfg = Int8DynamicActivationInt8WeightConfig(set_inductor_config = False)+ if not hasattr(cfg, "version"):+ pytest.skip("torchao predates versioned configs")+ cfg.version = 2+ torch.manual_seed(0)+ net = (+ torch.nn.Sequential(+ torch.nn.Linear(256, 512),+ torch.nn.Sequential(*[torch.nn.Linear(512, 512) for _ in range(3)]),+ )+ .cuda()+ .to(torch.bfloat16)+ )+ quantize_(net[1], cfg)+ net.requires_grad_(False)+ x = torch.randn(64, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ ref = net(x)+ return torch, net, x, ref+++def test_a_hookless_resident_transformer_takes_hooks_on_demand(monkeypatch):+ """The 16 GB tier keeps the int8 transformer resident without offload hooks; an oversized request (an edit's+ reference) gives it hooks, every group resident, so release_resident_groups can stream part of it."""+ torch, net, x, ref = _hookless_int8_transformer(monkeypatch)+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})+ assert dm.resident_group_mib(pipe) == 0+ assert dm.hook_resident_denoiser(pipe, "cuda")+ assert dm.resident_group_mib(pipe) > 0+ assert not dm.hook_resident_denoiser(pipe, "cuda") # already hooked+ with torch.no_grad():+ assert torch.equal(net(x), ref)+ restore = dm.release_resident_groups(pipe, 1)+ assert restore is not None+ for _ in range(2):+ assert torch.equal(net(x), ref)+ restore()+ assert torch.equal(net(x), ref)+++def test_an_unpinnable_resident_transformer_keeps_no_hooks(monkeypatch):+ """Without room to pin it the apply would copy the whole transformer to pageable host RAM and stream it without+ a copy stream, which the load refuses for torchao too; the request is refused as before instead."""+ torch, net, x, ref = _hookless_int8_transformer(monkeypatch)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", raising = False)+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: 0)+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})+ assert not dm.hook_resident_denoiser(pipe, "cuda")+ assert not dm._offload_groups(net)+ assert dm.resident_group_mib(pipe) == 0+ assert all(p.device.type == "cuda" for p in net.parameters())+ with torch.no_grad():+ assert torch.equal(net(x), ref)++[email protected]("failure", ["raises", "swallowed"])+def test_a_failed_hook_install_leaves_the_transformer_resident(monkeypatch, failure):+ """The apply moves the groups' weights to their host copies; a failure after it puts them back on the card,+ whether the residency step raises or (as _keep_groups_resident does) reports it as 0 MiB kept."""+ torch, net, x, ref = _hookless_int8_transformer(monkeypatch)+ from core.inference.diffusion_offload_prefetch import module_prefetcher++ keep = dm._keep_groups_resident++ def _fail(*args, **kwargs):+ assert module_prefetcher(net) is not None # fails after the prefetcher's install+ if failure == "raises":+ raise RuntimeError("injected")++ def _boom(*_a, **_k):+ raise RuntimeError("injected")++ with monkeypatch.context() as m:+ m.setattr(dm, "_storage_nbytes", _boom)+ assert keep(*args, **kwargs) == 0 # the real helper swallows it+ return 0++ monkeypatch.setattr(dm, "_keep_groups_resident", _fail)+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})+ assert not dm.hook_resident_denoiser(pipe, "cuda")+ assert not dm._offload_groups(net)+ assert module_prefetcher(net) is None+ assert not net._forward_pre_hooks and not net._forward_hooks+ with torch.no_grad():+ for _ in range(2):+ assert torch.equal(net(x), ref)+++def test_generate_hooks_the_resident_transformer_only_when_the_release_falls_short():+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ at = src.index("releasable_mib = resident_group_mib(state.pipe)")+ assert "if guard_condition_pixels > 0 and extra_mib > releasable_mib:" in src[at : at + 200]+ assert "hook_resident_denoiser(state.pipe," in src[at : at + 600]+ assert (+ src.index("restore_resident = release_resident_groups(state.pipe, extra_mib, logger)") > at+ )++def test_measured_request_extra(monkeypatch):monkeypatch.delenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", raising = False)pipe = types.SimpleNamespace(_unsloth_measured_reserve = (2304, "qwen-image-2.1", "default"))