Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: engage the fused int8 GEMM when the placement pins the whole denoiser, and restore torchao weights after an oversized request (#12524)
Daniel Han6a0cd598a92c
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 986211a0436..28f4f711120 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -132,6 +132,7 @@ from .diffusion_memory import (normalize_memory_mode,plan_diffusion_memory,plan_fits_total_capacity,+ denoisers_pinned_resident,plan_keeps_transformer_resident,prequant_seed_device,raise_on_image_activation_shortfall,@@ -168,6 +169,7 @@ from .diffusion_speed import (SPEED_MAX,SPEED_OFF,apply_speed_optims,+ engage_pinned_denoisers,auto_dynamic_active,compile_dynamic,compile_eligible,@@ -7153,6 +7155,9 @@ class DiffusionBackend:pipe._unsloth_cuda_graphs = ()pipe._unsloth_cuda_graph_reason = "offload active"speed_applied["cuda_graph"] = False+ # the speed layer saw only the plan; placement may have pinned every denoiser group since+ if denoisers_pinned_resident(pipe):+ engage_pinned_denoisers(pipe, speed_applied, logger)# Per-control provenance for status. cpu_offload=False is the unset default, so only True is# explicit.@@ -9022,6 +9027,8 @@ class DiffusionBackend:and _denoiser_hooked(state.pipe),logger = logger,)+ if denoisers_pinned_resident(state.pipe):+ engage_pinned_denoisers(state.pipe, speed_applied, logger)if getattr(getattr(state.pipe, "vae", None), "_unsloth_fp16_decode", False):speed_applied["vae_fp16_decode"] = Trueobject.__setattr__(state, "speed_mode", SPEED_DEFAULT)diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex c564794f0ea..b1b14dbf24c 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -2957,6 +2957,34 @@ def _background_pin_enabled() -> bool:)+_DENOISER_NAMES = ("transformer", "transformer_2", "unconditional_transformer", "unet")+++def denoisers_pinned_resident(pipe: Any) -> bool:+ """Every denoiser unhooked or with all its offload groups pinned, at least one pinned. The hooks stay so an+ oversized request can stream it again (``release_resident_groups``)."""+ pinned = False+ for name in _DENOISER_NAMES:+ module = getattr(pipe, name, None)+ if module is None:+ continue+ try:+ if any(getattr(sub, "_hf_hook", None) is not None for sub in module.modules()):+ return False # accelerate (model / sequential) offload moves it+ except Exception: # noqa: BLE001 - unreadable: assume it moves+ return False+ groups = _offload_groups(module)+ if not groups:+ registry = getattr(module, "_diffusers_hook", None)+ if any("offload" in str(key) for key in (getattr(registry, "hooks", None) or {})):+ return False+ continue+ if not all(getattr(g, "_unsloth_resident", False) for g in groups):+ return False+ pinned = True+ return pinned++def _offload_groups(module: Any) -> list:"""The diffusers offload groups hooked under ``module``, in registration (block) order."""try:diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex f0a8071d74c..fe857916a1f 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -602,6 +602,38 @@ def apply_speed_optims(return applied+def engage_pinned_denoisers(+ pipe: Any,+ applied: dict,+ logger: Any = None,+) -> dict:+ """Install the int8 GEMM the plan refused once every denoiser group is pinned (before the first forward). Graphs+ stay off: an oversized request can still stream the groups, and the hooks' copy-stream wait breaks capture."""+ if getattr(pipe, "_unsloth_cuda_graph_reason", None) == "offload active":+ try:+ pipe._unsloth_cuda_graph_reason = "denoiser pinned resident under offload hooks"+ except Exception: # noqa: BLE001+ pass+ if not applied.get("compiled") or applied.get("int8_gemm"):+ return applied+ try:+ from .diffusion_int8_gemm import install as install_int8_gemm+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "int8 fused-dequant gemm", exc)+ return applied+ for transformer in _denoiser_dits(pipe):+ try:+ transformer._unsloth_int8_gemm = install_int8_gemm(+ transformer, logger, offload_active = False+ )+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "int8 fused-dequant gemm", exc)+ applied["int8_gemm"] = any(+ bool(getattr(t, "_unsloth_int8_gemm", 0)) for t in _denoiser_dits(pipe)+ )+ return applied++def fp16_unet_offloaded(target: Any, pipe: Any, *, offload_active: bool) -> bool:"""An offloaded fp16 U-Net stays eager: fused QKV costs more transfer than compile saves (L4: 5.43 vs 4.86 s)."""return (diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.pyindex 2100e002b3a..c5b1abc7a68 100644--- a/studio/backend/tests/test_diffusion_int8_gemm.py+++ b/studio/backend/tests/test_diffusion_int8_gemm.py@@ -343,6 +343,83 @@ def test_small_m_and_misaligned_keep_stock(forced):assert g8.call_count() == before+@needs_cuda+def test_pinned_group_offloaded_denoiser_takes_the_gemm_and_survives_release(forced, monkeypatch):+ """Every group pinned: the GEMM engages after placement, bit-identical to torchao through release and restore."""+ pytest.importorskip("diffusers.hooks")+ from diffusers.hooks import apply_group_offloading+ import types as _types++ from core.inference import diffusion_memory as dm+ from core.inference import diffusion_speed as ds++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)++ class Dit(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj_in = torch.nn.Linear(256, 512).to(torch.bfloat16)+ self.blocks = torch.nn.ModuleList(+ torch.nn.Sequential(+ torch.nn.Linear(512, 1024), torch.nn.GELU(), torch.nn.Linear(1024, 512)+ )+ for _ in range(4)+ )++ def forward(self, x):+ x = self.proj_in(x)+ for block in self.blocks:+ x = x + block(x)+ return x++ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ torch.manual_seed(0)+ dit = Dit().cuda().to(torch.bfloat16)+ cfg = Int8DynamicActivationInt8WeightConfig(set_inductor_config = False)+ if not hasattr(cfg, "version"):+ pytest.skip("torchao without config versions")+ cfg.version = (+ 2 # what streams under diffusers group offloading (v1's subclass rejects is_pinned)+ )+ quantize_(dit.blocks, cfg)+ x = torch.randn(300, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ ref = dit(x)+ dit.to("cpu")+ kwargs = dict(+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ record_stream = True,+ non_blocking = True,+ )+ apply_group_offloading(dit, **dm._torchao_group_offload_kwargs(dit, kwargs, [0]))+ pipe = _types.SimpleNamespace(transformer = dit, components = {"transformer": dit})+ assert dm._keep_groups_resident(dit, 1024, "cuda") > 0+ assert dm.denoisers_pinned_resident(pipe)++ monkeypatch.setattr(ds, "_denoiser_dits", lambda p: [dit])+ applied = {"compiled": True, "int8_gemm": False}+ ds.engage_pinned_denoisers(pipe, applied)+ assert applied["int8_gemm"] and dit._unsloth_int8_gemm == 8+ # inference_mode rejects moving torchao weights+ with torch.no_grad():+ before = g8.call_count()+ assert torch.equal(dit(x), ref)+ assert g8.call_count() == before + 8+ restore = dm.release_resident_groups(pipe, 1024)+ assert restore is not None and not dm.denoisers_pinned_resident(pipe)+ for _ in range(2): # first streamed forward traces the prefetch order+ assert torch.equal(dit(x), ref)+ restore()+ assert dm.denoisers_pinned_resident(pipe)+ assert torch.equal(dit(x), ref)+ g8.uninstall(dit)++@needs_cudadef test_fused_mlp_down_projection_traces_without_breaks(forced, monkeypatch):"""The fused MLP's down projection reaches the GEMM through ``linear_from_q`` inside the compiled block: no hostdiff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pyindex bfa5ffd9516..5e2ce8d365b 100644--- a/studio/backend/tests/test_diffusion_measured_placement.py+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -496,6 +496,54 @@ def test_generate_releases_and_restores_resident_groups():assert "pipe._unsloth_measured_reserve = (" in src+def _pinned_pipe(+ monkeypatch,+ resident_flags,+ hf_hook = False,+):+ torch = pytest.importorskip("torch")+ net = torch.nn.Linear(4, 4)+ if hf_hook:+ net._hf_hook = object()+ groups = [types.SimpleNamespace(_unsloth_resident = flag) for flag in resident_flags]+ monkeypatch.setattr(dm, "_offload_groups", lambda m: groups if m is net else [])+ return types.SimpleNamespace(transformer = net, text_encoder = torch.nn.Linear(4, 4))+++def test_denoiser_residency_follows_the_final_placement(monkeypatch):+ """Residency reads the placement (16 GB: all 35 groups pinned), not the plan's stream flag."""+ assert dm.denoisers_pinned_resident(_pinned_pipe(monkeypatch, [True] * 35))+ assert not dm.denoisers_pinned_resident(_pinned_pipe(monkeypatch, [True] * 23 + [False] * 12))+ assert not dm.denoisers_pinned_resident(_pinned_pipe(monkeypatch, [True] * 35, hf_hook = True))+ assert not dm.denoisers_pinned_resident(_pinned_pipe(monkeypatch, []))+++def test_pinned_denoiser_reads_real_group_offload_hooks(monkeypatch):+ """Real group offloading: pinned -> resident, released -> moving, restored -> resident."""+ torch, net = _cuda_offload_model()+ from diffusers.hooks import apply_group_offloading++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)+ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ record_stream = True,+ non_blocking = True,+ )+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})+ assert not dm.denoisers_pinned_resident(pipe) # hooked, nothing pinned: streams+ assert dm._keep_groups_resident(net, 1024, "cuda") > 0+ assert dm.denoisers_pinned_resident(pipe)+ restore = dm.release_resident_groups(pipe, 1024)+ assert restore is not None and not dm.denoisers_pinned_resident(pipe)+ restore()+ assert dm.denoisers_pinned_resident(pipe)++def test_torchao_groups_stay_on_device_after_release_and_restore(monkeypatch):torch, _ = _cuda_offload_model()pytest.importorskip("torchao")diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex a3707ee95b9..ab05869541a 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -2357,6 +2357,37 @@ def test_family_compiles_regionally_closes_the_dynamo_import_window_first(monkeyassert events[:2] == ["guard", "probe"]+def test_pinned_denoiser_engages_the_int8_gemm_after_placement(monkeypatch):+ """Pinned after placement: the GEMM installs with offload_active False, only on a compiled DiT."""+ calls = []+ fake = types.ModuleType("core.inference.diffusion_int8_gemm")++ def _install(+ transformer,+ logger = None,+ offload_active = False,+ ):+ calls.append(offload_active)+ return 0 if offload_active else 60++ fake.install = _install+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_int8_gemm", fake)+ dit = types.SimpleNamespace()+ pipe = types.SimpleNamespace(_unsloth_cuda_graph_reason = "offload active")+ monkeypatch.setattr(ds_mod, "_denoiser_dits", lambda p: [dit])++ applied = {"compiled": True, "int8_gemm": False, "cuda_graph": False}+ ds_mod.engage_pinned_denoisers(pipe, applied)+ assert calls == [False] and applied["int8_gemm"] and dit._unsloth_int8_gemm == 60+ assert not applied["cuda_graph"]+ assert pipe._unsloth_cuda_graph_reason == "denoiser pinned resident under offload hooks"++ calls.clear()+ eager = {"compiled": False, "int8_gemm": False}+ ds_mod.engage_pinned_denoisers(pipe, eager)+ assert calls == [] and not eager["int8_gemm"]++@pytest.mark.parametrize("offload_active, denoiser_offloaded, expected",[(True, False, False), (True, True, True), (True, None, True), (False, None, False)],