Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: engage the fused int8 GEMM when the placement pins the whole denoiser, and restore torchao weights after an oversized request (#12524)

Daniel Han6a0cd598a92c
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 986211a0436..28f4f711120 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -132,6 +132,7 @@ from .diffusion_memory import (
normalize_memory_mode,
plan_diffusion_memory,
plan_fits_total_capacity,
+ denoisers_pinned_resident,
plan_keeps_transformer_resident,
prequant_seed_device,
raise_on_image_activation_shortfall,
@@ -168,6 +169,7 @@ from .diffusion_speed import (
SPEED_MAX,
SPEED_OFF,
apply_speed_optims,
+ engage_pinned_denoisers,
auto_dynamic_active,
compile_dynamic,
compile_eligible,
@@ -7153,6 +7155,9 @@ class DiffusionBackend:
pipe._unsloth_cuda_graphs = ()
pipe._unsloth_cuda_graph_reason = "offload active"
speed_applied["cuda_graph"] = False
+ # the speed layer saw only the plan; placement may have pinned every denoiser group since
+ if denoisers_pinned_resident(pipe):
+ engage_pinned_denoisers(pipe, speed_applied, logger)
# Per-control provenance for status. cpu_offload=False is the unset default, so only True is
# explicit.
@@ -9022,6 +9027,8 @@ class DiffusionBackend:
and _denoiser_hooked(state.pipe),
logger = logger,
)
+ if denoisers_pinned_resident(state.pipe):
+ engage_pinned_denoisers(state.pipe, speed_applied, logger)
if getattr(getattr(state.pipe, "vae", None), "_unsloth_fp16_decode", False):
speed_applied["vae_fp16_decode"] = True
object.__setattr__(state, "speed_mode", SPEED_DEFAULT)
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index c564794f0ea..b1b14dbf24c 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -2957,6 +2957,34 @@ def _background_pin_enabled() -> bool:
)
+_DENOISER_NAMES = ("transformer", "transformer_2", "unconditional_transformer", "unet")
+
+
+def denoisers_pinned_resident(pipe: Any) -> bool:
+ """Every denoiser unhooked or with all its offload groups pinned, at least one pinned. The hooks stay so an
+ oversized request can stream it again (``release_resident_groups``)."""
+ pinned = False
+ for name in _DENOISER_NAMES:
+ module = getattr(pipe, name, None)
+ if module is None:
+ continue
+ try:
+ if any(getattr(sub, "_hf_hook", None) is not None for sub in module.modules()):
+ return False # accelerate (model / sequential) offload moves it
+ except Exception: # noqa: BLE001 - unreadable: assume it moves
+ return False
+ groups = _offload_groups(module)
+ if not groups:
+ registry = getattr(module, "_diffusers_hook", None)
+ if any("offload" in str(key) for key in (getattr(registry, "hooks", None) or {})):
+ return False
+ continue
+ if not all(getattr(g, "_unsloth_resident", False) for g in groups):
+ return False
+ pinned = True
+ return pinned
+
+
def _offload_groups(module: Any) -> list:
"""The diffusers offload groups hooked under ``module``, in registration (block) order."""
try:
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index f0a8071d74c..fe857916a1f 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -602,6 +602,38 @@ def apply_speed_optims(
return applied
+def engage_pinned_denoisers(
+ pipe: Any,
+ applied: dict,
+ logger: Any = None,
+) -> dict:
+ """Install the int8 GEMM the plan refused once every denoiser group is pinned (before the first forward). Graphs
+ stay off: an oversized request can still stream the groups, and the hooks' copy-stream wait breaks capture."""
+ if getattr(pipe, "_unsloth_cuda_graph_reason", None) == "offload active":
+ try:
+ pipe._unsloth_cuda_graph_reason = "denoiser pinned resident under offload hooks"
+ except Exception: # noqa: BLE001
+ pass
+ if not applied.get("compiled") or applied.get("int8_gemm"):
+ return applied
+ try:
+ from .diffusion_int8_gemm import install as install_int8_gemm
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "int8 fused-dequant gemm", exc)
+ return applied
+ for transformer in _denoiser_dits(pipe):
+ try:
+ transformer._unsloth_int8_gemm = install_int8_gemm(
+ transformer, logger, offload_active = False
+ )
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "int8 fused-dequant gemm", exc)
+ applied["int8_gemm"] = any(
+ bool(getattr(t, "_unsloth_int8_gemm", 0)) for t in _denoiser_dits(pipe)
+ )
+ return applied
+
+
def fp16_unet_offloaded(target: Any, pipe: Any, *, offload_active: bool) -> bool:
"""An offloaded fp16 U-Net stays eager: fused QKV costs more transfer than compile saves (L4: 5.43 vs 4.86 s)."""
return (
diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.py
index 2100e002b3a..c5b1abc7a68 100644
--- a/studio/backend/tests/test_diffusion_int8_gemm.py
+++ b/studio/backend/tests/test_diffusion_int8_gemm.py
@@ -343,6 +343,83 @@ def test_small_m_and_misaligned_keep_stock(forced):
assert g8.call_count() == before
+@needs_cuda
+def test_pinned_group_offloaded_denoiser_takes_the_gemm_and_survives_release(forced, monkeypatch):
+ """Every group pinned: the GEMM engages after placement, bit-identical to torchao through release and restore."""
+ pytest.importorskip("diffusers.hooks")
+ from diffusers.hooks import apply_group_offloading
+ import types as _types
+
+ from core.inference import diffusion_memory as dm
+ from core.inference import diffusion_speed as ds
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)
+
+ class Dit(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(256, 512).to(torch.bfloat16)
+ self.blocks = torch.nn.ModuleList(
+ torch.nn.Sequential(
+ torch.nn.Linear(512, 1024), torch.nn.GELU(), torch.nn.Linear(1024, 512)
+ )
+ for _ in range(4)
+ )
+
+ def forward(self, x):
+ x = self.proj_in(x)
+ for block in self.blocks:
+ x = x + block(x)
+ return x
+
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ torch.manual_seed(0)
+ dit = Dit().cuda().to(torch.bfloat16)
+ cfg = Int8DynamicActivationInt8WeightConfig(set_inductor_config = False)
+ if not hasattr(cfg, "version"):
+ pytest.skip("torchao without config versions")
+ cfg.version = (
+ 2 # what streams under diffusers group offloading (v1's subclass rejects is_pinned)
+ )
+ quantize_(dit.blocks, cfg)
+ x = torch.randn(300, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = dit(x)
+ dit.to("cpu")
+ kwargs = dict(
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ record_stream = True,
+ non_blocking = True,
+ )
+ apply_group_offloading(dit, **dm._torchao_group_offload_kwargs(dit, kwargs, [0]))
+ pipe = _types.SimpleNamespace(transformer = dit, components = {"transformer": dit})
+ assert dm._keep_groups_resident(dit, 1024, "cuda") > 0
+ assert dm.denoisers_pinned_resident(pipe)
+
+ monkeypatch.setattr(ds, "_denoiser_dits", lambda p: [dit])
+ applied = {"compiled": True, "int8_gemm": False}
+ ds.engage_pinned_denoisers(pipe, applied)
+ assert applied["int8_gemm"] and dit._unsloth_int8_gemm == 8
+ # inference_mode rejects moving torchao weights
+ with torch.no_grad():
+ before = g8.call_count()
+ assert torch.equal(dit(x), ref)
+ assert g8.call_count() == before + 8
+ restore = dm.release_resident_groups(pipe, 1024)
+ assert restore is not None and not dm.denoisers_pinned_resident(pipe)
+ for _ in range(2): # first streamed forward traces the prefetch order
+ assert torch.equal(dit(x), ref)
+ restore()
+ assert dm.denoisers_pinned_resident(pipe)
+ assert torch.equal(dit(x), ref)
+ g8.uninstall(dit)
+
+
@needs_cuda
def test_fused_mlp_down_projection_traces_without_breaks(forced, monkeypatch):
"""The fused MLP's down projection reaches the GEMM through ``linear_from_q`` inside the compiled block: no host
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
index bfa5ffd9516..5e2ce8d365b 100644
--- a/studio/backend/tests/test_diffusion_measured_placement.py
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -496,6 +496,54 @@ def test_generate_releases_and_restores_resident_groups():
assert "pipe._unsloth_measured_reserve = (" in src
+def _pinned_pipe(
+ monkeypatch,
+ resident_flags,
+ hf_hook = False,
+):
+ torch = pytest.importorskip("torch")
+ net = torch.nn.Linear(4, 4)
+ if hf_hook:
+ net._hf_hook = object()
+ groups = [types.SimpleNamespace(_unsloth_resident = flag) for flag in resident_flags]
+ monkeypatch.setattr(dm, "_offload_groups", lambda m: groups if m is net else [])
+ return types.SimpleNamespace(transformer = net, text_encoder = torch.nn.Linear(4, 4))
+
+
+def test_denoiser_residency_follows_the_final_placement(monkeypatch):
+ """Residency reads the placement (16 GB: all 35 groups pinned), not the plan's stream flag."""
+ assert dm.denoisers_pinned_resident(_pinned_pipe(monkeypatch, [True] * 35))
+ assert not dm.denoisers_pinned_resident(_pinned_pipe(monkeypatch, [True] * 23 + [False] * 12))
+ assert not dm.denoisers_pinned_resident(_pinned_pipe(monkeypatch, [True] * 35, hf_hook = True))
+ assert not dm.denoisers_pinned_resident(_pinned_pipe(monkeypatch, []))
+
+
+def test_pinned_denoiser_reads_real_group_offload_hooks(monkeypatch):
+ """Real group offloading: pinned -> resident, released -> moving, restored -> resident."""
+ torch, net = _cuda_offload_model()
+ from diffusers.hooks import apply_group_offloading
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ record_stream = True,
+ non_blocking = True,
+ )
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ assert not dm.denoisers_pinned_resident(pipe) # hooked, nothing pinned: streams
+ assert dm._keep_groups_resident(net, 1024, "cuda") > 0
+ assert dm.denoisers_pinned_resident(pipe)
+ restore = dm.release_resident_groups(pipe, 1024)
+ assert restore is not None and not dm.denoisers_pinned_resident(pipe)
+ restore()
+ assert dm.denoisers_pinned_resident(pipe)
+
+
def test_torchao_groups_stay_on_device_after_release_and_restore(monkeypatch):
torch, _ = _cuda_offload_model()
pytest.importorskip("torchao")
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index a3707ee95b9..ab05869541a 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -2357,6 +2357,37 @@ def test_family_compiles_regionally_closes_the_dynamo_import_window_first(monkey
assert events[:2] == ["guard", "probe"]
+def test_pinned_denoiser_engages_the_int8_gemm_after_placement(monkeypatch):
+ """Pinned after placement: the GEMM installs with offload_active False, only on a compiled DiT."""
+ calls = []
+ fake = types.ModuleType("core.inference.diffusion_int8_gemm")
+
+ def _install(
+ transformer,
+ logger = None,
+ offload_active = False,
+ ):
+ calls.append(offload_active)
+ return 0 if offload_active else 60
+
+ fake.install = _install
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_int8_gemm", fake)
+ dit = types.SimpleNamespace()
+ pipe = types.SimpleNamespace(_unsloth_cuda_graph_reason = "offload active")
+ monkeypatch.setattr(ds_mod, "_denoiser_dits", lambda p: [dit])
+
+ applied = {"compiled": True, "int8_gemm": False, "cuda_graph": False}
+ ds_mod.engage_pinned_denoisers(pipe, applied)
+ assert calls == [False] and applied["int8_gemm"] and dit._unsloth_int8_gemm == 60
+ assert not applied["cuda_graph"]
+ assert pipe._unsloth_cuda_graph_reason == "denoiser pinned resident under offload hooks"
+
+ calls.clear()
+ eager = {"compiled": False, "int8_gemm": False}
+ ds_mod.engage_pinned_denoisers(pipe, eager)
+ assert calls == [] and not eager["int8_gemm"]
+
+
@pytest.mark.parametrize(
"offload_active, denoiser_offloaded, expected",
[(True, False, False), (True, True, True), (True, None, True), (False, None, False)],