Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: give stored-fp8 denoisers their own planner label instead of dense (#14631)
Daniel Han38570e286f25
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 88c8595e50f..453b8c961b7 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1636,11 +1636,24 @@ def _pipe_denoisers_hold_torchao(pipe: Any) -> bool:)+def _holds_fp8_stored(module: Any) -> bool:+ """Whether ``module`` holds fp8-STORED Linears; codes and scales are buffers, so a parameter scan reads it dense."""+ if module is None:+ return False+ try:+ from .diffusion_comfy_quant import fp8_stored_layers+ return fp8_stored_layers(module) > 0+ except Exception: # noqa: BLE001 - unreadable: today's parameter verdict stands+ return False++def _pipe_denoisers_hold_packed(pipe: Any) -> bool:- """GGUF / int8 weights dequantize a whole Linear per forward: unmeasured by the dense eager table."""+ """GGUF / int8 / fp8-stored weights dequantize a whole Linear per forward: unmeasured by the dense eager table."""try:import torchfor name in ("transformer", "transformer_2", "unconditional_transformer", "unet"):+ if _holds_fp8_stored(getattr(pipe, name, None)):+ return Trueparams = getattr(getattr(pipe, name, None), "parameters", None)if callable(params) and any(type(p).__name__ == "GGUFParameter" or p.dtype in (torch.int8, torch.uint8)@@ -2893,7 +2906,8 @@ def vae_saver_tag(tiling: Any, slicing: Any) -> str:def denoiser_quant_label(pipe: Any) -> str:"""Stable label of the denoiser weights' format for the learned-peak key: torchao subclasses, GGUF, int8 / uint8- packed, or the dense dtype."""+ packed, fp8-stored (codes in buffers, widened per layer: ``fp8_stored:<float dtypes>#<variant>``), or the dense+ dtype."""try:import torch@@ -2920,6 +2934,9 @@ def denoiser_quant_label(pipe: Any) -> str:dtypes.add(str(param.dtype).replace("torch.", ""))if "int8" in dtypes:return "packed:int8" + _quant_variant_tag(dits)+ if any(_holds_fp8_stored(dit) for dit in dits):+ # Never the dense key: same model file, same float dtype, but a per-layer widen the dense run lacks.+ return "fp8_stored:" + "+".join(sorted(dtypes)) + _quant_variant_tag(dits)return "dense:" + "+".join(sorted(dtypes)) if dtypes else "unknown"except Exception: # noqa: BLE001return "unknown"@@ -3081,7 +3098,7 @@ def refine_plan_from_loaded_weights(Keeps the flat plan's offload hooks and makes whole groups resident (denoiser first, then encoders) within thesafe budget left after the measured peak x margin and the base overhead. No-op for explicit modes, non-CUDA / unified memory, unmeasured families (dense denoisers: the eager-tier table)- or speed tiers and ``model`` plans. A bf16 or GGUF / int8 packed dense denoiser (no static table measured it)+ or speed tiers and ``model`` plans. A bf16, GGUF / int8 packed or fp8-stored denoiser (no static table measured it)places only from a learned peak recorded for its exact build."""try:if getattr(plan, "requested_mode", None) != MEMORY_MODE_AUTO:@@ -3106,9 +3123,17 @@ def refine_plan_from_loaded_weights(return plandense_mib = None if _pipe_denoisers_hold_torchao(pipe) else ditcompute_bytes = _denoiser_compute_bytes(pipe)- # The dense table covers fp16 / fp32 unpacked weights only: bf16 and packed denoisers use a learned peak alone.+ # The dense table covers fp16 / fp32 unpacked weights only; everything else uses a learned peak alone.+ # Stored fp8 layers widen per forward, which no static table priced, even beside torchao layers.static_seed = not (- dense_mib is not None and (compute_bytes is None or _pipe_denoisers_hold_packed(pipe))+ any(+ _holds_fp8_stored(getattr(pipe, name, None))+ for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")+ )+ or (+ dense_mib is not None+ and (compute_bytes is None or _pipe_denoisers_hold_packed(pipe))+ ))quant = learned_peak_label(pipe, attention, model, vae, te, cache, graphs)try:diff --git a/studio/backend/tests/test_diffusion_fp8_stored_label.py b/studio/backend/tests/test_diffusion_fp8_stored_label.pynew file mode 100644index 00000000000..3397e522370--- /dev/null+++ b/studio/backend/tests/test_diffusion_fp8_stored_label.py@@ -0,0 +1,213 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Planner identity of an fp8-STORED denoiser: its own learned-peak label, packed verdict, dense labels unchanged."""++from __future__ import annotations++import types++import pytest++import core.inference.diffusion_memory as dm++torch = pytest.importorskip("torch")++ENVS = (+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION",+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE",+ "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",+)+KLEIN_LOADED = {+ "transformer": (3700, "dit"),+ "text_encoder": (7673, "text_encoder"),+ "vae": (160, "other"),+}+KLEIN_FLAT = dict(model_dense_mib = 15258, companion_dense_mib = 7820, text_encoder_dense_mib = 7629)+T4_TOTAL = 15095+++class _Dit(torch.nn.Module):+ """A diffusers-like denoiser: ``dtype`` reads the first floating parameter."""++ def __init__(self, dtype, stored: bool):+ super().__init__()+ from core.inference.diffusion_comfy_quant import FP8_STORED_ATTR+ from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class++ linear = torch.nn.Linear(64, 64).to(dtype)+ if stored:+ codes = torch.randn(64, 64).to(torch.float8_e4m3fn)+ linear = native_linear_class()(+ linear, NATIVE_FP8, codes = codes, scale = torch.ones(64, 1, dtype = torch.float32)+ )+ setattr(self, FP8_STORED_ATTR, 1)+ self.proj = linear+ self.norm = torch.nn.LayerNorm(64).to(dtype)++ @property+ def dtype(self):+ return next(p.dtype for p in self.parameters() if p.is_floating_point())+++def _pipe(dtype = torch.bfloat16, stored = True):+ return types.SimpleNamespace(transformer = _Dit(dtype, stored))+++def test_stored_fp8_holds_its_codes_as_buffers_not_parameters():+ dit = _Dit(torch.bfloat16, True)+ assert {p.dtype for p in dit.parameters()} == {torch.bfloat16}+ assert dict(dit.proj.named_buffers())["weight_q"].dtype == torch.uint8++[email protected]("dtype", [torch.bfloat16, torch.float16])+def test_stored_fp8_denoiser_has_its_own_label(dtype):+ stored = dm.denoiser_quant_label(_pipe(dtype, True))+ dense = dm.denoiser_quant_label(_pipe(dtype, False))+ name = str(dtype).replace("torch.", "")+ assert dense == f"dense:{name}" # dense keys unchanged: recorded peaks keep matching+ assert stored.startswith(f"fp8_stored:{name}#")+ assert stored != dense+++def test_stored_fp8_label_separates_e4m3_and_e5m2_codes():+ from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class++ e4 = _pipe()+ e5 = _pipe()+ codes = torch.randn(64, 64).to(torch.float8_e5m2)+ e5.transformer.proj = native_linear_class()(+ torch.nn.Linear(64, 64).to(torch.bfloat16),+ NATIVE_FP8,+ codes = codes,+ scale = torch.ones(64, 1),+ )+ assert dm.denoiser_quant_label(e4) != dm.denoiser_quant_label(e5)+++def test_stored_fp8_learned_peak_key_does_not_collide_with_the_dense_model():+ kw = dict(attention = "sdpa", model = "flux-2-klein-4b.safetensors", vae = "full", te = "bf16")+ stored = dm.learned_peak_label(_pipe(torch.bfloat16, True), **kw)+ dense = dm.learned_peak_label(_pipe(torch.bfloat16, False), **kw)+ assert stored != dense+ assert dense.startswith("dense:bfloat16|")+++def test_packed_detector_sees_the_stored_fp8_buffers():+ assert dm._pipe_denoisers_hold_packed(_pipe(torch.float16, True)) is True+ assert dm._pipe_denoisers_hold_packed(_pipe(torch.float16, False)) is False+ # Wan 2.2: the second expert alone stored+ pipe = types.SimpleNamespace(+ transformer = _Dit(torch.bfloat16, False), transformer_2 = _Dit(torch.bfloat16, True)+ )+ assert dm._pipe_denoisers_hold_packed(pipe) is True+ assert dm.denoiser_quant_label(pipe).startswith("fp8_stored:bfloat16#")+++def test_unrelated_packed_and_gguf_labels_are_unchanged():+ class GGUFParameter(torch.nn.Parameter):+ pass++ lin = torch.nn.Linear(4, 4)+ lin.weight = GGUFParameter(lin.weight.data, requires_grad = False)+ assert dm.denoiser_quant_label(types.SimpleNamespace(transformer = lin)) == "gguf"+ import core.inference.diffusion_small_host as sh++ dit = torch.nn.Sequential(torch.nn.Linear(2048, 2048)).to(torch.bfloat16)+ sh.quantize_int8_weight_(dit, compute_dtype = torch.float16, work_device = "cpu")+ assert dm.denoiser_quant_label(types.SimpleNamespace(transformer = dit)).startswith(+ "packed:int8#"+ )+++# ------------------------------------------------------------------------------------------------ refinement / pricing+++def learned_store(tmp_path, monkeypatch):+ import core.inference.diffusion_measured_peaks as mp++ for name in ENVS:+ monkeypatch.delenv(name, raising = False)+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(tmp_path / "peaks.json"))+ monkeypatch.delenv(mp.LEARNED_PEAKS_ENV, raising = False)+ mp._cache.update(path = None, mtime = None, data = None)+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA T4", raising = False)+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(KLEIN_LOADED))+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)+ return mp+++def _flat_plan(budget_mib: int = 12758):+ reserve = dm._reserve_mib("discrete_vram", T4_TOTAL)+ memory = dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, T4_TOTAL)+ return dm.plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = memory,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "flux.2-klein"+ ),+ requested_mode = None,+ **KLEIN_FLAT,+ )+++def _refine(pipe, plan):+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = "flux.2-klein", speed_mode = "off")+++def test_fp16_dense_control_still_takes_the_static_seed(learned_store):+ new = _refine(_pipe(torch.float16, False), _flat_plan())+ assert new.estimates["measured_static_seed"] is True+ assert new.estimates["measured_runtime_headroom_mib"] == 3072+++def test_stored_fp8_never_borrows_the_dense_static_seed(learned_store):+ pipe = _pipe(torch.float16, True)+ plan = _flat_plan()+ # nothing learned for this build: the flat plan stands (the dense fp16 table never ran the per-layer widen)+ assert _refine(pipe, plan) is plan+ assert pipe._unsloth_peak_identity[1].startswith("fp8_stored:float16#")+++def test_stored_fp8_places_from_its_own_learned_peak_only(learned_store):+ pipe = _pipe(torch.float16, True)+ dense_label = dm.learned_peak_label(_pipe(torch.float16, False))+ stored_label = dm.learned_peak_label(pipe)+ kw = dict(family = "flux.2-klein", speed_mode = "off", width = 1024, height = 1024)+ learned_store.record(**kw, quant = dense_label, peak_mib = 1000)+ plan = _flat_plan()+ assert _refine(pipe, plan) is plan+ learned_store.record(**kw, quant = stored_label, peak_mib = 2000)+ new = _refine(pipe, plan)+ assert new is not plan+ assert new.estimates["measured_static_seed"] is False+ assert new.estimates["measured_learned_peak_mib"] == 2000+ assert new.estimates["measured_runtime_headroom_mib"] == 2304 # 2000 x 1.15 -> 256 MiB step+++def _refine_default(pipe, plan):+ return dm.refine_plan_from_loaded_weights(+ pipe, plan, family = "flux.2-klein", speed_mode = "default"+ )+++def test_stored_fp8_beside_torchao_layers_never_takes_a_static_seed(learned_store, monkeypatch):+ # a ComfyUI file whose int8 layers run on torchao while its fp8 layers stay stored: the torchao quant table+ # (dense_mib None) never priced the stored layers' per-forward widen+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ monkeypatch.setitem(dm._MEASURED_IMAGE_PEAK_MIB, "flux.2-klein", 2000)+ plan = _flat_plan()+ dense = _refine_default(_pipe(torch.float16, False), plan)+ assert (+ dense.estimates["measured_static_seed"] is True+ ) # control: a torchao-only load keeps its seed+ pipe = _pipe(torch.float16, True)+ assert _refine_default(pipe, plan) is plan+ kw = dict(family = "flux.2-klein", speed_mode = "default", width = 1024, height = 1024)+ learned_store.record(**kw, quant = dm.learned_peak_label(pipe), peak_mib = 1000)+ new = _refine_default(pipe, plan)+ assert new.estimates["measured_static_seed"] is False+ assert (+ new.estimates["measured_runtime_headroom_mib"] == 1280+ ) # 1000 x 1.15, not the 2000 table seed