Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: give stored-fp8 denoisers their own planner label instead of dense (#14631)

Daniel Han38570e286f25
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 88c8595e50f..453b8c961b7 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1636,11 +1636,24 @@ def _pipe_denoisers_hold_torchao(pipe: Any) -> bool:
)
+def _holds_fp8_stored(module: Any) -> bool:
+ """Whether ``module`` holds fp8-STORED Linears; codes and scales are buffers, so a parameter scan reads it dense."""
+ if module is None:
+ return False
+ try:
+ from .diffusion_comfy_quant import fp8_stored_layers
+ return fp8_stored_layers(module) > 0
+ except Exception: # noqa: BLE001 - unreadable: today's parameter verdict stands
+ return False
+
+
def _pipe_denoisers_hold_packed(pipe: Any) -> bool:
- """GGUF / int8 weights dequantize a whole Linear per forward: unmeasured by the dense eager table."""
+ """GGUF / int8 / fp8-stored weights dequantize a whole Linear per forward: unmeasured by the dense eager table."""
try:
import torch
for name in ("transformer", "transformer_2", "unconditional_transformer", "unet"):
+ if _holds_fp8_stored(getattr(pipe, name, None)):
+ return True
params = getattr(getattr(pipe, name, None), "parameters", None)
if callable(params) and any(
type(p).__name__ == "GGUFParameter" or p.dtype in (torch.int8, torch.uint8)
@@ -2893,7 +2906,8 @@ def vae_saver_tag(tiling: Any, slicing: Any) -> str:
def denoiser_quant_label(pipe: Any) -> str:
"""Stable label of the denoiser weights' format for the learned-peak key: torchao subclasses, GGUF, int8 / uint8
- packed, or the dense dtype."""
+ packed, fp8-stored (codes in buffers, widened per layer: ``fp8_stored:<float dtypes>#<variant>``), or the dense
+ dtype."""
try:
import torch
@@ -2920,6 +2934,9 @@ def denoiser_quant_label(pipe: Any) -> str:
dtypes.add(str(param.dtype).replace("torch.", ""))
if "int8" in dtypes:
return "packed:int8" + _quant_variant_tag(dits)
+ if any(_holds_fp8_stored(dit) for dit in dits):
+ # Never the dense key: same model file, same float dtype, but a per-layer widen the dense run lacks.
+ return "fp8_stored:" + "+".join(sorted(dtypes)) + _quant_variant_tag(dits)
return "dense:" + "+".join(sorted(dtypes)) if dtypes else "unknown"
except Exception: # noqa: BLE001
return "unknown"
@@ -3081,7 +3098,7 @@ def refine_plan_from_loaded_weights(
Keeps the flat plan's offload hooks and makes whole groups resident (denoiser first, then encoders) within the
safe budget left after the measured peak x margin and the base overhead. No-op for explicit modes, non-CUDA / unified memory, unmeasured families (dense denoisers: the eager-tier table)
- or speed tiers and ``model`` plans. A bf16 or GGUF / int8 packed dense denoiser (no static table measured it)
+ or speed tiers and ``model`` plans. A bf16, GGUF / int8 packed or fp8-stored denoiser (no static table measured it)
places only from a learned peak recorded for its exact build."""
try:
if getattr(plan, "requested_mode", None) != MEMORY_MODE_AUTO:
@@ -3106,9 +3123,17 @@ def refine_plan_from_loaded_weights(
return plan
dense_mib = None if _pipe_denoisers_hold_torchao(pipe) else dit
compute_bytes = _denoiser_compute_bytes(pipe)
- # The dense table covers fp16 / fp32 unpacked weights only: bf16 and packed denoisers use a learned peak alone.
+ # The dense table covers fp16 / fp32 unpacked weights only; everything else uses a learned peak alone.
+ # Stored fp8 layers widen per forward, which no static table priced, even beside torchao layers.
static_seed = not (
- dense_mib is not None and (compute_bytes is None or _pipe_denoisers_hold_packed(pipe))
+ any(
+ _holds_fp8_stored(getattr(pipe, name, None))
+ for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")
+ )
+ or (
+ dense_mib is not None
+ and (compute_bytes is None or _pipe_denoisers_hold_packed(pipe))
+ )
)
quant = learned_peak_label(pipe, attention, model, vae, te, cache, graphs)
try:
diff --git a/studio/backend/tests/test_diffusion_fp8_stored_label.py b/studio/backend/tests/test_diffusion_fp8_stored_label.py
new file mode 100644
index 00000000000..3397e522370
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_fp8_stored_label.py
@@ -0,0 +1,213 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Planner identity of an fp8-STORED denoiser: its own learned-peak label, packed verdict, dense labels unchanged."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+import core.inference.diffusion_memory as dm
+
+torch = pytest.importorskip("torch")
+
+ENVS = (
+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION",
+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE",
+ "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",
+)
+KLEIN_LOADED = {
+ "transformer": (3700, "dit"),
+ "text_encoder": (7673, "text_encoder"),
+ "vae": (160, "other"),
+}
+KLEIN_FLAT = dict(model_dense_mib = 15258, companion_dense_mib = 7820, text_encoder_dense_mib = 7629)
+T4_TOTAL = 15095
+
+
+class _Dit(torch.nn.Module):
+ """A diffusers-like denoiser: ``dtype`` reads the first floating parameter."""
+
+ def __init__(self, dtype, stored: bool):
+ super().__init__()
+ from core.inference.diffusion_comfy_quant import FP8_STORED_ATTR
+ from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class
+
+ linear = torch.nn.Linear(64, 64).to(dtype)
+ if stored:
+ codes = torch.randn(64, 64).to(torch.float8_e4m3fn)
+ linear = native_linear_class()(
+ linear, NATIVE_FP8, codes = codes, scale = torch.ones(64, 1, dtype = torch.float32)
+ )
+ setattr(self, FP8_STORED_ATTR, 1)
+ self.proj = linear
+ self.norm = torch.nn.LayerNorm(64).to(dtype)
+
+ @property
+ def dtype(self):
+ return next(p.dtype for p in self.parameters() if p.is_floating_point())
+
+
+def _pipe(dtype = torch.bfloat16, stored = True):
+ return types.SimpleNamespace(transformer = _Dit(dtype, stored))
+
+
+def test_stored_fp8_holds_its_codes_as_buffers_not_parameters():
+ dit = _Dit(torch.bfloat16, True)
+ assert {p.dtype for p in dit.parameters()} == {torch.bfloat16}
+ assert dict(dit.proj.named_buffers())["weight_q"].dtype == torch.uint8
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
+def test_stored_fp8_denoiser_has_its_own_label(dtype):
+ stored = dm.denoiser_quant_label(_pipe(dtype, True))
+ dense = dm.denoiser_quant_label(_pipe(dtype, False))
+ name = str(dtype).replace("torch.", "")
+ assert dense == f"dense:{name}" # dense keys unchanged: recorded peaks keep matching
+ assert stored.startswith(f"fp8_stored:{name}#")
+ assert stored != dense
+
+
+def test_stored_fp8_label_separates_e4m3_and_e5m2_codes():
+ from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class
+
+ e4 = _pipe()
+ e5 = _pipe()
+ codes = torch.randn(64, 64).to(torch.float8_e5m2)
+ e5.transformer.proj = native_linear_class()(
+ torch.nn.Linear(64, 64).to(torch.bfloat16),
+ NATIVE_FP8,
+ codes = codes,
+ scale = torch.ones(64, 1),
+ )
+ assert dm.denoiser_quant_label(e4) != dm.denoiser_quant_label(e5)
+
+
+def test_stored_fp8_learned_peak_key_does_not_collide_with_the_dense_model():
+ kw = dict(attention = "sdpa", model = "flux-2-klein-4b.safetensors", vae = "full", te = "bf16")
+ stored = dm.learned_peak_label(_pipe(torch.bfloat16, True), **kw)
+ dense = dm.learned_peak_label(_pipe(torch.bfloat16, False), **kw)
+ assert stored != dense
+ assert dense.startswith("dense:bfloat16|")
+
+
+def test_packed_detector_sees_the_stored_fp8_buffers():
+ assert dm._pipe_denoisers_hold_packed(_pipe(torch.float16, True)) is True
+ assert dm._pipe_denoisers_hold_packed(_pipe(torch.float16, False)) is False
+ # Wan 2.2: the second expert alone stored
+ pipe = types.SimpleNamespace(
+ transformer = _Dit(torch.bfloat16, False), transformer_2 = _Dit(torch.bfloat16, True)
+ )
+ assert dm._pipe_denoisers_hold_packed(pipe) is True
+ assert dm.denoiser_quant_label(pipe).startswith("fp8_stored:bfloat16#")
+
+
+def test_unrelated_packed_and_gguf_labels_are_unchanged():
+ class GGUFParameter(torch.nn.Parameter):
+ pass
+
+ lin = torch.nn.Linear(4, 4)
+ lin.weight = GGUFParameter(lin.weight.data, requires_grad = False)
+ assert dm.denoiser_quant_label(types.SimpleNamespace(transformer = lin)) == "gguf"
+ import core.inference.diffusion_small_host as sh
+
+ dit = torch.nn.Sequential(torch.nn.Linear(2048, 2048)).to(torch.bfloat16)
+ sh.quantize_int8_weight_(dit, compute_dtype = torch.float16, work_device = "cpu")
+ assert dm.denoiser_quant_label(types.SimpleNamespace(transformer = dit)).startswith(
+ "packed:int8#"
+ )
+
+
+# ------------------------------------------------------------------------------------------------ refinement / pricing
+
+
+def learned_store(tmp_path, monkeypatch):
+ import core.inference.diffusion_measured_peaks as mp
+
+ for name in ENVS:
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(tmp_path / "peaks.json"))
+ monkeypatch.delenv(mp.LEARNED_PEAKS_ENV, raising = False)
+ mp._cache.update(path = None, mtime = None, data = None)
+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA T4", raising = False)
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(KLEIN_LOADED))
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)
+ return mp
+
+
+def _flat_plan(budget_mib: int = 12758):
+ reserve = dm._reserve_mib("discrete_vram", T4_TOTAL)
+ memory = dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, T4_TOTAL)
+ return dm.plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = memory,
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "flux.2-klein"
+ ),
+ requested_mode = None,
+ **KLEIN_FLAT,
+ )
+
+
+def _refine(pipe, plan):
+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = "flux.2-klein", speed_mode = "off")
+
+
+def test_fp16_dense_control_still_takes_the_static_seed(learned_store):
+ new = _refine(_pipe(torch.float16, False), _flat_plan())
+ assert new.estimates["measured_static_seed"] is True
+ assert new.estimates["measured_runtime_headroom_mib"] == 3072
+
+
+def test_stored_fp8_never_borrows_the_dense_static_seed(learned_store):
+ pipe = _pipe(torch.float16, True)
+ plan = _flat_plan()
+ # nothing learned for this build: the flat plan stands (the dense fp16 table never ran the per-layer widen)
+ assert _refine(pipe, plan) is plan
+ assert pipe._unsloth_peak_identity[1].startswith("fp8_stored:float16#")
+
+
+def test_stored_fp8_places_from_its_own_learned_peak_only(learned_store):
+ pipe = _pipe(torch.float16, True)
+ dense_label = dm.learned_peak_label(_pipe(torch.float16, False))
+ stored_label = dm.learned_peak_label(pipe)
+ kw = dict(family = "flux.2-klein", speed_mode = "off", width = 1024, height = 1024)
+ learned_store.record(**kw, quant = dense_label, peak_mib = 1000)
+ plan = _flat_plan()
+ assert _refine(pipe, plan) is plan
+ learned_store.record(**kw, quant = stored_label, peak_mib = 2000)
+ new = _refine(pipe, plan)
+ assert new is not plan
+ assert new.estimates["measured_static_seed"] is False
+ assert new.estimates["measured_learned_peak_mib"] == 2000
+ assert new.estimates["measured_runtime_headroom_mib"] == 2304 # 2000 x 1.15 -> 256 MiB step
+
+
+def _refine_default(pipe, plan):
+ return dm.refine_plan_from_loaded_weights(
+ pipe, plan, family = "flux.2-klein", speed_mode = "default"
+ )
+
+
+def test_stored_fp8_beside_torchao_layers_never_takes_a_static_seed(learned_store, monkeypatch):
+ # a ComfyUI file whose int8 layers run on torchao while its fp8 layers stay stored: the torchao quant table
+ # (dense_mib None) never priced the stored layers' per-forward widen
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ monkeypatch.setitem(dm._MEASURED_IMAGE_PEAK_MIB, "flux.2-klein", 2000)
+ plan = _flat_plan()
+ dense = _refine_default(_pipe(torch.float16, False), plan)
+ assert (
+ dense.estimates["measured_static_seed"] is True
+ ) # control: a torchao-only load keeps its seed
+ pipe = _pipe(torch.float16, True)
+ assert _refine_default(pipe, plan) is plan
+ kw = dict(family = "flux.2-klein", speed_mode = "default", width = 1024, height = 1024)
+ learned_store.record(**kw, quant = dm.learned_peak_label(pipe), peak_mib = 1000)
+ new = _refine_default(pipe, plan)
+ assert new.estimates["measured_static_seed"] is False
+ assert (
+ new.estimates["measured_runtime_headroom_mib"] == 1280
+ ) # 1000 x 1.15, not the 2000 table seed