Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio FLUX.3: price float32 compute on fp16-only GPUs from the measured activation table (x2), so a GGUF Q4 denoiser stays resident on a T4 (#17219)

Daniel Han3977ee2fb823
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index a0b05040ed5..1ee71a0e262 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -214,6 +214,7 @@ from .diffusion_memory import (
OFFLOAD_STREAMING,
apply_memory_plan,
calibrated_image_activation,
+ fp32_calibrated,
engage_vae_tiling,
estimate_gguf_resident_mib,
estimate_image_runtime_mib,
@@ -2983,25 +2984,30 @@ def _calibrated_activation(
target: Any,
*,
unpriced_references_ok: bool = False,
+ fp32_scaled: bool = False,
) -> Any:
"""Measured activations apply only where measured: NVIDIA with a sub-quadratic attention kernel.
- ``unpriced_references_ok``: skip the reference rule; the caller sizes references via measured_request_extra_mib."""
+ ``unpriced_references_ok``: skip the reference rule; the caller sizes references via measured_request_extra_mib.
+ ``fp32_scaled``: a float32 ``fp32_calibrated`` load reads the table scaled 2x; False keeps it unmeasured (post-load seeds)."""
if getattr(target, "backend", None) != "cuda" or getattr(target, "vendor", None) != "nvidia":
return None
# Measured without input images: references the guard cannot size (no reference_resolutions) would overflow.
# FLUX.3 can: every reference is resized to 1 MP and the request guard charges those pixels on top.
if not unpriced_references_ok and _references_unpriced(fam):
return None
- # Measured at 16-bit only; fp32-promoted families (Z-Image pre-Ampere) do not qualify.
+ # Measured at 16-bit; fp32-promoted families (Z-Image pre-Ampere) do not qualify, except the scaled FLUX.3 table.
compute = _resolve_diffusion_compute_dtype(fam, getattr(target, "dtype", None))
- if (_float_load_itemsize(compute) or 2) > 2:
+ compute_bytes = _float_load_itemsize(compute) or 2
+ if compute_bytes > 2 and not (fp32_scaled and fp32_calibrated(getattr(fam, "name", None))):
return None
requested = _PLANNED_SPEED_MODE.get(_SPEED_UNKNOWN)
try:
max_speed = requested is _SPEED_UNKNOWN or normalize_speed_mode(requested) == SPEED_MAX
except ValueError:
max_speed = True
- activation = calibrated_image_activation(getattr(fam, "name", None), max_speed = max_speed)
+ activation = calibrated_image_activation(
+ getattr(fam, "name", None), max_speed = max_speed, compute_bytes = compute_bytes
+ )
if activation is None:
return None
try:
@@ -13413,12 +13419,21 @@ class DiffusionBackend:
runtime_headroom_mib = runtime_headroom,
requested_mode = memory_mode,
explicit_offload = cpu_offload,
- calibrated_activation = _calibrated_activation(fam, target)
+ calibrated_activation = _calibrated_activation(fam, target, fp32_scaled = True)
if probe_calibration
else activation,
)
if resident_replan:
+ # Float32 compute widens only the scaled FLUX.3 tiers; other families keep the 16-bit tiers here.
+ replan_compute_bytes = (
+ _float_load_itemsize(
+ _resolve_diffusion_compute_dtype(fam, getattr(target, "dtype", None))
+ )
+ or 2
+ if fp32_calibrated(fam.name)
+ else 2
+ )
def can_stop(memory: DeviceMemory) -> bool:
if (
@@ -13430,8 +13445,12 @@ class DiffusionBackend:
# Check flat and both measured tiers without running the allocating SDPA probe during sampling.
for activation in (
None,
- calibrated_image_activation(fam.name, max_speed = False),
- calibrated_image_activation(fam.name, max_speed = True),
+ calibrated_image_activation(
+ fam.name, max_speed = False, compute_bytes = replan_compute_bytes
+ ),
+ calibrated_image_activation(
+ fam.name, max_speed = True, compute_bytes = replan_compute_bytes
+ ),
):
candidate = _plan(
model_dense_mib, memory, probe_calibration = False, activation = activation
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index f3bbdd688e8..23c2a4341fb 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1744,17 +1744,37 @@ def _resident_only_hint(family: Optional[str]) -> bool:
def calibrated_image_activation(
- family: Optional[str], *, max_speed: bool = True
+ family: Optional[str],
+ *,
+ max_speed: bool = True,
+ compute_bytes: int = 2,
) -> Optional[CalibratedImageActivation]:
+ """``compute_bytes`` 4 (float32 compute) scales every column 2x, only for ``FP32_CALIBRATED_FAMILIES``: the
+ tables were measured at 16-bit, and every other fp32-promoted family keeps the flat estimate (None)."""
measured = _MEASURED_IMAGE_ACTIVATION_MIB.get(str(family or ""))
if measured is None:
return None
+ widen = 1.0
+ if int(compute_bytes or 2) > 2:
+ if not fp32_calibrated(family):
+ return None
+ widen = int(compute_bytes) / 2.0
return CalibratedImageActivation(
- *(int(v * _ACTIVATION_MARGIN) for v in measured[1 if max_speed else 0]),
+ *(int(v * _ACTIVATION_MARGIN * widen) for v in measured[1 if max_speed else 0]),
resident_only = str(family or "") in _RESIDENT_ONLY_CALIBRATED,
)
+# fp16-only GPUs: price float32 compute at 2x the 16-bit table (FLUX.3 T4 peaks measured below that); without it the
+# flat 8 GiB estimate streams even a 3.5 GiB GGUF denoiser through pageable host memory.
+FP32_CALIBRATED_ENV = "UNSLOTH_DIFFUSION_FP32_CALIBRATED"
+FP32_CALIBRATED_FAMILIES = frozenset({"flux.3-klein", "flux.3-klein-base"})
+
+
+def fp32_calibrated(family: Optional[str]) -> bool:
+ return str(family or "") in FP32_CALIBRATED_FAMILIES and not _env_off(FP32_CALIBRATED_ENV)
+
+
# ComfyUI's minimum_inference_memory() (comfy/model_management.py); activations are priced separately here.
COMFY_MIN_INFERENCE_MIB = 800
COMFY_EXTRA_RESERVED_MIB = 400
diff --git a/studio/backend/tests/test_diffusion_flux3_fp32_calibrated.py b/studio/backend/tests/test_diffusion_flux3_fp32_calibrated.py
new file mode 100644
index 00000000000..27d376c727b
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_flux3_fp32_calibrated.py
@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.3 on fp16-only GPUs (float32 compute): the 16-bit activation table, scaled 2x, prices the plan instead of
+the flat 8 GiB, so a GGUF denoiser that fits stays resident instead of streaming every step."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+import core.inference.diffusion_memory as dm
+from core.inference.diffusion_memory import (
+ OFFLOAD_GROUP,
+ DeviceMemory,
+ calibrated_image_activation,
+ plan_diffusion_memory,
+)
+
+# Planner inputs (MiB) logged by a real FLUX.3 klein Q4_K_M load on a Kaggle T4 (float32 compute, dense encoder).
+F3_GGUF_T4 = dict(model_dense_mib = 20_920, companion_dense_mib = 17_249, text_encoder_dense_mib = 16_928)
+GIB = 1024
+
+
+def _t4():
+ # safe_device_budget_mib 12_758 on that load
+ return DeviceMemory("cuda", "cuda:0", "discrete_vram", 14_806, 15_095)
+
+
+def _plan(act, memory = None):
+ return plan_diffusion_memory(
+ target = types.SimpleNamespace(
+ device = "cuda", backend = "cuda", supports_model_cpu_offload = True
+ ),
+ device_memory = memory or _t4(),
+ runtime_headroom_mib = 8192,
+ calibrated_activation = act,
+ **F3_GGUF_T4,
+ )
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ monkeypatch.delenv(dm.FP32_CALIBRATED_ENV, raising = False)
+
+
[email protected]("family", sorted(dm.FP32_CALIBRATED_FAMILIES))
[email protected]("max_speed", (False, True))
+def test_float32_compute_doubles_every_column(family, max_speed):
+ half = calibrated_image_activation(family, max_speed = max_speed)
+ full = calibrated_image_activation(family, max_speed = max_speed, compute_bytes = 4)
+ assert half == calibrated_image_activation(family, max_speed = max_speed, compute_bytes = 2)
+ for name in (
+ "text_encoder_mib",
+ "denoise_mib",
+ "decode_mib",
+ "tiled_decode_mib",
+ "max_canvas_mib",
+ ):
+ assert abs(getattr(full, name) - 2 * getattr(half, name)) <= 2
+ assert full.resident_only == half.resident_only
+
+
+def test_other_families_stay_unmeasured_at_float32():
+ for family in dm._MEASURED_IMAGE_ACTIVATION_MIB:
+ if family in dm.FP32_CALIBRATED_FAMILIES:
+ continue
+ assert calibrated_image_activation(family, compute_bytes = 4) is None
+ assert calibrated_image_activation(family) is not None
+
+
+def test_kill_switch_restores_the_flat_plan(monkeypatch):
+ monkeypatch.setenv(dm.FP32_CALIBRATED_ENV, "0")
+ assert calibrated_image_activation("flux.3-klein", compute_bytes = 4) is None
+ assert calibrated_image_activation("flux.3-klein") is not None
+
+
+def test_t4_gguf_keeps_the_denoiser_resident():
+ flat = _plan(None)
+ assert flat.offload_policy == OFFLOAD_GROUP and flat.stream_transformer
+ act = calibrated_image_activation("flux.3-klein", compute_bytes = 4)
+ plan = _plan(act)
+ assert not plan.stream_transformer
+ assert plan.stream_text_encoders
+ assert plan.estimates["calibrated_headroom_mib"] == act.headroom(plan.vae_tiling)
+
+
+def test_t4_under_a_6_gib_budget_still_streams():
+ memory = DeviceMemory("cuda", "cuda:0", "discrete_vram", 6 * GIB, 15_095)
+ act = calibrated_image_activation("flux.3-klein", compute_bytes = 4)
+ assert _plan(act, memory).stream_transformer
+
+
+def test_calibrated_activation_routes_float32_flux3_only(monkeypatch):
+ import torch
+
+ import core.inference.diffusion as d
+ import core.inference.diffusion_fp16_guard as guard
+ from core.inference.diffusion_families import detect_family
+
+ monkeypatch.setattr(d, "sdpa_subquadratic_confirmed", lambda target: True)
+ monkeypatch.setattr(guard, "fp16_promotes_to_fp32", lambda fam: True)
+ flux3 = detect_family("unsloth/flux-3-image-klein-GGUF")
+ zimage = detect_family("unsloth/Z-Image-Turbo-GGUF")
+ assert flux3.name == "flux.3-klein" and flux3.fp16_incompatible
+
+ def on(dtype):
+ return types.SimpleNamespace(backend = "cuda", vendor = "nvidia", dtype = dtype)
+
+ for dtype in (torch.float16, torch.float32):
+ act = d._calibrated_activation(flux3, on(dtype), fp32_scaled = True)
+ assert act == calibrated_image_activation("flux.3-klein", compute_bytes = 4)
+ # the post-load phase seeds (the default) read the table unscaled, so they keep a float32 load unmeasured
+ assert d._calibrated_activation(flux3, on(dtype)) is None
+ assert d._calibrated_activation(zimage, on(dtype), fp32_scaled = True) is None
+ assert d._calibrated_activation(flux3, on(torch.bfloat16)) == calibrated_image_activation(
+ "flux.3-klein"
+ )
+ monkeypatch.setenv(dm.FP32_CALIBRATED_ENV, "0")
+ assert d._calibrated_activation(flux3, on(torch.float32), fp32_scaled = True) is None
+ assert d._calibrated_activation(flux3, on(torch.bfloat16)) is not None
+
+
+def test_only_the_load_plan_reads_the_scaled_table():
+ import ast
+ import inspect
+
+ import core.inference.diffusion as d
+
+ src = ast.unparse(ast.parse(inspect.getsource(d.DiffusionBackend)))
+ assert "calibrated_activation=_calibrated_activation(fam, target, fp32_scaled=True)" in src
+ assert src.count("fp32_scaled=True") == 1