Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio FLUX.3: price float32 compute on fp16-only GPUs from the measured activation table (x2), so a GGUF Q4 denoiser stays resident on a T4 (#17219)
Daniel Han3977ee2fb823
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex a0b05040ed5..1ee71a0e262 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -214,6 +214,7 @@ from .diffusion_memory import (OFFLOAD_STREAMING,apply_memory_plan,calibrated_image_activation,+ fp32_calibrated,engage_vae_tiling,estimate_gguf_resident_mib,estimate_image_runtime_mib,@@ -2983,25 +2984,30 @@ def _calibrated_activation(target: Any,*,unpriced_references_ok: bool = False,+ fp32_scaled: bool = False,) -> Any:"""Measured activations apply only where measured: NVIDIA with a sub-quadratic attention kernel.- ``unpriced_references_ok``: skip the reference rule; the caller sizes references via measured_request_extra_mib."""+ ``unpriced_references_ok``: skip the reference rule; the caller sizes references via measured_request_extra_mib.+ ``fp32_scaled``: a float32 ``fp32_calibrated`` load reads the table scaled 2x; False keeps it unmeasured (post-load seeds)."""if getattr(target, "backend", None) != "cuda" or getattr(target, "vendor", None) != "nvidia":return None# Measured without input images: references the guard cannot size (no reference_resolutions) would overflow.# FLUX.3 can: every reference is resized to 1 MP and the request guard charges those pixels on top.if not unpriced_references_ok and _references_unpriced(fam):return None- # Measured at 16-bit only; fp32-promoted families (Z-Image pre-Ampere) do not qualify.+ # Measured at 16-bit; fp32-promoted families (Z-Image pre-Ampere) do not qualify, except the scaled FLUX.3 table.compute = _resolve_diffusion_compute_dtype(fam, getattr(target, "dtype", None))- if (_float_load_itemsize(compute) or 2) > 2:+ compute_bytes = _float_load_itemsize(compute) or 2+ if compute_bytes > 2 and not (fp32_scaled and fp32_calibrated(getattr(fam, "name", None))):return Nonerequested = _PLANNED_SPEED_MODE.get(_SPEED_UNKNOWN)try:max_speed = requested is _SPEED_UNKNOWN or normalize_speed_mode(requested) == SPEED_MAXexcept ValueError:max_speed = True- activation = calibrated_image_activation(getattr(fam, "name", None), max_speed = max_speed)+ activation = calibrated_image_activation(+ getattr(fam, "name", None), max_speed = max_speed, compute_bytes = compute_bytes+ )if activation is None:return Nonetry:@@ -13413,12 +13419,21 @@ class DiffusionBackend:runtime_headroom_mib = runtime_headroom,requested_mode = memory_mode,explicit_offload = cpu_offload,- calibrated_activation = _calibrated_activation(fam, target)+ calibrated_activation = _calibrated_activation(fam, target, fp32_scaled = True)if probe_calibrationelse activation,)if resident_replan:+ # Float32 compute widens only the scaled FLUX.3 tiers; other families keep the 16-bit tiers here.+ replan_compute_bytes = (+ _float_load_itemsize(+ _resolve_diffusion_compute_dtype(fam, getattr(target, "dtype", None))+ )+ or 2+ if fp32_calibrated(fam.name)+ else 2+ )def can_stop(memory: DeviceMemory) -> bool:if (@@ -13430,8 +13445,12 @@ class DiffusionBackend:# Check flat and both measured tiers without running the allocating SDPA probe during sampling.for activation in (None,- calibrated_image_activation(fam.name, max_speed = False),- calibrated_image_activation(fam.name, max_speed = True),+ calibrated_image_activation(+ fam.name, max_speed = False, compute_bytes = replan_compute_bytes+ ),+ calibrated_image_activation(+ fam.name, max_speed = True, compute_bytes = replan_compute_bytes+ ),):candidate = _plan(model_dense_mib, memory, probe_calibration = False, activation = activationdiff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex f3bbdd688e8..23c2a4341fb 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1744,17 +1744,37 @@ def _resident_only_hint(family: Optional[str]) -> bool:def calibrated_image_activation(- family: Optional[str], *, max_speed: bool = True+ family: Optional[str],+ *,+ max_speed: bool = True,+ compute_bytes: int = 2,) -> Optional[CalibratedImageActivation]:+ """``compute_bytes`` 4 (float32 compute) scales every column 2x, only for ``FP32_CALIBRATED_FAMILIES``: the+ tables were measured at 16-bit, and every other fp32-promoted family keeps the flat estimate (None)."""measured = _MEASURED_IMAGE_ACTIVATION_MIB.get(str(family or ""))if measured is None:return None+ widen = 1.0+ if int(compute_bytes or 2) > 2:+ if not fp32_calibrated(family):+ return None+ widen = int(compute_bytes) / 2.0return CalibratedImageActivation(- *(int(v * _ACTIVATION_MARGIN) for v in measured[1 if max_speed else 0]),+ *(int(v * _ACTIVATION_MARGIN * widen) for v in measured[1 if max_speed else 0]),resident_only = str(family or "") in _RESIDENT_ONLY_CALIBRATED,)+# fp16-only GPUs: price float32 compute at 2x the 16-bit table (FLUX.3 T4 peaks measured below that); without it the+# flat 8 GiB estimate streams even a 3.5 GiB GGUF denoiser through pageable host memory.+FP32_CALIBRATED_ENV = "UNSLOTH_DIFFUSION_FP32_CALIBRATED"+FP32_CALIBRATED_FAMILIES = frozenset({"flux.3-klein", "flux.3-klein-base"})+++def fp32_calibrated(family: Optional[str]) -> bool:+ return str(family or "") in FP32_CALIBRATED_FAMILIES and not _env_off(FP32_CALIBRATED_ENV)++# ComfyUI's minimum_inference_memory() (comfy/model_management.py); activations are priced separately here.COMFY_MIN_INFERENCE_MIB = 800COMFY_EXTRA_RESERVED_MIB = 400diff --git a/studio/backend/tests/test_diffusion_flux3_fp32_calibrated.py b/studio/backend/tests/test_diffusion_flux3_fp32_calibrated.pynew file mode 100644index 00000000000..27d376c727b--- /dev/null+++ b/studio/backend/tests/test_diffusion_flux3_fp32_calibrated.py@@ -0,0 +1,133 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.3 on fp16-only GPUs (float32 compute): the 16-bit activation table, scaled 2x, prices the plan instead of+the flat 8 GiB, so a GGUF denoiser that fits stays resident instead of streaming every step."""++from __future__ import annotations++import types++import pytest++import core.inference.diffusion_memory as dm+from core.inference.diffusion_memory import (+ OFFLOAD_GROUP,+ DeviceMemory,+ calibrated_image_activation,+ plan_diffusion_memory,+)++# Planner inputs (MiB) logged by a real FLUX.3 klein Q4_K_M load on a Kaggle T4 (float32 compute, dense encoder).+F3_GGUF_T4 = dict(model_dense_mib = 20_920, companion_dense_mib = 17_249, text_encoder_dense_mib = 16_928)+GIB = 1024+++def _t4():+ # safe_device_budget_mib 12_758 on that load+ return DeviceMemory("cuda", "cuda:0", "discrete_vram", 14_806, 15_095)+++def _plan(act, memory = None):+ return plan_diffusion_memory(+ target = types.SimpleNamespace(+ device = "cuda", backend = "cuda", supports_model_cpu_offload = True+ ),+ device_memory = memory or _t4(),+ runtime_headroom_mib = 8192,+ calibrated_activation = act,+ **F3_GGUF_T4,+ )++[email protected](autouse = True)+def _clean_env(monkeypatch):+ monkeypatch.delenv(dm.FP32_CALIBRATED_ENV, raising = False)++[email protected]("family", sorted(dm.FP32_CALIBRATED_FAMILIES))[email protected]("max_speed", (False, True))+def test_float32_compute_doubles_every_column(family, max_speed):+ half = calibrated_image_activation(family, max_speed = max_speed)+ full = calibrated_image_activation(family, max_speed = max_speed, compute_bytes = 4)+ assert half == calibrated_image_activation(family, max_speed = max_speed, compute_bytes = 2)+ for name in (+ "text_encoder_mib",+ "denoise_mib",+ "decode_mib",+ "tiled_decode_mib",+ "max_canvas_mib",+ ):+ assert abs(getattr(full, name) - 2 * getattr(half, name)) <= 2+ assert full.resident_only == half.resident_only+++def test_other_families_stay_unmeasured_at_float32():+ for family in dm._MEASURED_IMAGE_ACTIVATION_MIB:+ if family in dm.FP32_CALIBRATED_FAMILIES:+ continue+ assert calibrated_image_activation(family, compute_bytes = 4) is None+ assert calibrated_image_activation(family) is not None+++def test_kill_switch_restores_the_flat_plan(monkeypatch):+ monkeypatch.setenv(dm.FP32_CALIBRATED_ENV, "0")+ assert calibrated_image_activation("flux.3-klein", compute_bytes = 4) is None+ assert calibrated_image_activation("flux.3-klein") is not None+++def test_t4_gguf_keeps_the_denoiser_resident():+ flat = _plan(None)+ assert flat.offload_policy == OFFLOAD_GROUP and flat.stream_transformer+ act = calibrated_image_activation("flux.3-klein", compute_bytes = 4)+ plan = _plan(act)+ assert not plan.stream_transformer+ assert plan.stream_text_encoders+ assert plan.estimates["calibrated_headroom_mib"] == act.headroom(plan.vae_tiling)+++def test_t4_under_a_6_gib_budget_still_streams():+ memory = DeviceMemory("cuda", "cuda:0", "discrete_vram", 6 * GIB, 15_095)+ act = calibrated_image_activation("flux.3-klein", compute_bytes = 4)+ assert _plan(act, memory).stream_transformer+++def test_calibrated_activation_routes_float32_flux3_only(monkeypatch):+ import torch++ import core.inference.diffusion as d+ import core.inference.diffusion_fp16_guard as guard+ from core.inference.diffusion_families import detect_family++ monkeypatch.setattr(d, "sdpa_subquadratic_confirmed", lambda target: True)+ monkeypatch.setattr(guard, "fp16_promotes_to_fp32", lambda fam: True)+ flux3 = detect_family("unsloth/flux-3-image-klein-GGUF")+ zimage = detect_family("unsloth/Z-Image-Turbo-GGUF")+ assert flux3.name == "flux.3-klein" and flux3.fp16_incompatible++ def on(dtype):+ return types.SimpleNamespace(backend = "cuda", vendor = "nvidia", dtype = dtype)++ for dtype in (torch.float16, torch.float32):+ act = d._calibrated_activation(flux3, on(dtype), fp32_scaled = True)+ assert act == calibrated_image_activation("flux.3-klein", compute_bytes = 4)+ # the post-load phase seeds (the default) read the table unscaled, so they keep a float32 load unmeasured+ assert d._calibrated_activation(flux3, on(dtype)) is None+ assert d._calibrated_activation(zimage, on(dtype), fp32_scaled = True) is None+ assert d._calibrated_activation(flux3, on(torch.bfloat16)) == calibrated_image_activation(+ "flux.3-klein"+ )+ monkeypatch.setenv(dm.FP32_CALIBRATED_ENV, "0")+ assert d._calibrated_activation(flux3, on(torch.float32), fp32_scaled = True) is None+ assert d._calibrated_activation(flux3, on(torch.bfloat16)) is not None+++def test_only_the_load_plan_reads_the_scaled_table():+ import ast+ import inspect++ import core.inference.diffusion as d++ src = ast.unparse(ast.parse(inspect.getsource(d.DiffusionBackend)))+ assert "calibrated_activation=_calibrated_activation(fam, target, fp32_scaled=True)" in src+ assert src.count("fp32_scaled=True") == 1