Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: price a streamed Qwen-Image-2.1 edit's reference from its token count, not the flat allowance (A100 8 GiB edit -15%, ahead of the baseline) (#17233)
Daniel Han8efcf85adf96
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 415536b41e0..84cef6d0ed9 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -14951,6 +14951,7 @@ class DiffusionBackend:guard_release_mib, guard_capture_gen = 0, Noneguard_batch = 1guard_condition_pixels = 0+ guard_reference_pixels = 0try:# Forward dims, not sliders: conditioned workflows size from the input image; same derivation as the compile-cache registration below.@@ -14983,6 +14984,11 @@ class DiffusionBackend:)))+ # unweighted reference pixels: a measured placement prices them from their token count+ if ref_resolution is not None and not layer_count:+ guard_reference_pixels = int(+ (1 + len(ref_extra)) * ref_resolution * ref_resolution+ )if layer_count:# layers + 1 extra canvas-sized frames beside the output-sized one.guard_condition_pixels += (layer_count + 1) * guard_width * guard_height@@ -15000,6 +15006,9 @@ class DiffusionBackend:and not is_flux3_family(fam))_follow_live_graph_mode(state)+ guard_quadratic = _quadratic_attention(+ guard_target, getattr(state, "attention_backend", None), pipe+ )guard_cfg, extra_mib = split_aware_request_extra(state.pipe,guard_cfg,@@ -15013,6 +15022,8 @@ class DiffusionBackend:cfg = cfg,unpriced_references = guard_unpriced_refs,workflow = _peak_workflow(fam, workflow),+ reference_pixels = guard_reference_pixels,+ quadratic_attention = guard_quadratic,tier_peak_mib = _tier_request_peak(state,width = guard_width,@@ -15058,9 +15069,7 @@ class DiffusionBackend:condition_pixels = guard_condition_pixels,vae_tile_side = vae_tile_side(getattr(pipe, "vae", None)),vae_sliced = vae_can_slice(getattr(pipe, "vae", None)),- quadratic_attention = _quadratic_attention(- guard_target, getattr(state, "attention_backend", None), pipe- ),+ quadratic_attention = guard_quadratic,allow_oversized = allow_oversized,calibrated_placement = bool(getattr(state, "calibrated_placement", False)),controlnet = workflow == "controlnet" and control_pil is not None,diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 1971cc11359..1b4cd2c7b72 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -6069,6 +6069,89 @@ def install_encode_release(MEASURED_RESERVE_DENOISE_PHASE_INDEX = 13+DERIVED_REFERENCE_PRICE_ENV = "UNSLOTH_DIFFUSION_DERIVED_REFERENCE_PRICE"+# References priced by token count, not the flat 8192 MiB/MP (Qwen-Image-2.1 edit 1024^2: 3271-3447 MiB, 2061 K/V cache).+_DERIVED_REFERENCE_FAMILIES = frozenset({"qwen-image-2.1"})+++def derived_reference_pricing_enabled() -> bool:+ return not _env_off(DERIVED_REFERENCE_PRICE_ENV)+++def _reference_pixels_per_token(pipe: Any) -> Optional[int]:+ try:+ scale = int(getattr(pipe, "vae_scale_factor", 0) or 0)+ config = getattr(getattr(pipe, "transformer", None), "config", None)+ patch = int(getattr(config, "patch_size", 1) or 1) if config is not None else 0+ except Exception: # noqa: BLE001+ return None+ if scale <= 0 or patch <= 0:+ return None+ return (scale * patch) ** 2+++def _prefix_kv_bytes_per_token(pipe: Any, compute_bytes: int) -> Optional[int]:+ """Condition K/V cache bytes per cached token (0: no prefix cache; None: unreadable)."""+ transformer = getattr(pipe, "transformer", None)+ config = getattr(transformer, "config", None)+ if transformer is None or config is None:+ return None+ try:+ from .diffusion_prefix_kv import _takes_prefix_kv++ if not _takes_prefix_kv(transformer) or not getattr(config, "causal_condition", False):+ return 0+ layers = int(getattr(config, "num_layers", 0) or 0)+ heads = int(+ getattr(config, "num_key_value_heads", None)+ or getattr(config, "num_attention_heads", 0)+ or 0+ )+ head_dim = int(getattr(config, "attention_head_dim", 0) or 0)+ except Exception: # noqa: BLE001+ return None+ if layers <= 0 or heads <= 0 or head_dim <= 0:+ return None+ return layers * 2 * heads * head_dim * max(1, int(compute_bytes or 2))+++def derived_reference_need_mib(+ pipe: Any,+ *,+ family: Optional[str],+ phase_mib: int,+ width: Optional[int],+ height: Optional[int],+ batch_size: int = 1,+ reference_pixels: int = 0,+ cfg: Optional[bool] = None,+ compute_bytes: int = 2,+ quadratic_attention: bool = False,+) -> Optional[int]:+ """Denoise MiB of a reference request: ``phase_mib`` scaled by joint sequence / 1024^2 plus the condition K/V cache,+ doubled under CFG. None keeps the flat allowance (unmeasured family, kill switch, quadratic attention, unreadable)."""+ if not derived_reference_pricing_enabled() or quadratic_attention:+ return None+ name = (str(family or "").split() or [""])[0].lower()+ if name not in _DERIVED_REFERENCE_FAMILIES or not phase_mib or int(reference_pixels or 0) <= 0:+ return None+ per_token = _reference_pixels_per_token(pipe)+ kv_bytes = _prefix_kv_bytes_per_token(pipe, compute_bytes)+ if per_token is None or kv_bytes is None:+ return None+ batch = max(1, int(batch_size or 1))+ target = max(64, int(width or DEFAULT_IMAGE_WIDTH)) * max(+ 64, int(height or DEFAULT_IMAGE_HEIGHT)+ )+ reference = int(reference_pixels)+ bucket = float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT)+ branches = 1 if cfg is False else 2+ activation = float(phase_mib) * max(1.0, (target + reference) * batch / bucket) * branches+ cache = (reference // per_token) * batch * branches * kv_bytes / float(1 << 20)+ step = _MEASURED_PEAK_ROUND_MIB+ total = activation + cache * _MEASURED_PEAK_MARGIN+ return int(-(-total // step) * step)+def measured_request_extra_mib(pipe: Any,@@ -6081,6 +6164,8 @@ def measured_request_extra_mib(unpriced_references: bool = False,workflow: Optional[str] = None,tier_peak_mib: Optional[int] = None,+ reference_pixels: int = 0,+ quadratic_attention: bool = False,) -> int:"""MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made). ``tier_peak_mib``:a deferred profile's tier peak (``TIER_MOVED_ATTR``), raising eager pricing; None keeps the flat estimate. ``workflow``:@@ -6094,7 +6179,7 @@ def measured_request_extra_mib(only that phase's reserve while the steps run: a request it was sized for (text-to-image, at most the 1024x1024bucket, no condition images or tier peak, within the whole-render reserve) releases nothing, its encode and decodepriced by their own hooks; any other request is priced against the denoise reserve, which first streams the blocks- the phase sizing added."""+ the phase sizing added. ``reference_pixels`` (unweighted): priced by ``derived_reference_need_mib`` when set."""reserve = getattr(pipe, "_unsloth_measured_reserve", None)if not reserve:return 0@@ -6188,6 +6273,8 @@ def measured_request_extra_mib(need = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)if tier_peak is not None:need = tier_peak if need is None else max(int(need), tier_peak)+ base_need = need+ own_learned = Nonecond = max(0, int(condition_pixels or 0)) * max(1, int(batch_size or 1))allowance = int(8192 * cond / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT) * _MEASURED_PEAK_MARGIN@@ -6210,6 +6297,7 @@ def measured_request_extra_mib(own = measured_image_runtime_mib(family, speed_mode, quant = off, **sized)if own is None:own = measured_image_runtime_mib(family, speed_mode, quant = keyed, **sized)+ own_learned = ownif own is not None:# the learned workflow peak is the other complete estimate: it was measured with the condition images# resident, so the allowance is not charged on top of it@@ -6222,14 +6310,33 @@ def measured_request_extra_mib(* max(64, int(height or DEFAULT_IMAGE_HEIGHT))* max(1, int(batch_size or 1)))- sized_for = (- sized_for- and allowance == 0- and tier_peak is None+ within = (+ tier_peak is Noneand pixels <= DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT- and int(need) <= int(headroom)+ and base_need is not None+ and int(base_need) <= int(headroom))- return 0 if sized_for else max(0, int(need) - int(phase))+ if sized_for and allowance == 0 and within and int(need) <= int(headroom):+ return 0+ if within and allowance > 0 and int(reference_pixels or 0) > 0:+ # encode and decode run under their own hooks; only the steps grow with the reference+ derived = derived_reference_need_mib(+ pipe,+ family = family,+ phase_mib = int(phase),+ width = width,+ height = height,+ batch_size = batch_size,+ reference_pixels = int(reference_pixels),+ cfg = cfg,+ compute_bytes = compute_bytes,+ quadratic_attention = quadratic_attention,+ )+ if derived is not None:+ if own_learned is not None:+ derived = max(int(derived), int(own_learned))+ return max(0, int(derived) - int(phase))+ return max(0, int(need) - int(phase))return max(0, int(need) - int(headroom))diff --git a/studio/backend/tests/test_diffusion_reference_pricing.py b/studio/backend/tests/test_diffusion_reference_pricing.pynew file mode 100644index 00000000000..9eafce59002--- /dev/null+++ b/studio/backend/tests/test_diffusion_reference_pricing.py@@ -0,0 +1,227 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A denoise-phase placement prices a reference image from its token count, not the flat 8192 MiB/MP allowance; CPU only.++Numbers from Qwen-Image-2.1 int8 edits on an A100 under an 8 GiB budget: 768 MiB denoise reserve, 2304 MiB whole-render+reserve, one 1024 reference (4096 tokens), condition K/V cache 2061 MiB, learned edit peak 3271-3447 MiB."""++from __future__ import annotations++import types++import pytest++import core.inference.diffusion_measured_peaks as mp+import core.inference.diffusion_memory as dm++QUANT = "torchao:Int8Tensor"+PHASE = 768+REF = 1024 * 1024+# condition_pixels the guard passes for that reference (Qwen-Image-2.1's condition_pixel_weight)+COND = int(REF * 0.32)+++def store(tmp_path, monkeypatch):+ path = tmp_path / "peaks.json"+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(path))+ for env in (+ mp.LEARNED_PEAKS_ENV,+ mp.PHASE_PEAKS_ENV,+ dm.MEASURED_ACTIVATION_ENV,+ dm.DENOISE_PHASE_RESIDENCY_ENV,+ dm.DERIVED_REFERENCE_PRICE_ENV,+ ):+ monkeypatch.delenv(env, raising = False)+ mp._cache.update(path = None, mtime = None, data = None)+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA A100-SXM4-40GB")+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA A100-SXM4-40GB")+ return path+++class _Qwen21:+ def __init__(self, causal = True):+ self.config = types.SimpleNamespace(+ num_layers = 32,+ num_attention_heads = 32,+ attention_head_dim = 128,+ patch_size = 1,+ causal_condition = causal,+ )++ def forward(+ self,+ hidden_states,+ kv_cache = None,+ kv_cache_mode = None,+ ):+ return hidden_states+++class _NoCache(_Qwen21):+ def forward(self, hidden_states):+ return hidden_states+++def _pipe(+ phase = PHASE,+ family = "qwen-image-2.1",+ transformer = None,+ identity = None,+):+ reserve = (+ 2304,+ family,+ "default",+ None,+ None,+ None,+ None,+ None,+ True,+ False,+ False,+ False,+ False,+ phase,+ )+ return types.SimpleNamespace(+ _unsloth_measured_reserve = reserve,+ _unsloth_phase_seeds = True,+ _unsloth_peak_identity = identity,+ transformer = transformer if transformer is not None else _Qwen21(),+ vae_scale_factor = 16,+ )+++def _edit(pipe, **kw):+ args = dict(+ width = 1024,+ height = 1024,+ condition_pixels = COND,+ reference_pixels = REF,+ cfg = False,+ workflow = "edit",+ )+ args.update(kw)+ return dm.measured_request_extra_mib(pipe, **args)+++def _flat_extra(phase = PHASE) -> int:+ allowance = int(8192 * COND / float(1024 * 1024) * dm._MEASURED_PEAK_MARGIN)+ return 2304 + allowance - phase+++def test_kv_cache_size_matches_the_logged_release():+ """32 blocks x K and V x 4096 dims x bf16 per token; 4096 reference tokens = 2048 MiB (logged: 2061 with the prompt)."""+ per_token = dm._prefix_kv_bytes_per_token(_pipe(), 2)+ assert per_token == 32 * 2 * 32 * 128 * 2+ assert dm._reference_pixels_per_token(_pipe()) == 256+ assert (REF // 256) * per_token >> 20 == 2048+++def test_one_1024_reference_is_priced_from_its_tokens(store):+ # (768 x 2 joint sequence + 2048 MiB cache x 1.15), rounded up to 256: 4096, against the 768 denoise reserve+ assert _edit(_pipe()) == 4096 - PHASE+ assert _flat_extra() == 4550+ assert _edit(_pipe()) < _flat_extra()+++def test_the_derived_need_covers_the_measured_edit_peak():+ need = dm.derived_reference_need_mib(+ _pipe(),+ family = "qwen-image-2.1",+ phase_mib = PHASE,+ width = 1024,+ height = 1024,+ reference_pixels = REF,+ cfg = False,+ )+ # learned edit peak (window included) and the condition cache with the margin+ assert need >= 3447 and need >= int(2061 * dm._MEASURED_PEAK_MARGIN)+++def test_kill_switch_keeps_the_flat_allowance(store, monkeypatch):+ monkeypatch.setenv(dm.DERIVED_REFERENCE_PRICE_ENV, "0")+ assert _edit(_pipe()) == _flat_extra()+++def test_the_price_scales_with_the_reference_tokens(store):+ small = _edit(_pipe(), reference_pixels = 512 * 512, condition_pixels = int(512 * 512 * 0.32))+ two = _edit(_pipe(), reference_pixels = 2 * REF, condition_pixels = 2 * COND)+ one = _edit(_pipe())+ assert small < one < two+ # 512^2: 768 x 1.25 + 512 MiB x 1.15 = 1549 -> 1792+ assert small == 1792 - PHASE+++def test_a_guidance_request_doubles_the_cache_and_the_branches(store):+ on = _edit(_pipe(), cfg = None)+ assert on == 7936 - PHASE # 2 x (1536 + 2355) = 7782 -> 7936+ assert on > _edit(_pipe())+++def test_a_larger_learned_workflow_peak_wins(store):+ pipe = _pipe(identity = ("qwen-image-2.1", QUANT, "default"))+ assert _edit(pipe) == 4096 - PHASE+ mp.record(+ family = "qwen-image-2.1",+ quant = dm.with_workflow(QUANT, "edit"),+ speed_mode = "default",+ width = 1024,+ height = 1024,+ batch_size = 1,+ peak_mib = 4000,+ device_name = "NVIDIA A100-SXM4-40GB",+ )+ mp._cache.update(path = None, mtime = None, data = None)+ learned = dm.measured_image_runtime_mib(+ "qwen-image-2.1",+ "default",+ quant = dm.with_workflow(QUANT, "edit"),+ width = 1024,+ height = 1024,+ phase_seeds = True,+ static_seed = True,+ )+ assert learned is not None and learned > 4096+ assert _edit(pipe) == learned - PHASE+++def test_no_prefix_cache_prices_only_the_joint_sequence(store):+ assert _edit(_pipe(transformer = _NoCache())) == 1536 - PHASE+ assert _edit(_pipe(transformer = _Qwen21(causal = False))) == 1536 - PHASE+++ "case",+ ["quadratic", "family", "no_phase", "no_reference_pixels", "large_canvas", "no_vae_scale"],+)+def test_unpriceable_requests_keep_the_flat_allowance(store, case):+ pipe, kw = _pipe(), {}+ if case == "quadratic":+ kw["quadratic_attention"] = True+ elif case == "family":+ pipe = _pipe(family = "flux.3-klein")+ elif case == "no_phase":+ pipe = _pipe(phase = None)+ elif case == "no_reference_pixels":+ kw["reference_pixels"] = 0+ elif case == "large_canvas":+ kw.update(width = 2048, height = 2048)+ elif case == "no_vae_scale":+ pipe.vae_scale_factor = None+ old = dict(kw, reference_pixels = 0, quadratic_attention = False)+ if case == "quadratic":+ old.pop("quadratic_attention")+ assert _edit(pipe, **kw) == _edit(pipe, **old)+ if case in ("quadratic", "no_reference_pixels", "no_vae_scale"):+ assert _edit(pipe, **kw) == _flat_extra()+++def test_text_to_image_is_unchanged(store):+ assert dm.measured_request_extra_mib(_pipe(), width = 1024, height = 1024, reference_pixels = 0) == 0+ assert (+ dm.measured_request_extra_mib(_pipe(), width = 1024, height = 1024, reference_pixels = REF) == 0+ )