Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop the max speed tier recompiling on every new prompt length (#11731)
Daniel Hanabb11de53c2f
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 8ceb290d2a4..2a945aaf87a 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -5571,10 +5571,12 @@ class DiffusionBackend:"fullgraph": cache_engaged is Noneand not cache_may_toggleand plan.offload_policy == OFFLOAD_NONE,- # The value apply_speed_optims resolves (automatic for torchao weights), so a bundle- # from an explicit-dynamic build is not reused for an automatic-dynamic one.+ # max compiles DiTs with automatic dynamic (None), as does the default tier for+ # torchao weights. Key on the value apply_speed_optims resolves, so a bundle from a+ # static or explicit-dynamic build is not reused for an automatic-dynamic one."dynamic": compile_dynamic(- getattr(pipe, "transformer", None), effective_speed != SPEED_MAX+ getattr(pipe, "transformer", None),+ None if effective_speed == SPEED_MAX else True,),"mode": "max-autotune-no-cudagraphs"if effective_speed == SPEED_MAX@@ -7428,82 +7430,92 @@ class DiffusionBackend:images: list[Any] = []per_image_seeds: list[int] = []chunk_shapes: list[int] = []- pending = list(chunks)graphs_before = dynamo_graph_count()- while pending:- chunk = pending.pop(0)- chunk_kwargs = dict(kwargs)- shared = uniform_prompt(chunk)- generators = [- torch.Generator(device = state.device).manual_seed(s) for _, s in chunk- ]- if len(jobs) == 1:- chunk_kwargs["prompt"] = shared- chunk_kwargs["generator"] = generators[0]- chunk_kwargs["num_images_per_prompt"] = 1- elif shared is not None:- chunk_kwargs["prompt"] = shared- chunk_kwargs["generator"] = generators- chunk_kwargs["num_images_per_prompt"] = len(chunk)- else:- # Distinct prompts: one image per prompt in a single forward. The negative prompt must be- # broadcast to match, else the pipeline asserts or fails in the txt/img concat.- chunk_kwargs["prompt"] = [p for p, _ in chunk]- chunk_kwargs["generator"] = generators- chunk_kwargs["num_images_per_prompt"] = 1- if isinstance(chunk_kwargs.get("negative_prompt"), str):- chunk_kwargs["negative_prompt"] = [- chunk_kwargs["negative_prompt"]- ] * len(chunk)- # A step cache keys residuals on the cond/uncond context, which a graph key- # cannot see. Per chunk because an AUTO decision is re-taken per generation.- if state.cuda_graphs:- cuda_graph.set_bypass(state.cuda_graphs, bool(state.transformer_cache))- # Start every forward from a clean step cache: diffusers only resets FBCache after a SUCCESSFUL- # __call__, so a raised call leaves a residual the next forward trips over.- if state.transformer_cache:- self._reset_step_cache(state.pipe)+ try:+ pending = list(chunks)+ while pending:+ chunk = pending.pop(0)+ chunk_kwargs = dict(kwargs)+ shared = uniform_prompt(chunk)+ generators = [+ torch.Generator(device = state.device).manual_seed(s) for _, s in chunk+ ]+ if len(jobs) == 1:+ chunk_kwargs["prompt"] = shared+ chunk_kwargs["generator"] = generators[0]+ chunk_kwargs["num_images_per_prompt"] = 1+ elif shared is not None:+ chunk_kwargs["prompt"] = shared+ chunk_kwargs["generator"] = generators+ chunk_kwargs["num_images_per_prompt"] = len(chunk)+ else:+ # Distinct prompts: one image per prompt in a single forward. The negative prompt must be+ # broadcast to match, else the pipeline asserts or fails in the txt/img concat.+ chunk_kwargs["prompt"] = [p for p, _ in chunk]+ chunk_kwargs["generator"] = generators+ chunk_kwargs["num_images_per_prompt"] = 1+ if isinstance(chunk_kwargs.get("negative_prompt"), str):+ chunk_kwargs["negative_prompt"] = [+ chunk_kwargs["negative_prompt"]+ ] * len(chunk)+ # A step cache keys residuals on the cond/uncond context, which a graph key+ # cannot see. Per chunk because an AUTO decision is re-taken per generation.+ if state.cuda_graphs:+ cuda_graph.set_bypass(state.cuda_graphs, bool(state.transformer_cache))+ # Start every forward from a clean step cache: diffusers only resets FBCache after a SUCCESSFUL+ # __call__, so a raised call leaves a residual the next forward trips over.+ if state.transformer_cache:+ self._reset_step_cache(state.pipe)+ try:+ # inference_mode is faster than no_grad and numerically identical here.+ with torch.inference_mode():+ out = pipe(**chunk_kwargs).images+ except Exception as exc: # noqa: BLE001 - reraised unless a splittable OOM+ oom = is_oom_error(exc)+ if oom:+ # Drop the graphs on ANY OOM, before the split decision: they are shaped for the failed+ # attempt and empty_cache() cannot reclaim them (live statics and outputs, and a graph+ # pool is segregated from the ordinary allocator), so the halved retry below and the+ # user's next smaller request would both run a step's worth of VRAM short. The shape+ # that finally renders re-captures on its first step.+ cuda_graph.reset_all(state.cuda_graphs)+ if len(chunk) < 2 or not oom:+ raise+ # OOM backoff: halve the failed chunk and retry; per-image seeds keep every retry+ # reproducible.+ empty_cache = getattr(getattr(torch, "cuda", None), "empty_cache", None)+ if callable(empty_cache):+ empty_cache()+ first_half, second_half = split_chunk(chunk)+ pending[:0] = [first_half, second_half]+ gen.total_steps += steps # one extra chunk to run+ logger.warning(+ "diffusion.generate: batch of %d hit OOM; retrying as %d + %d",+ len(chunk),+ len(first_half),+ len(second_half),+ )+ continue+ finally:+ # A guarded block that fell back to eager (compile failed at its first forward) no longer+ # runs compiled: report it on every exit, cancel and error included, so status, LoRA gating+ # and the compile-cache shape registry stop treating it as compiled.+ settle_compile_fallback(state, state.pipe, logger)+ if cancel.is_set():+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)+ images.extend(out)+ per_image_seeds.extend(s for _, s in chunk)+ chunk_shapes.append(len(chunk))+ steps_done[0] += steps+ except BaseException:+ # A cancelled or failed render may already have generalised a graph that the next render reuses+ # without compiling, so the success path below would never see the count grow: dirty it now.try:- # inference_mode is faster than no_grad and numerically identical here.- with torch.inference_mode():- out = pipe(**chunk_kwargs).images- except Exception as exc: # noqa: BLE001 - reraised unless a splittable OOM- oom = is_oom_error(exc)- if oom:- # Drop the graphs on ANY OOM, before the split decision: they are shaped for the failed- # attempt and empty_cache() cannot reclaim them (live statics and outputs, and a graph- # pool is segregated from the ordinary allocator), so the halved retry below and the- # user's next smaller request would both run a step's worth of VRAM short. The shape- # that finally renders re-captures on its first step.- cuda_graph.reset_all(state.cuda_graphs)- if len(chunk) < 2 or not oom:- raise- # OOM backoff: halve the failed chunk and retry; per-image seeds keep every retry- # reproducible.- empty_cache = getattr(getattr(torch, "cuda", None), "empty_cache", None)- if callable(empty_cache):- empty_cache()- first_half, second_half = split_chunk(chunk)- pending[:0] = [first_half, second_half]- gen.total_steps += steps # one extra chunk to run- logger.warning(- "diffusion.generate: batch of %d hit OOM; retrying as %d + %d",- len(chunk),- len(first_half),- len(second_half),- )- continue- finally:- # A guarded block that fell back to eager (compile failed at its first forward) no longer runs- # compiled: report it on every exit, cancel and error included, so status, LoRA gating and the- # compile-cache shape registry stop treating it as compiled.- settle_compile_fallback(state, state.pipe, logger)- if cancel.is_set():- raise RuntimeError(DIFFUSION_CANCELLED_MSG)- images.extend(out)- per_image_seeds.extend(s for _, s in chunk)- chunk_shapes.append(len(chunk))- steps_done[0] += steps+ if auto_dynamic_active(state.pipe) and dynamo_graph_count() > graphs_before:+ compile_cache.mark_recompiled(state.compile_cache_ctx)+ except Exception: # noqa: BLE001 - bookkeeping must not mask the render's own error+ pass+ raise# Keep progress ACTIVE through the post-denoise work: the route persists the image after this returns,# so a mount probe reading idle would refresh the gallery too early. Persist the warm compile bundle;# a STATIC compile makes new artifacts per (w,h,batch), so register this shape. The write itself isdiff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.pyindex 4e6f6504ce9..e18d269aa63 100644--- a/studio/backend/core/inference/diffusion_cache.py+++ b/studio/backend/core/inference/diffusion_cache.py@@ -179,7 +179,7 @@ def _compile_hooked_block_inners(transformer: Any, logger: Any = None) -> int:continue # not a plain bound method; arming would miss the block# fullgraph=False / dynamic=True: a cache is active (its decision graph-breaks) and this matches the# default tier. Dynamo caches per code object, so re-arming after a toggle is ~free.- # Automatic dynamic when the speed layer chose it (torchao weights): see _compile_repeated_blocks.+ # Automatic dynamic when the speed layer chose it (max tier or torchao weights), as the blocks do.dynamic = None if getattr(transformer, "_unsloth_auto_dynamic", False) else Truecompiled = torch.compile(orig, fullgraph = False, dynamic = dynamic)# Same runtime fallback as the block's own compile: a lowering failure on the first computed stepdiff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex b5c8475e559..9cc666eed98 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -382,12 +382,15 @@ def _denoiser_unet(pipe: Any) -> Any:def compiled_shapes_are_static(pipe: Any, speed_mode: Optional[str]) -> bool:"""Whether this load's compiled artifacts are per-(width, height, batch).- ``max`` compiles regional blocks dynamic=False and U-Net whole-module is always static;+ ``max`` compiles regional DiT blocks with automatic dynamic (static until a dimension changes, then one+ generalised graph; tracked by graph count, not shape) and U-Net whole-module is always static;``default`` DiT compiles dynamic=True (one artifact across shapes). The compile-cache layerkeys on this to re-save its bundle when a session hits an uncovered shape."""mode = normalize_speed_mode(speed_mode)if mode == SPEED_MAX:- return True+ # An auto-dynamic DiT generalises a dimension once and then reuses that graph for unseen values, so a new+ # (width, height, batch) is not a new artifact; the Dynamo graph-count delta marks the renders that compiled.+ return _denoiser_unet(pipe) is not None or not auto_dynamic_active(pipe)if mode != SPEED_DEFAULT:return False# A torchao-quantised DiT compiles with automatic dynamic: its first shapes get their own artifacts.@@ -424,12 +427,14 @@ def _compile_repeated_blocks(if not dits and unet is None:return False# default: dynamic=True, fast cold start, no recompile on resolution change. max: max-autotune-no-cudagraphs +- # dynamic=False, a few % more for a longer compile and a recompile per resolution. Inductor's own cudagraph modes- # fail on the regional block -- "accessing tensor output of CUDAGraphs that has been overwritten" -- so the tier- # stays on -no-cudagraphs and the capture is taken one level up, at the denoiser module.+ # automatic dynamic (None): the first shape compiles static and autotuned, and a dimension that then changes is+ # generalised once. dynamic=False recompiled on every new prompt length for DiTs whose blocks see the text tokens+ # (Qwen-Image-2.1: 4-6s on about half of new prompts). Inductor's own cudagraph modes fail on the regional block --+ # "accessing tensor output of CUDAGraphs that has been overwritten" -- so the tier stays on -no-cudagraphs and the+ # capture is taken one level up, at the denoiser module.kwargs: dict[str, Any] = {"fullgraph": not (cache_active or offload_active),- "dynamic": not max_autotune,+ "dynamic": None if max_autotune else True,}if max_autotune:kwargs["mode"] = "max-autotune-no-cudagraphs"@@ -472,8 +477,8 @@ def _compile_repeated_blocks(for transformer in dits:dit_kwargs = dict(kwargs)dit_kwargs["dynamic"] = compile_dynamic(transformer, kwargs["dynamic"])- if dit_kwargs["dynamic"] is None:- transformer._unsloth_auto_dynamic = True+ # Read by auto_dynamic_active: the generalising recompile on a new text length must reach the bundle.+ transformer._unsloth_auto_dynamic = dit_kwargs["dynamic"] is Nonetry:transformer.compile_repeated_blocks(**dit_kwargs)engaged = True@@ -682,7 +687,7 @@ def settle_compile_fallback(def auto_dynamic_active(pipe: Any) -> bool:- """Whether any denoiser DiT compiled with automatic dynamic (torchao weights on the default tier)."""+ """Whether any denoiser DiT compiled with automatic dynamic (the max tier, or torchao weights on the default)."""return any(getattr(t, "_unsloth_auto_dynamic", False) for t in _guarded_dits(pipe))diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex b338ace4008..b255e997126 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -4465,8 +4465,8 @@ class VideoBackend:attention_engaged = None# HunyuanVideo-1.5 only, and once for the whole pipe (the installer fans out over every denoiser DiT itself).# Before apply_attention_backend below, so the requested kernel pins onto the new processors. Held off on- # SPEED_OFF (which must stay bit-identical) and on SPEED_MAX (its blocks compile with dynamic=False, and the- # trimmed text length varies per prompt, so every prompt would be a fresh graph).+ # SPEED_OFF (which must stay bit-identical) and on SPEED_MAX (its blocks compile static until a dimension+ # changes, and the trimmed text length varies per prompt, so the first prompts would each recompile).attention_trim_engaged = (install_hunyuan_attention_trim(pipe, fam, logger = logger)if effective_speed not in (SPEED_OFF, SPEED_MAX)@@ -5095,9 +5095,9 @@ class VideoBackend:# so "off" has to be known in time to decline it.effective_speed = resolve_speed_mode(speed_mode, is_gguf = False, dense_default = SPEED_DEFAULT)if effective_speed == SPEED_MAX:- # SPEED_MAX compiles with dynamic=False. H3's packed sequence length carries the caption's token rows, so a- # static graph recompiles per prompt: measured 0.957-1.000 s/step static against 1.000-1.040 dynamic, and- # two recompiles paid for it.+ # SPEED_MAX compiles static until a dimension changes. H3's packed sequence length carries the caption's+ # token rows, so a static graph recompiles on new prompts: measured 0.957-1.000 s/step static against+ # 1.000-1.040 dynamic, and two recompiles paid for it.logger.info("video.speed_mode: MiniMax-H3 runs the 'default' regional profile under max ""(a static graph retraces on the caption's contribution to the packed length)"diff --git a/studio/backend/tests/test_diffusion_compile_cache.py b/studio/backend/tests/test_diffusion_compile_cache.pyindex b7abd546942..c74d4d7dfc5 100644--- a/studio/backend/tests/test_diffusion_compile_cache.py+++ b/studio/backend/tests/test_diffusion_compile_cache.py@@ -1155,3 +1155,16 @@ def test_mark_recompiled_dirties_a_context_whose_shape_is_already_registered(tmpcc.mark_recompiled(ctx)assert ctx.saved is False and ctx.dirty_seq == seq + 1cc.mark_recompiled(None) # no context, no error+++def test_a_failed_or_cancelled_render_still_dirties_the_bundle_after_a_recompile():+ # The exception path marks the context too: a later render reusing the generalised graph compiles nothing.+ import inspect++ from core.inference import diffusion++ src = inspect.getsource(diffusion.DiffusionBackend)+ before = src.index("graphs_before = dynamo_graph_count()")+ handler = src.index("except BaseException:", before)+ reraise = src.index("raise\n", handler)+ assert "compile_cache.mark_recompiled(state.compile_cache_ctx)" in src[handler:reraise]diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex 76befa7d08c..e525116857d 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -365,9 +365,33 @@ def test_speed_max_enables_tf32_and_fused_qkv(monkeypatch):)assert applied["tf32"] is True and torch.backends.cuda.matmul.allow_tf32 is Trueassert applied["fused_qkv"] is True and pipe.fused is True- # max opts into autotuned kernels (static shapes); CUDA-graph modes are avoided.+ # max opts into autotuned kernels with automatic dynamic (static until a dimension changes, so a new prompt+ # length does not recompile every time); CUDA-graph modes are avoided.assert pipe.compile_kwargs["mode"] == "max-autotune-no-cudagraphs"- assert pipe.compile_kwargs["dynamic"] is False+ assert pipe.compile_kwargs["dynamic"] is None+ assert ds_mod.auto_dynamic_active(pipe) is True+++def test_default_tier_does_not_mark_automatic_dynamic(monkeypatch):+ _stub_torch(monkeypatch)+ pipe = _Pipe(with_compile = True)+ apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_DEFAULT)+ assert pipe.compile_kwargs["dynamic"] is True+ assert ds_mod.auto_dynamic_active(pipe) is False+ assert isinstance(ds_mod.dynamo_graph_count(), int)+++def test_max_tier_auto_dynamic_dit_shapes_are_not_tracked_as_static(monkeypatch):+ # A generalised DiT reuses one graph for unseen shapes; only the graph-count delta dirties its bundle.+ _stub_torch(monkeypatch)+ pipe = _Pipe(with_compile = True)+ apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_MAX)+ assert ds_mod.auto_dynamic_active(pipe) is True+ assert ds_mod.compiled_shapes_are_static(pipe, SPEED_MAX) is False+ # A max-tier pipe whose DiT did not compile auto-dynamic keeps per-shape tracking.+ plain = _Pipe(with_compile = True)+ assert ds_mod.compiled_shapes_are_static(plain, SPEED_MAX) is True+ assert ds_mod.compiled_shapes_are_static(plain, SPEED_DEFAULT) is False# ── U-Net whole-module compile fallback (SDXL) ─────────────────────────────────@@ -1362,13 +1386,16 @@ def test_compile_dynamic_is_the_value_the_cache_fingerprint_keys_on():assert ds_mod.compile_dynamic(None, True) is True-def test_max_tier_keeps_static_compile_for_torchao_dit(monkeypatch):+def test_max_tier_compiles_torchao_dit_with_automatic_dynamic(monkeypatch):+ # max compiles every DiT with automatic dynamic, so a torchao DiT never gets dynamic=True (CantSplit) there either._stub_torch(monkeypatch)_stub_gguf_accel(monkeypatch)pipe = _Pipe(with_compile = True)pipe.transformer.parameters = lambda: iter([_TorchaoWeight()])apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_MAX)- assert pipe.compile_kwargs["dynamic"] is False+ assert pipe.compile_kwargs["dynamic"] is None+ assert ds_mod.auto_dynamic_active(pipe) is True+ assert ds_mod.compiled_shapes_are_static(pipe, SPEED_MAX) is Falsedef test_auto_dynamic_active_follows_the_torchao_marker():