Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop the max speed tier recompiling on every new prompt length (#11731)

Daniel Hanabb11de53c2f
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 8ceb290d2a4..2a945aaf87a 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -5571,10 +5571,12 @@ class DiffusionBackend:
"fullgraph": cache_engaged is None
and not cache_may_toggle
and plan.offload_policy == OFFLOAD_NONE,
- # The value apply_speed_optims resolves (automatic for torchao weights), so a bundle
- # from an explicit-dynamic build is not reused for an automatic-dynamic one.
+ # max compiles DiTs with automatic dynamic (None), as does the default tier for
+ # torchao weights. Key on the value apply_speed_optims resolves, so a bundle from a
+ # static or explicit-dynamic build is not reused for an automatic-dynamic one.
"dynamic": compile_dynamic(
- getattr(pipe, "transformer", None), effective_speed != SPEED_MAX
+ getattr(pipe, "transformer", None),
+ None if effective_speed == SPEED_MAX else True,
),
"mode": "max-autotune-no-cudagraphs"
if effective_speed == SPEED_MAX
@@ -7428,82 +7430,92 @@ class DiffusionBackend:
images: list[Any] = []
per_image_seeds: list[int] = []
chunk_shapes: list[int] = []
- pending = list(chunks)
graphs_before = dynamo_graph_count()
- while pending:
- chunk = pending.pop(0)
- chunk_kwargs = dict(kwargs)
- shared = uniform_prompt(chunk)
- generators = [
- torch.Generator(device = state.device).manual_seed(s) for _, s in chunk
- ]
- if len(jobs) == 1:
- chunk_kwargs["prompt"] = shared
- chunk_kwargs["generator"] = generators[0]
- chunk_kwargs["num_images_per_prompt"] = 1
- elif shared is not None:
- chunk_kwargs["prompt"] = shared
- chunk_kwargs["generator"] = generators
- chunk_kwargs["num_images_per_prompt"] = len(chunk)
- else:
- # Distinct prompts: one image per prompt in a single forward. The negative prompt must be
- # broadcast to match, else the pipeline asserts or fails in the txt/img concat.
- chunk_kwargs["prompt"] = [p for p, _ in chunk]
- chunk_kwargs["generator"] = generators
- chunk_kwargs["num_images_per_prompt"] = 1
- if isinstance(chunk_kwargs.get("negative_prompt"), str):
- chunk_kwargs["negative_prompt"] = [
- chunk_kwargs["negative_prompt"]
- ] * len(chunk)
- # A step cache keys residuals on the cond/uncond context, which a graph key
- # cannot see. Per chunk because an AUTO decision is re-taken per generation.
- if state.cuda_graphs:
- cuda_graph.set_bypass(state.cuda_graphs, bool(state.transformer_cache))
- # Start every forward from a clean step cache: diffusers only resets FBCache after a SUCCESSFUL
- # __call__, so a raised call leaves a residual the next forward trips over.
- if state.transformer_cache:
- self._reset_step_cache(state.pipe)
+ try:
+ pending = list(chunks)
+ while pending:
+ chunk = pending.pop(0)
+ chunk_kwargs = dict(kwargs)
+ shared = uniform_prompt(chunk)
+ generators = [
+ torch.Generator(device = state.device).manual_seed(s) for _, s in chunk
+ ]
+ if len(jobs) == 1:
+ chunk_kwargs["prompt"] = shared
+ chunk_kwargs["generator"] = generators[0]
+ chunk_kwargs["num_images_per_prompt"] = 1
+ elif shared is not None:
+ chunk_kwargs["prompt"] = shared
+ chunk_kwargs["generator"] = generators
+ chunk_kwargs["num_images_per_prompt"] = len(chunk)
+ else:
+ # Distinct prompts: one image per prompt in a single forward. The negative prompt must be
+ # broadcast to match, else the pipeline asserts or fails in the txt/img concat.
+ chunk_kwargs["prompt"] = [p for p, _ in chunk]
+ chunk_kwargs["generator"] = generators
+ chunk_kwargs["num_images_per_prompt"] = 1
+ if isinstance(chunk_kwargs.get("negative_prompt"), str):
+ chunk_kwargs["negative_prompt"] = [
+ chunk_kwargs["negative_prompt"]
+ ] * len(chunk)
+ # A step cache keys residuals on the cond/uncond context, which a graph key
+ # cannot see. Per chunk because an AUTO decision is re-taken per generation.
+ if state.cuda_graphs:
+ cuda_graph.set_bypass(state.cuda_graphs, bool(state.transformer_cache))
+ # Start every forward from a clean step cache: diffusers only resets FBCache after a SUCCESSFUL
+ # __call__, so a raised call leaves a residual the next forward trips over.
+ if state.transformer_cache:
+ self._reset_step_cache(state.pipe)
+ try:
+ # inference_mode is faster than no_grad and numerically identical here.
+ with torch.inference_mode():
+ out = pipe(**chunk_kwargs).images
+ except Exception as exc: # noqa: BLE001 - reraised unless a splittable OOM
+ oom = is_oom_error(exc)
+ if oom:
+ # Drop the graphs on ANY OOM, before the split decision: they are shaped for the failed
+ # attempt and empty_cache() cannot reclaim them (live statics and outputs, and a graph
+ # pool is segregated from the ordinary allocator), so the halved retry below and the
+ # user's next smaller request would both run a step's worth of VRAM short. The shape
+ # that finally renders re-captures on its first step.
+ cuda_graph.reset_all(state.cuda_graphs)
+ if len(chunk) < 2 or not oom:
+ raise
+ # OOM backoff: halve the failed chunk and retry; per-image seeds keep every retry
+ # reproducible.
+ empty_cache = getattr(getattr(torch, "cuda", None), "empty_cache", None)
+ if callable(empty_cache):
+ empty_cache()
+ first_half, second_half = split_chunk(chunk)
+ pending[:0] = [first_half, second_half]
+ gen.total_steps += steps # one extra chunk to run
+ logger.warning(
+ "diffusion.generate: batch of %d hit OOM; retrying as %d + %d",
+ len(chunk),
+ len(first_half),
+ len(second_half),
+ )
+ continue
+ finally:
+ # A guarded block that fell back to eager (compile failed at its first forward) no longer
+ # runs compiled: report it on every exit, cancel and error included, so status, LoRA gating
+ # and the compile-cache shape registry stop treating it as compiled.
+ settle_compile_fallback(state, state.pipe, logger)
+ if cancel.is_set():
+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)
+ images.extend(out)
+ per_image_seeds.extend(s for _, s in chunk)
+ chunk_shapes.append(len(chunk))
+ steps_done[0] += steps
+ except BaseException:
+ # A cancelled or failed render may already have generalised a graph that the next render reuses
+ # without compiling, so the success path below would never see the count grow: dirty it now.
try:
- # inference_mode is faster than no_grad and numerically identical here.
- with torch.inference_mode():
- out = pipe(**chunk_kwargs).images
- except Exception as exc: # noqa: BLE001 - reraised unless a splittable OOM
- oom = is_oom_error(exc)
- if oom:
- # Drop the graphs on ANY OOM, before the split decision: they are shaped for the failed
- # attempt and empty_cache() cannot reclaim them (live statics and outputs, and a graph
- # pool is segregated from the ordinary allocator), so the halved retry below and the
- # user's next smaller request would both run a step's worth of VRAM short. The shape
- # that finally renders re-captures on its first step.
- cuda_graph.reset_all(state.cuda_graphs)
- if len(chunk) < 2 or not oom:
- raise
- # OOM backoff: halve the failed chunk and retry; per-image seeds keep every retry
- # reproducible.
- empty_cache = getattr(getattr(torch, "cuda", None), "empty_cache", None)
- if callable(empty_cache):
- empty_cache()
- first_half, second_half = split_chunk(chunk)
- pending[:0] = [first_half, second_half]
- gen.total_steps += steps # one extra chunk to run
- logger.warning(
- "diffusion.generate: batch of %d hit OOM; retrying as %d + %d",
- len(chunk),
- len(first_half),
- len(second_half),
- )
- continue
- finally:
- # A guarded block that fell back to eager (compile failed at its first forward) no longer runs
- # compiled: report it on every exit, cancel and error included, so status, LoRA gating and the
- # compile-cache shape registry stop treating it as compiled.
- settle_compile_fallback(state, state.pipe, logger)
- if cancel.is_set():
- raise RuntimeError(DIFFUSION_CANCELLED_MSG)
- images.extend(out)
- per_image_seeds.extend(s for _, s in chunk)
- chunk_shapes.append(len(chunk))
- steps_done[0] += steps
+ if auto_dynamic_active(state.pipe) and dynamo_graph_count() > graphs_before:
+ compile_cache.mark_recompiled(state.compile_cache_ctx)
+ except Exception: # noqa: BLE001 - bookkeeping must not mask the render's own error
+ pass
+ raise
# Keep progress ACTIVE through the post-denoise work: the route persists the image after this returns,
# so a mount probe reading idle would refresh the gallery too early. Persist the warm compile bundle;
# a STATIC compile makes new artifacts per (w,h,batch), so register this shape. The write itself is
diff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.py
index 4e6f6504ce9..e18d269aa63 100644
--- a/studio/backend/core/inference/diffusion_cache.py
+++ b/studio/backend/core/inference/diffusion_cache.py
@@ -179,7 +179,7 @@ def _compile_hooked_block_inners(transformer: Any, logger: Any = None) -> int:
continue # not a plain bound method; arming would miss the block
# fullgraph=False / dynamic=True: a cache is active (its decision graph-breaks) and this matches the
# default tier. Dynamo caches per code object, so re-arming after a toggle is ~free.
- # Automatic dynamic when the speed layer chose it (torchao weights): see _compile_repeated_blocks.
+ # Automatic dynamic when the speed layer chose it (max tier or torchao weights), as the blocks do.
dynamic = None if getattr(transformer, "_unsloth_auto_dynamic", False) else True
compiled = torch.compile(orig, fullgraph = False, dynamic = dynamic)
# Same runtime fallback as the block's own compile: a lowering failure on the first computed step
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index b5c8475e559..9cc666eed98 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -382,12 +382,15 @@ def _denoiser_unet(pipe: Any) -> Any:
def compiled_shapes_are_static(pipe: Any, speed_mode: Optional[str]) -> bool:
"""Whether this load's compiled artifacts are per-(width, height, batch).
- ``max`` compiles regional blocks dynamic=False and U-Net whole-module is always static;
+ ``max`` compiles regional DiT blocks with automatic dynamic (static until a dimension changes, then one
+ generalised graph; tracked by graph count, not shape) and U-Net whole-module is always static;
``default`` DiT compiles dynamic=True (one artifact across shapes). The compile-cache layer
keys on this to re-save its bundle when a session hits an uncovered shape."""
mode = normalize_speed_mode(speed_mode)
if mode == SPEED_MAX:
- return True
+ # An auto-dynamic DiT generalises a dimension once and then reuses that graph for unseen values, so a new
+ # (width, height, batch) is not a new artifact; the Dynamo graph-count delta marks the renders that compiled.
+ return _denoiser_unet(pipe) is not None or not auto_dynamic_active(pipe)
if mode != SPEED_DEFAULT:
return False
# A torchao-quantised DiT compiles with automatic dynamic: its first shapes get their own artifacts.
@@ -424,12 +427,14 @@ def _compile_repeated_blocks(
if not dits and unet is None:
return False
# default: dynamic=True, fast cold start, no recompile on resolution change. max: max-autotune-no-cudagraphs +
- # dynamic=False, a few % more for a longer compile and a recompile per resolution. Inductor's own cudagraph modes
- # fail on the regional block -- "accessing tensor output of CUDAGraphs that has been overwritten" -- so the tier
- # stays on -no-cudagraphs and the capture is taken one level up, at the denoiser module.
+ # automatic dynamic (None): the first shape compiles static and autotuned, and a dimension that then changes is
+ # generalised once. dynamic=False recompiled on every new prompt length for DiTs whose blocks see the text tokens
+ # (Qwen-Image-2.1: 4-6s on about half of new prompts). Inductor's own cudagraph modes fail on the regional block --
+ # "accessing tensor output of CUDAGraphs that has been overwritten" -- so the tier stays on -no-cudagraphs and the
+ # capture is taken one level up, at the denoiser module.
kwargs: dict[str, Any] = {
"fullgraph": not (cache_active or offload_active),
- "dynamic": not max_autotune,
+ "dynamic": None if max_autotune else True,
}
if max_autotune:
kwargs["mode"] = "max-autotune-no-cudagraphs"
@@ -472,8 +477,8 @@ def _compile_repeated_blocks(
for transformer in dits:
dit_kwargs = dict(kwargs)
dit_kwargs["dynamic"] = compile_dynamic(transformer, kwargs["dynamic"])
- if dit_kwargs["dynamic"] is None:
- transformer._unsloth_auto_dynamic = True
+ # Read by auto_dynamic_active: the generalising recompile on a new text length must reach the bundle.
+ transformer._unsloth_auto_dynamic = dit_kwargs["dynamic"] is None
try:
transformer.compile_repeated_blocks(**dit_kwargs)
engaged = True
@@ -682,7 +687,7 @@ def settle_compile_fallback(
def auto_dynamic_active(pipe: Any) -> bool:
- """Whether any denoiser DiT compiled with automatic dynamic (torchao weights on the default tier)."""
+ """Whether any denoiser DiT compiled with automatic dynamic (the max tier, or torchao weights on the default)."""
return any(getattr(t, "_unsloth_auto_dynamic", False) for t in _guarded_dits(pipe))
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index b338ace4008..b255e997126 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -4465,8 +4465,8 @@ class VideoBackend:
attention_engaged = None
# HunyuanVideo-1.5 only, and once for the whole pipe (the installer fans out over every denoiser DiT itself).
# Before apply_attention_backend below, so the requested kernel pins onto the new processors. Held off on
- # SPEED_OFF (which must stay bit-identical) and on SPEED_MAX (its blocks compile with dynamic=False, and the
- # trimmed text length varies per prompt, so every prompt would be a fresh graph).
+ # SPEED_OFF (which must stay bit-identical) and on SPEED_MAX (its blocks compile static until a dimension
+ # changes, and the trimmed text length varies per prompt, so the first prompts would each recompile).
attention_trim_engaged = (
install_hunyuan_attention_trim(pipe, fam, logger = logger)
if effective_speed not in (SPEED_OFF, SPEED_MAX)
@@ -5095,9 +5095,9 @@ class VideoBackend:
# so "off" has to be known in time to decline it.
effective_speed = resolve_speed_mode(speed_mode, is_gguf = False, dense_default = SPEED_DEFAULT)
if effective_speed == SPEED_MAX:
- # SPEED_MAX compiles with dynamic=False. H3's packed sequence length carries the caption's token rows, so a
- # static graph recompiles per prompt: measured 0.957-1.000 s/step static against 1.000-1.040 dynamic, and
- # two recompiles paid for it.
+ # SPEED_MAX compiles static until a dimension changes. H3's packed sequence length carries the caption's
+ # token rows, so a static graph recompiles on new prompts: measured 0.957-1.000 s/step static against
+ # 1.000-1.040 dynamic, and two recompiles paid for it.
logger.info(
"video.speed_mode: MiniMax-H3 runs the 'default' regional profile under max "
"(a static graph retraces on the caption's contribution to the packed length)"
diff --git a/studio/backend/tests/test_diffusion_compile_cache.py b/studio/backend/tests/test_diffusion_compile_cache.py
index b7abd546942..c74d4d7dfc5 100644
--- a/studio/backend/tests/test_diffusion_compile_cache.py
+++ b/studio/backend/tests/test_diffusion_compile_cache.py
@@ -1155,3 +1155,16 @@ def test_mark_recompiled_dirties_a_context_whose_shape_is_already_registered(tmp
cc.mark_recompiled(ctx)
assert ctx.saved is False and ctx.dirty_seq == seq + 1
cc.mark_recompiled(None) # no context, no error
+
+
+def test_a_failed_or_cancelled_render_still_dirties_the_bundle_after_a_recompile():
+ # The exception path marks the context too: a later render reusing the generalised graph compiles nothing.
+ import inspect
+
+ from core.inference import diffusion
+
+ src = inspect.getsource(diffusion.DiffusionBackend)
+ before = src.index("graphs_before = dynamo_graph_count()")
+ handler = src.index("except BaseException:", before)
+ reraise = src.index("raise\n", handler)
+ assert "compile_cache.mark_recompiled(state.compile_cache_ctx)" in src[handler:reraise]
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 76befa7d08c..e525116857d 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -365,9 +365,33 @@ def test_speed_max_enables_tf32_and_fused_qkv(monkeypatch):
)
assert applied["tf32"] is True and torch.backends.cuda.matmul.allow_tf32 is True
assert applied["fused_qkv"] is True and pipe.fused is True
- # max opts into autotuned kernels (static shapes); CUDA-graph modes are avoided.
+ # max opts into autotuned kernels with automatic dynamic (static until a dimension changes, so a new prompt
+ # length does not recompile every time); CUDA-graph modes are avoided.
assert pipe.compile_kwargs["mode"] == "max-autotune-no-cudagraphs"
- assert pipe.compile_kwargs["dynamic"] is False
+ assert pipe.compile_kwargs["dynamic"] is None
+ assert ds_mod.auto_dynamic_active(pipe) is True
+
+
+def test_default_tier_does_not_mark_automatic_dynamic(monkeypatch):
+ _stub_torch(monkeypatch)
+ pipe = _Pipe(with_compile = True)
+ apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_DEFAULT)
+ assert pipe.compile_kwargs["dynamic"] is True
+ assert ds_mod.auto_dynamic_active(pipe) is False
+ assert isinstance(ds_mod.dynamo_graph_count(), int)
+
+
+def test_max_tier_auto_dynamic_dit_shapes_are_not_tracked_as_static(monkeypatch):
+ # A generalised DiT reuses one graph for unseen shapes; only the graph-count delta dirties its bundle.
+ _stub_torch(monkeypatch)
+ pipe = _Pipe(with_compile = True)
+ apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_MAX)
+ assert ds_mod.auto_dynamic_active(pipe) is True
+ assert ds_mod.compiled_shapes_are_static(pipe, SPEED_MAX) is False
+ # A max-tier pipe whose DiT did not compile auto-dynamic keeps per-shape tracking.
+ plain = _Pipe(with_compile = True)
+ assert ds_mod.compiled_shapes_are_static(plain, SPEED_MAX) is True
+ assert ds_mod.compiled_shapes_are_static(plain, SPEED_DEFAULT) is False
# ── U-Net whole-module compile fallback (SDXL) ─────────────────────────────────
@@ -1362,13 +1386,16 @@ def test_compile_dynamic_is_the_value_the_cache_fingerprint_keys_on():
assert ds_mod.compile_dynamic(None, True) is True
-def test_max_tier_keeps_static_compile_for_torchao_dit(monkeypatch):
+def test_max_tier_compiles_torchao_dit_with_automatic_dynamic(monkeypatch):
+ # max compiles every DiT with automatic dynamic, so a torchao DiT never gets dynamic=True (CantSplit) there either.
_stub_torch(monkeypatch)
_stub_gguf_accel(monkeypatch)
pipe = _Pipe(with_compile = True)
pipe.transformer.parameters = lambda: iter([_TorchaoWeight()])
apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_MAX)
- assert pipe.compile_kwargs["dynamic"] is False
+ assert pipe.compile_kwargs["dynamic"] is None
+ assert ds_mod.auto_dynamic_active(pipe) is True
+ assert ds_mod.compiled_shapes_are_static(pipe, SPEED_MAX) is False
def test_auto_dynamic_active_follows_the_torchao_marker():