Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: pre-seed the float specialisations of an image load's first compiled render (#16917)

Daniel Han06d5015f28c0
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 24fc5f52abf..9432744fd69 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -1685,6 +1685,8 @@ class _LoadState:
# Pre-warmed torch.compile cache context when a compiled tier ran, else None.
compile_cache_ctx: Any = None
bg_compile: Any = None
+ # The first inline compile's float seeds (diffusion_bg_compile.InlineFloatSeeds); holds the DiTs until it kept.
+ inline_float_seeds: Any = None
# GraphedForward handles installed on the denoiser modules.
cuda_graphs: tuple = ()
# Token kept so LoRA adapters selected at generate time can be fetched.
@@ -14085,6 +14087,29 @@ class DiffusionBackend:
else "",
)
+ def _inline_float_seeds(self, state: Any) -> Any:
+ """This load's seeder for a compile inside a render until it kept its names, else None. A load with a
+ background compile seeds there; a load not compiled yet (deferred tier) is asked again on the next render."""
+ held = getattr(state, "inline_float_seeds", None)
+ if held is not None:
+ return None if held.done() else held
+ try:
+ if getattr(state, "bg_compile", None) is not None:
+ return None
+ if "compiled" not in tuple(getattr(state, "speed_optims", None) or ()):
+ return None
+ seed_dir = getattr(getattr(state, "compile_cache_ctx", None), "dir", None)
+ if seed_dir is None or not bg_compile.float_seeds_enabled():
+ return None
+ dits = bg_compile.float_seed_dits(state.pipe)
+ if not dits:
+ return None
+ seeds = bg_compile.InlineFloatSeeds(dits, seed_dir, logger = logger)
+ object.__setattr__(state, "inline_float_seeds", seeds)
+ return seeds
+ except Exception: # noqa: BLE001 - no seeds, main's compile
+ return None
+
@_release_render_on_unload
@holds_inference
@with_hires
@@ -14185,6 +14210,7 @@ class DiffusionBackend:
kick_bg = False
bg_eager = False
bg_token = None
+ float_seeds = None
# A Detailer crop inpaint (diffusion_bg_compile.auxiliary_pass): eager, uncounted, nothing recorded.
aux_pass = bg_compile.auxiliary()
restore_eager_gate: Optional[Callable[[], None]] = None
@@ -14244,6 +14270,8 @@ class DiffusionBackend:
bg_eager = bg is not None and bg.pending()
if bg_eager:
bg_token = bg_compile._FORCE_EAGER.set(True)
+ # An inline compile (no background compile) skips the float restart a returning load already learned.
+ float_seeds = None if bg_eager or aux_pass else self._inline_float_seeds(state)
if aux_pass:
# The force-eager signal never reaches a whole-module compiled U-Net on its own.
restore_eager_gate = bg_compile.gate_compiled_eager(
@@ -15117,6 +15145,9 @@ class DiffusionBackend:
chunk_placement: list = []
def _enter_denoise_phase(gen = gen) -> None:
+ if float_seeds is not None:
+ # Before the first (compiling) denoiser forward.
+ float_seeds.seed()
if gen.phase == "encode":
gen.phase = "denoise"
# The prompt is encoded: the encoders are idle until the next prompt, so free them.
@@ -15146,6 +15177,9 @@ class DiffusionBackend:
"""The decoder was entered: a HOST position, possibly far ahead of the GPU. Mark the
boundary in the stream and let the poller flip once the GPU reaches it; with no event
to wait on (CPU) there is no queue to be ahead of, so flip now."""
+ if float_seeds is not None:
+ # Before the VAE decode's compile adds its own names.
+ float_seeds.keep()
# every step ran (deferred returns settled), the pin still holds, no VAE OOM unpin yet
chunk_placement.append(
denoiser_offload(state.pipe, state.offload_policy)
@@ -15187,6 +15221,11 @@ class DiffusionBackend:
),
protect_ctx,
denoise_phase(pipe, _enter_denoise_phase),
+ (
+ bg_compile.seed_before_first_forward(float_seeds)
+ if float_seeds is not None
+ else nullcontext()
+ ),
zimage_cfg_split_render(pipe, len(chunk)),
decode_phase(pipe, _enter_decode_phase),
_completed_step_poller(_pump),
diff --git a/studio/backend/core/inference/diffusion_bg_compile.py b/studio/backend/core/inference/diffusion_bg_compile.py
index 0e870c769a6..fb79292527b 100644
--- a/studio/backend/core/inference/diffusion_bg_compile.py
+++ b/studio/backend/core/inference/diffusion_bg_compile.py
@@ -775,6 +775,40 @@ class InlineFloatSeeds:
return {"float_seeds": self.float_seeds, "float_seeds_learned": self.float_seeds_learned}
+def float_seed_dits(pipe: Any) -> list:
+ """DiTs compiled with ``dynamic=True`` (SymFloat restarts); static and torchao compiles never restart."""
+ try:
+ from .diffusion_speed import _guarded_dits
+
+ dits = []
+ for dit in _guarded_dits(pipe):
+ kwargs = getattr(dit, "_unsloth_regional_compile_kwargs", None)
+ if isinstance(kwargs, dict) and kwargs.get("dynamic") is True:
+ dits.append(dit)
+ return dits
+ except Exception: # noqa: BLE001 - no seeds, main's compile
+ return []
+
+
+def seed_before_first_forward(float_seeds: Any):
+ """Also seed at the first DiT forward, for a loop that opens no progress bar (H3 modular's is its loop wrapper's)."""
+ handles = []
+
+ def _pre(*_args: Any) -> None:
+ float_seeds.seed()
+
+ try:
+ for module in list(getattr(float_seeds, "modules", None) or ()):
+ with contextlib.suppress(Exception):
+ handles.append(module.register_forward_pre_hook(_pre))
+ yield
+ finally:
+ for handle in handles:
+ with contextlib.suppress(Exception):
+ handle.remove()
+
+
def load_time_enabled() -> bool:
"""Opt-in (``=1``): render 1 eager and render 2 compiled would break "same seed twice repeats"."""
raw = (os.environ.get(_ENV) or "").strip().lower()
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 887ab80d224..7fa2a2c39cf 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -173,6 +173,7 @@ from .diffusion_memory import (
from .diffusion_torchao_patches import install_torchao_int_mm_patch
from .media_decode_phase import decode_phase as _decode_phase
from .media_decode_phase import denoise_phase as _denoise_phase
+from . import diffusion_bg_compile
from . import diffusion_render_thread as render_thread
from .diffusion_fp16_guard import fp16_promotes_to_fp32
from .diffusion_speed import (
@@ -1260,38 +1261,9 @@ def _settle_bg_compile(bg: Any, *, i2v: bool) -> None:
bg.abandon("an image-to-video render records no compile sample")
-def _float_seed_dits(pipe: Any) -> list:
- """DiTs compiled with ``dynamic=True`` (SymFloat restarts); static and torchao compiles never restart."""
- try:
- from .diffusion_speed import _guarded_dits
-
- dits = []
- for dit in _guarded_dits(pipe):
- kwargs = getattr(dit, "_unsloth_regional_compile_kwargs", None)
- if isinstance(kwargs, dict) and kwargs.get("dynamic") is True:
- dits.append(dit)
- return dits
- except Exception: # noqa: BLE001 - no seeds, main's compile
- return []
-
-
-def _seed_before_first_forward(float_seeds: Any):
- """Also seed at the first DiT forward: H3 modular's progress bar is its loop wrapper's, not the pipe's."""
- handles = []
-
- def _pre(*_args: Any) -> None:
- float_seeds.seed()
-
- try:
- for module in list(getattr(float_seeds, "modules", None) or ()):
- with contextlib.suppress(Exception):
- handles.append(module.register_forward_pre_hook(_pre))
- yield
- finally:
- for handle in handles:
- with contextlib.suppress(Exception):
- handle.remove()
+# Shared with the image backend's first compiled render.
+_float_seed_dits = diffusion_bg_compile.float_seed_dits
+_seed_before_first_forward = diffusion_bg_compile.seed_before_first_forward
def _video_float_seed_dir(
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 52936fca9c8..d962beae5fa 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -14668,3 +14668,110 @@ def test_a_pass_that_never_decodes_reads_the_placement_once_it_returns(
monkeypatch.setattr(diffusion, "denoiser_offload", lambda pipe, policy: "resident")
assert backend.generate(prompt = "p", steps = 4)["denoiser_offload"] == "resident"
+
+
+class _SeedsProbe:
+ """Records when generate() seeds and keeps the first inline compile's float specialisations."""
+
+ def __init__(
+ self,
+ events,
+ modules = (),
+ ):
+ self.events = events
+ self.modules = list(modules)
+
+ def seed(self):
+ if "seed" not in self.events:
+ self.events.append("seed")
+
+ def keep(self):
+ self.events.append("keep")
+
+
+def _loop_pipe(
+ pipe,
+ events,
+ dit = None,
+):
+ """The fake pipe as a diffusers loop: a progress bar, the denoiser, then the VAE decode."""
+
+ class _LoopPipe(type(pipe)):
+ def progress_bar(self, *args, **kwargs):
+ return contextlib.nullcontext()
+
+ def __call__(self, **kwargs):
+ if dit is None:
+ with self.progress_bar(total = 2):
+ events.append("denoiser")
+ else:
+ dit()
+ self.vae.decode()
+ return super().__call__(**kwargs)
+
+ pipe.__class__ = _LoopPipe
+ pipe.vae = types.SimpleNamespace(decode = lambda *a, **k: events.append("decode"))
+
+
+def test_first_compiled_render_seeds_floats_before_the_denoise_and_keeps_them_before_the_decode(
+ fake_runtime, tmp_path, monkeypatch
+):
+ """An inline compile (no background compile) pre-seeds the learned float specialisations, as video does."""
+ backend = _loaded_backend(tmp_path)
+ events: list = []
+ asked: list = []
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_inline_float_seeds",
+ lambda self, state: asked.append(state) or _SeedsProbe(events),
+ )
+ _loop_pipe(backend._state.pipe, events)
+ backend.generate(prompt = "a sloth", steps = 2)
+ assert asked == [backend._state]
+ assert events[:2] == ["seed", "denoiser"]
+ assert events.index("keep") < events.index("decode")
+ assert events.count("keep") == 1
+
+
+def test_first_compiled_render_seeds_at_the_first_dit_forward_without_a_progress_bar(
+ fake_runtime, tmp_path, monkeypatch
+):
+ backend = _loaded_backend(tmp_path)
+ events: list = []
+
+ class _Dit:
+ def __init__(self):
+ self.hooks = []
+
+ def register_forward_pre_hook(self, fn):
+ self.hooks.append(fn)
+ return types.SimpleNamespace(remove = lambda: self.hooks.remove(fn))
+
+ def __call__(self):
+ for hook in list(self.hooks):
+ hook(self, ())
+ events.append("denoiser")
+
+ dit = _Dit()
+ monkeypatch.setattr(
+ DiffusionBackend, "_inline_float_seeds", lambda self, state: _SeedsProbe(events, [dit])
+ )
+ _loop_pipe(backend._state.pipe, events, dit = dit)
+ backend.generate(prompt = "a sloth", steps = 2)
+ assert events[:2] == ["seed", "denoiser"]
+ assert dit.hooks == []
+
+
+def test_a_background_compile_render_takes_no_inline_seeds(fake_runtime, tmp_path, monkeypatch):
+ from core.inference import diffusion_bg_compile as bgc
+
+ backend = _loaded_backend(tmp_path)
+ asked: list = []
+ monkeypatch.setattr(
+ DiffusionBackend, "_inline_float_seeds", lambda self, state: asked.append(state) or None
+ )
+ with bgc.auxiliary_pass():
+ backend.generate(prompt = "a crop inpaint", steps = 2)
+ assert asked == []
+ backend.generate(prompt = "a sloth", steps = 2)
+ assert asked == [backend._state]
diff --git a/studio/backend/tests/test_diffusion_inline_float_seeds.py b/studio/backend/tests/test_diffusion_inline_float_seeds.py
new file mode 100644
index 00000000000..0e5ad1debfa
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_inline_float_seeds.py
@@ -0,0 +1,176 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Image first-render compile float seeds (CPU, ``aot_eager``): a load that compiles inside its first render (no
+background compile) pre-seeds the SymFloat specialisations its key learned, so a returning load traces once."""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("torch._dynamo.symbolic_convert")
+
+from core.inference import diffusion as dmod # noqa: E402
+from core.inference import diffusion_bg_compile as bg # noqa: E402
+
+if (
+ bg._tensorify_state() is None
+): # pragma: no cover - a torch without TensorifyState has nothing to seed
+ pytest.skip("this torch has no TensorifyState", allow_module_level = True)
+
+
+class Block(torch.nn.Module):
+ def __init__(self) -> None:
+ super().__init__()
+ self.norm = torch.nn.LayerNorm(16, eps = 1e-6)
+ self.lin = torch.nn.Linear(16, 16)
+
+ def forward(self, x):
+ return x + self.lin(self.norm(x))
+
+
+class Net(torch.nn.Module):
+ _repeated_blocks = ["Block"]
+
+ def __init__(self, dynamic = True) -> None:
+ super().__init__()
+ self.blocks = torch.nn.ModuleList([Block() for _ in range(3)])
+ self._unsloth_regional_compile_kwargs = {"fullgraph": True, "dynamic": dynamic}
+ self._unsloth_compile_guard = types.SimpleNamespace(error = None)
+
+ def forward(self, x):
+ for block in self.blocks:
+ x = block(x)
+ return x
+
+
[email protected](autouse = True)
+def _fresh_dynamo(monkeypatch):
+ monkeypatch.delenv(bg.FLOAT_SEEDS_ENV, raising = False)
+ torch._dynamo.reset()
+ yield
+ torch._dynamo.reset()
+
+
+def restarts(monkeypatch):
+ count = {"n": 0}
+ cls = torch._dynamo.exc.TensorifyScalarRestartAnalysis
+ orig = cls.__init__
+
+ def counting(self, *args, **kwargs):
+ count["n"] += 1
+ orig(self, *args, **kwargs)
+
+ monkeypatch.setattr(cls, "__init__", counting)
+ return count
+
+
+def _state(
+ net,
+ seed_dir,
+ *,
+ speed_optims = ("compiled",),
+ bg_compile = None,
+):
+ return types.SimpleNamespace(
+ pipe = types.SimpleNamespace(transformer = net),
+ speed_optims = speed_optims,
+ compile_cache_ctx = types.SimpleNamespace(dir = seed_dir) if seed_dir is not None else None,
+ bg_compile = bg_compile,
+ inline_float_seeds = None,
+ )
+
+
+def _net(dynamic = True) -> Net:
+ torch.manual_seed(0)
+ net = Net(dynamic = dynamic).eval()
+ for block in net.blocks:
+ block.compile(backend = "aot_eager", dynamic = dynamic)
+ return net
+
+
+def _first_render(seed_dir, x):
+ """One load's first compiled render as generate() runs it: seed at the denoise entry, keep at the decode."""
+ net = _net()
+ state = _state(net, seed_dir)
+ seeds = dmod.DiffusionBackend()._inline_float_seeds(state)
+ assert seeds is not None
+ with bg.seed_before_first_forward(seeds), torch.no_grad():
+ out = net(x)
+ seeds.keep()
+ return seeds, out
+
+
+def test_load_state_carries_the_seeder():
+ assert "inline_float_seeds" in {f.name for f in dataclasses.fields(dmod._LoadState)}
+
+
+def test_returning_load_traces_its_first_render_once_into_the_same_output(tmp_path, restarts):
+ x = torch.randn(2, 5, 16)
+ first, out_first = _first_render(tmp_path, x)
+ assert restarts["n"] >= 1 # a first-ever load still learns through the restart, as on main
+ saved = json.loads((tmp_path / bg.FLOAT_SEEDS_FILE).read_text(encoding = "utf-8"))
+ assert saved["names"] and first.float_seeds == 0
+
+ torch._dynamo.reset()
+ restarts["n"] = 0
+ second, out_second = _first_render(tmp_path, x)
+ assert restarts["n"] == 0
+ assert second.float_seeds == len(saved["names"])
+ assert torch.equal(out_first, out_second)
+
+
+def test_seeds_file_is_the_background_compiles(tmp_path):
+ """Same key dir and file as the deferred tier's background compile, so either path seeds the other."""
+ x = torch.randn(2, 5, 16)
+ _first_render(tmp_path, x)
+ saved = json.loads((tmp_path / bg.FLOAT_SEEDS_FILE).read_text(encoding = "utf-8"))
+ assert saved["signature"] == bg._modules_signature(_net())
+
+
+def test_one_seeder_per_load_until_it_kept(tmp_path):
+ backend = dmod.DiffusionBackend()
+ state = _state(Net(), tmp_path)
+ seeds = backend._inline_float_seeds(state)
+ assert isinstance(seeds, bg.InlineFloatSeeds)
+ assert backend._inline_float_seeds(state) is seeds
+ seeds.seed()
+ seeds.keep()
+ assert backend._inline_float_seeds(state) is None
+ assert seeds.modules == [] # the DiTs are not held past the keep
+
+
+ "case",
+ ["background_compile", "not_compiled", "static", "no_cache_ctx", "kill_switch"],
+)
+def test_no_inline_seeds(tmp_path, monkeypatch, case):
+ backend = dmod.DiffusionBackend()
+ if case == "background_compile":
+ state = _state(Net(), tmp_path, bg_compile = object())
+ elif case == "not_compiled":
+ state = _state(Net(), tmp_path, speed_optims = ("channels_last",))
+ elif case == "static":
+ state = _state(Net(dynamic = False), tmp_path)
+ elif case == "no_cache_ctx":
+ state = _state(Net(), None)
+ else:
+ monkeypatch.setenv(bg.FLOAT_SEEDS_ENV, "0")
+ state = _state(Net(), tmp_path)
+ assert backend._inline_float_seeds(state) is None
+ assert state.inline_float_seeds is None
+
+
+def test_deferred_tier_is_asked_again_once_it_compiles(tmp_path):
+ """Renders 1-2 of the deferred tier run eager; the render that engages an inline compile still seeds."""
+ backend = dmod.DiffusionBackend()
+ state = _state(Net(), tmp_path, speed_optims = ())
+ assert backend._inline_float_seeds(state) is None
+ state.speed_optims = ("compiled",)
+ assert isinstance(backend._inline_float_seeds(state), bg.InlineFloatSeeds)