Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: cross-forward prefetch for video renders (#15108)

Daniel Han2d0fbd6925b5
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.py
index 8d786c9f258..de27c1b80d1 100644
--- a/studio/backend/core/inference/diffusion_offload_prefetch.py
+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py
@@ -1186,6 +1186,8 @@ def cross_forward_render(pipe: Any, enabled: bool = True) -> Iterator[list]:
beside = isinstance(comps, dict) and any(
"controlnet" in str(name).lower() and value is not None for name, value in comps.items()
)
+ # unreadable second-denoiser routing: the next forward's module is unknown
+ beside = beside or _experts(pipe) is _UNKNOWN
pfs = _pipe_prefetchers(pipe) if enabled and not beside and cross_forward_enabled() else []
if pfs:
_LAST_BEGUN.pf = None # interleaving counts within this render only
@@ -1202,13 +1204,74 @@ def cross_forward_render(pipe: Any, enabled: bool = True) -> Iterator[list]:
pass
+_UNKNOWN = object()
+
+
+def _experts(pipe: Any) -> Any:
+ """``(transformer, transformer_2)`` under diffusers' Wan loop, None for one denoiser, else ``_UNKNOWN``."""
+ second = getattr(pipe, "transformer_2", None)
+ first = getattr(pipe, "transformer", None)
+ if second is None or second is first:
+ return None
+ if first is None:
+ return _UNKNOWN
+ # the routing read below is diffusers' Wan loop: an overriding ``__call__`` makes it unknown
+ caller = next((c for c in type(pipe).__mro__ if "__call__" in vars(c)), None)
+ if caller is None or not str(caller.__module__).startswith("diffusers.pipelines.wan."):
+ return _UNKNOWN
+ return first, second
+
+
+def _expert_router(pipe: Any, step_index: int, timestep: Any, total: int) -> Any:
+ """Step -> denoiser by the pipeline's ``t >= boundary`` rule; None for one denoiser, ``_UNKNOWN`` if unreadable."""
+ experts = _experts(pipe)
+ if experts is None or experts is _UNKNOWN:
+ return experts
+ first, second = experts
+ try:
+ ratio = getattr(getattr(pipe, "config", None), "boundary_ratio", None)
+ if ratio is None:
+ return lambda j: first
+ boundary = ratio * pipe.scheduler.config.num_train_timesteps
+ timesteps = pipe.scheduler.timesteps
+ if timesteps is None or len(timesteps) != total:
+ return _UNKNOWN
+ # the loop's ``t`` is a view into this schedule (no device read); any other schedule is unreadable
+ if timestep is not None and (
+ not hasattr(timestep, "data_ptr")
+ or timestep.data_ptr() != timesteps[step_index].data_ptr()
+ ):
+ return _UNKNOWN
+ high = _high_noise_steps(timesteps, boundary)
+ return lambda j: first if high[j] else second
+ except Exception: # noqa: BLE001 - unreadable: carry nothing
+ return _UNKNOWN
+
+
+# one host copy of the schedule per render, not one device read per step
+_ROUTES: dict = {}
+
+
+def _high_noise_steps(timesteps: Any, boundary: float) -> list:
+ key = (id(timesteps), timesteps.data_ptr(), len(timesteps), float(boundary))
+ hit = _ROUTES.get(key)
+ if hit is not None and hit[0]() is timesteps:
+ return hit[1]
+ # the pipeline's own ``t >= boundary_timestep`` comparison
+ high = [bool(v) for v in (timesteps.detach().to("cpu") >= boundary)]
+ _ROUTES.clear()
+ _ROUTES[key] = (weakref.ref(timesteps), high)
+ return high
+
+
def cross_forward_step(
pipe: Any,
step_index: int,
total: Optional[int] = None,
+ timestep: Any = None,
) -> None:
"""From ``callback_on_step_end``: the final step drops carried copies so nothing is on the card at decode (also
- interrupted or unknown-length renders)."""
+ interrupted or unknown-length renders). Two experts: carry only into the same expert's next forward."""
pfs = [
pf
for pf in _pipe_prefetchers(pipe)
@@ -1222,9 +1285,30 @@ def cross_forward_step(
total = int(total) if total is not None else None
except (TypeError, ValueError):
total = None
+ step_index = int(step_index)
stop = total is None or bool(getattr(pipe, "_interrupt", False))
+ router = (
+ None
+ if stop or step_index + 1 >= total
+ else _expert_router(pipe, step_index, timestep, total)
+ )
+ if router is _UNKNOWN:
+ stop = True
+ nxt = nxt2 = None
+ if router is not None and not stop:
+ nxt = module_prefetcher(router(step_index + 1))
+ if step_index + 2 < total:
+ nxt2 = module_prefetcher(router(step_index + 2))
for pf in pfs:
- if stop or int(step_index) + 1 >= total:
+ if stop or step_index + 1 >= total:
_disarm(pf)
- elif int(step_index) + 2 >= total:
+ elif step_index + 2 >= total:
pf.armed = False # the last step reads what this one carried, and carries nothing on
+ if router is not None and pf is not nxt and not getattr(pf, "active", False):
+ pf.drop_carried() # the last step runs another expert
+ elif router is not None:
+ if pf is not nxt:
+ if not getattr(pf, "active", False):
+ pf.drop_carried() # the next step runs another expert: these copies would not be read
+ elif nxt2 is not pf:
+ pf.armed = False # the expert switches after the next step: carry nothing past it
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index e7a59075bd7..f76cc5a1365 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -228,6 +228,7 @@ from .diffusion import (
from .diffusion_native_quant import native_quant_reason
from .diffusion_batched import is_oom_error
from .diffusion_block_graph import compile_pipe_below_offload_hooks
+from .diffusion_offload_prefetch import cross_forward_render, cross_forward_step
from .diffusion_precision import (
effective_te_quant,
normalize_te_quant,
@@ -3312,6 +3313,12 @@ def _video_seed_stays_resident(
return plan_keeps_transformer_resident(planned)
+def _carried_call(pipe: Any, kwargs: dict, has_step_callback: bool) -> Any:
+ """Armed on the render thread (the interleave check is per thread); needs the step callback to stop before decode."""
+ with cross_forward_render(pipe, enabled = has_step_callback):
+ return pipe(**kwargs)
+
+
def _render_under_no_grad(state: Any) -> bool:
"""Render under no_grad: torchao weight subclasses fail an offload hook's in-place swap under inference_mode.
Stored fp8 codes too: moved under inference_mode they become unversionable, re-copied to host every offload."""
@@ -12538,12 +12545,14 @@ class VideoBackend:
# the step's latent update is already submitted when the marker goes down.
if cancel.is_set():
p._interrupt = True
+ cross_forward_step(p, step_index, timestep = timestep)
return callback_kwargs
_tick(step_index + 1)
if previewer is not None:
previewer.on_step(
callback_kwargs.get("latents"), getattr(p, "scheduler", None)
)
+ cross_forward_step(p, step_index, timestep = timestep)
return callback_kwargs
def _on_scheduler_step_cancel(done: int) -> None:
@@ -12676,7 +12685,7 @@ class VideoBackend:
"video",
ltx25_render_call
if ltx25_render_call is not None
- else (lambda: pipe(**kwargs)),
+ else (lambda: _carried_call(pipe, kwargs, has_step_callback)),
)
except _VideoGenerationCancelled:
# Unwinding by exception skips maybe_free_model_hooks(); under offload the onloaded modules would
diff --git a/studio/backend/tests/test_video_offload_cross_forward.py b/studio/backend/tests/test_video_offload_cross_forward.py
new file mode 100644
index 00000000000..37932bf5ad1
--- /dev/null
+++ b/studio/backend/tests/test_video_offload_cross_forward.py
@@ -0,0 +1,251 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Cross-forward carry on the video path: two-expert (Wan2.2-A14B) routing and the render wiring."""
+
+from __future__ import annotations
+
+import copy
+import os
+import sys
+import types
+
+import pytest
+import torch
+
+import core.inference.diffusion_memory as dm
+import core.inference.diffusion_offload_prefetch as op
+
+_TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_diffusion_offload_cross_forward import (
+ _Net,
+ _clean_env, # noqa: F401 - autouse fixture
+ _cpu_prefetcher,
+ _cuda,
+ _forward,
+) # noqa: E402
+
+
+def _wan_class(module = "diffusers.pipelines.wan.pipeline_wan"):
+ def call(self, **kwargs):
+ return None
+
+ return type("WanPipeline", (), {"__call__": call, "__module__": module})
+
+
+def _wan_pipe(
+ first,
+ second,
+ timesteps,
+ ratio = 0.875,
+ cls = None,
+):
+ pipe = (cls or _wan_class())()
+ pipe.transformer = first
+ pipe.transformer_2 = second
+ pipe.components = {"transformer": first, "transformer_2": second}
+ pipe.config = types.SimpleNamespace(boundary_ratio = ratio)
+ pipe.scheduler = types.SimpleNamespace(
+ timesteps = torch.tensor(timesteps, dtype = torch.float32),
+ config = types.SimpleNamespace(num_train_timesteps = 1000),
+ )
+ pipe._interrupt = False
+ pipe._num_timesteps = len(timesteps)
+ return pipe
+
+
+def _holder(pf):
+ return types.SimpleNamespace(**{op.PREFETCHER_ATTR: pf})
+
+
+def _two_experts(depth = 2):
+ a, ga, la = _cpu_prefetcher("abcdef", depth = depth)
+ b, gb, lb = _cpu_prefetcher("uvwxyz", depth = depth)
+ _forward(a, ga) # records the orders
+ _forward(b, gb)
+ return (a, ga, la), (b, gb, lb)
+
+
+def _wan_loop(
+ pipe,
+ experts,
+ cfg = True,
+ timesteps_seen = None,
+):
+ """diffusers' WanPipeline denoise loop: the expert picked per step by ``t >= boundary``."""
+ boundary = pipe.config.boundary_ratio * 1000 if pipe.config.boundary_ratio is not None else None
+ ran = []
+ with op.cross_forward_render(pipe):
+ seq = (
+ torch.tensor(timesteps_seen, dtype = torch.float32)
+ if timesteps_seen
+ else pipe.scheduler.timesteps
+ )
+ for i, t in enumerate(seq):
+ pf, groups = experts[0] if boundary is None or t >= boundary else experts[1]
+ for _ in range(2 if cfg else 1):
+ _forward(pf, groups)
+ ran.append(pf)
+ op.cross_forward_step(pipe, i, timestep = t)
+ return ran
+
+
[email protected]("cfg", [False, True])
[email protected]("switch_at", [1, 2, 4, 5])
+def test_two_experts_never_carry_across_the_switch(cfg, switch_at):
+ (a, ga, la), (b, gb, lb) = _two_experts()
+ steps = 6
+ ts = [999.0 - i for i in range(switch_at)] + [500.0 - i for i in range(steps - switch_at)]
+ pipe = _wan_pipe(_holder(a), _holder(b), ts)
+ la.clear()
+ lb.clear()
+ begun_b = b.begin
+
+ def begin_b():
+ assert (
+ not a.ready and not a.carried
+ ), "the high-noise expert's copies sat into the low-noise forward"
+ begun_b()
+
+ b.begin = begin_b
+ ran = _wan_loop(pipe, [(a, ga), (b, gb)], cfg = cfg)
+ assert ran == [a] * switch_at + [b] * (steps - switch_at)
+ for pf in (a, b):
+ # every carried copy was read by the next forward of the same expert, except a switch right after step 0:
+ # no callback runs before step 0's carry, so step 0's callback drops it (before the other expert begins)
+ wasted = 2 if pf is a and switch_at == 1 else 0
+ assert pf.stats.get("carried", 0) == pf.stats.get("carried_hits", 0) + wasted
+ assert pf.stats.get("carried_dropped", 0) == wasted
+ assert not pf.ready and not pf.carried and pf.inflight_bytes == 0
+ assert pf.stats["missed"] == 0
+ # the low-noise expert carries within its own run of steps (none when it runs only the last step)
+ assert (b.stats.get("carried", 0) > 0) == (steps - switch_at >= 2)
+ if switch_at > 1 or cfg:
+ assert a.stats.get("carried", 0) > 0
+
+
+def test_unreadable_schedule_carries_nothing_more():
+ (a, ga, _), (b, gb, _) = _two_experts()
+ ts = [999.0, 998.0, 997.0, 500.0]
+ pipe = _wan_pipe(_holder(a), _holder(b), ts)
+ # the timesteps the loop runs are not the scheduler's: the routing cannot be read back
+ _wan_loop(pipe, [(a, ga), (b, gb)], cfg = False, timesteps_seen = [999.0, 600.0, 599.0, 500.0])
+ assert a.stats.get("carried_hits", 0) == 0 and not a.armed and not b.armed
+ assert not a.ready and not b.ready and a.inflight_bytes == 0 and b.inflight_bytes == 0
+
+
+def test_second_denoiser_under_another_loop_is_not_armed():
+ (a, _, _), (b, _, _) = _two_experts()
+ pipe = _wan_pipe(
+ _holder(a), _holder(b), [999.0, 500.0], cls = _wan_class("diffusers.pipelines.lucy.x")
+ )
+ with op.cross_forward_render(pipe) as pfs:
+ assert pfs == [] and not a.armed and not b.armed
+
+
+def test_subclass_overriding_the_loop_is_not_armed():
+ base = _wan_class()
+ sub = type("Sub", (base,), {"__call__": lambda self, **k: None, "__module__": "studio.x"})
+ (a, _, _), (b, _, _) = _two_experts()
+ pipe = _wan_pipe(_holder(a), _holder(b), [999.0, 500.0], cls = sub)
+ with op.cross_forward_render(pipe) as pfs:
+ assert pfs == []
+ sub2 = type("Sub2", (base,), {"__module__": "studio.x"}) # inherits Wan's loop unchanged
+ pipe = _wan_pipe(_holder(a), _holder(b), [999.0, 500.0], cls = sub2)
+ with op.cross_forward_render(pipe) as pfs:
+ assert len(pfs) == 2
+
+
+def test_no_boundary_runs_the_first_expert_throughout():
+ (a, ga, _), (b, gb, _) = _two_experts()
+ pipe = _wan_pipe(_holder(a), _holder(b), [999.0, 500.0, 100.0, 50.0], ratio = None)
+ ran = _wan_loop(pipe, [(a, ga), (b, gb)], cfg = False)
+ assert ran == [a] * 4
+ assert (
+ a.stats["carried"] == a.stats["carried_hits"] > 0 and a.stats.get("carried_dropped", 0) == 0
+ )
+
+
+def test_video_render_arms_only_with_the_step_callback(monkeypatch):
+ import core.inference.video as video
+
+ pf, groups, _ = _cpu_prefetcher("abcdef")
+ seen = []
+
+ class _Pipe:
+ def __init__(self):
+ self.components = {"transformer": _holder(pf)}
+ self._interrupt = False
+
+ def __call__(self, **kwargs):
+ seen.append(pf.armed)
+ return "out"
+
+ assert video._carried_call(_Pipe(), {}, True) == "out"
+ assert video._carried_call(_Pipe(), {}, False) == "out"
+ monkeypatch.setenv(op.CROSS_FORWARD_ENV, "0")
+ assert video._carried_call(_Pipe(), {}, True) == "out"
+ assert seen == [True, False, False] and not pf.armed
+
+
+def _streamed(net):
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net}, _interrupt = False)
+ assert dm._apply_group_offload(pipe, "cuda", None)
+ return op.module_prefetcher(net)
+
+
[email protected]("cfg", [False, True])
+def test_streamed_two_experts_bit_identical(cfg):
+ _cuda()
+ torch.manual_seed(0)
+ nets = [_Net(), _Net()]
+ refs = [copy.deepcopy(n).cuda() for n in nets]
+ pfs = [_streamed(n) for n in nets]
+ pipe = _wan_pipe(nets[0], nets[1], [999.0, 950.0, 900.0, 600.0, 400.0, 200.0])
+ pipe.scheduler.timesteps = (
+ pipe.scheduler.timesteps.cuda()
+ ) # the device schedule diffusers keeps
+ ts = pipe.scheduler.timesteps
+ xs = [torch.randn(4, 64, device = "cuda") for _ in range(2)]
+
+ def run(models, on_step):
+ x = xs[0]
+ for i, t in enumerate(ts):
+ model = models[0] if t >= 875.0 else models[1] # the Wan loop's own routing
+ v = model(x)
+ if cfg:
+ v = v + 0.5 * model(xs[1] + x)
+ x = x - 0.1 * v
+ on_step(i, t)
+ return x
+
+ with torch.no_grad():
+ want = run(refs, lambda i, t: None)
+ for n in nets:
+ n(xs[0]) # records the order
+ with op.cross_forward_render(pipe):
+ got = run(nets, lambda i, t: op.cross_forward_step(pipe, i, timestep = t))
+ torch.cuda.synchronize()
+ for n, pf in zip(nets, pfs):
+ assert all(b.weight.device.type == "cpu" for b in n.blocks)
+ assert not pf.ready and not pf.carried and pf.inflight_bytes == 0
+ assert torch.equal(got, want)
+ for pf in pfs:
+ assert pf.stats["carried_hits"] == pf.stats["carried"] > 0
+ assert pf.stats.get("carried_dropped", 0) == 0 and pf.stats["missed"] == 0
+
+
+def test_a_new_schedule_is_routed_afresh():
+ (a, ga, _), (b, gb, _) = _two_experts()
+ pipe = _wan_pipe(_holder(a), _holder(b), [999.0, 998.0, 997.0, 500.0, 499.0])
+ assert _wan_loop(pipe, [(a, ga), (b, gb)], cfg = False) == [a] * 3 + [b] * 2
+ pipe.scheduler.timesteps = torch.tensor([999.0, 500.0, 499.0, 498.0, 497.0])
+ assert _wan_loop(pipe, [(a, ga), (b, gb)], cfg = False) == [a] + [b] * 4
+ for pf in (a, b):
+ assert pf.stats.get("carried", 0) == pf.stats.get("carried_hits", 0) + pf.stats.get(
+ "carried_dropped", 0
+ )
+ assert not pf.ready and pf.inflight_bytes == 0