Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: cross-forward prefetch for video renders (#15108)
Daniel Han2d0fbd6925b5
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.pyindex 8d786c9f258..de27c1b80d1 100644--- a/studio/backend/core/inference/diffusion_offload_prefetch.py+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py@@ -1186,6 +1186,8 @@ def cross_forward_render(pipe: Any, enabled: bool = True) -> Iterator[list]:beside = isinstance(comps, dict) and any("controlnet" in str(name).lower() and value is not None for name, value in comps.items())+ # unreadable second-denoiser routing: the next forward's module is unknown+ beside = beside or _experts(pipe) is _UNKNOWNpfs = _pipe_prefetchers(pipe) if enabled and not beside and cross_forward_enabled() else []if pfs:_LAST_BEGUN.pf = None # interleaving counts within this render only@@ -1202,13 +1204,74 @@ def cross_forward_render(pipe: Any, enabled: bool = True) -> Iterator[list]:pass+_UNKNOWN = object()+++def _experts(pipe: Any) -> Any:+ """``(transformer, transformer_2)`` under diffusers' Wan loop, None for one denoiser, else ``_UNKNOWN``."""+ second = getattr(pipe, "transformer_2", None)+ first = getattr(pipe, "transformer", None)+ if second is None or second is first:+ return None+ if first is None:+ return _UNKNOWN+ # the routing read below is diffusers' Wan loop: an overriding ``__call__`` makes it unknown+ caller = next((c for c in type(pipe).__mro__ if "__call__" in vars(c)), None)+ if caller is None or not str(caller.__module__).startswith("diffusers.pipelines.wan."):+ return _UNKNOWN+ return first, second+++def _expert_router(pipe: Any, step_index: int, timestep: Any, total: int) -> Any:+ """Step -> denoiser by the pipeline's ``t >= boundary`` rule; None for one denoiser, ``_UNKNOWN`` if unreadable."""+ experts = _experts(pipe)+ if experts is None or experts is _UNKNOWN:+ return experts+ first, second = experts+ try:+ ratio = getattr(getattr(pipe, "config", None), "boundary_ratio", None)+ if ratio is None:+ return lambda j: first+ boundary = ratio * pipe.scheduler.config.num_train_timesteps+ timesteps = pipe.scheduler.timesteps+ if timesteps is None or len(timesteps) != total:+ return _UNKNOWN+ # the loop's ``t`` is a view into this schedule (no device read); any other schedule is unreadable+ if timestep is not None and (+ not hasattr(timestep, "data_ptr")+ or timestep.data_ptr() != timesteps[step_index].data_ptr()+ ):+ return _UNKNOWN+ high = _high_noise_steps(timesteps, boundary)+ return lambda j: first if high[j] else second+ except Exception: # noqa: BLE001 - unreadable: carry nothing+ return _UNKNOWN+++# one host copy of the schedule per render, not one device read per step+_ROUTES: dict = {}+++def _high_noise_steps(timesteps: Any, boundary: float) -> list:+ key = (id(timesteps), timesteps.data_ptr(), len(timesteps), float(boundary))+ hit = _ROUTES.get(key)+ if hit is not None and hit[0]() is timesteps:+ return hit[1]+ # the pipeline's own ``t >= boundary_timestep`` comparison+ high = [bool(v) for v in (timesteps.detach().to("cpu") >= boundary)]+ _ROUTES.clear()+ _ROUTES[key] = (weakref.ref(timesteps), high)+ return high++def cross_forward_step(pipe: Any,step_index: int,total: Optional[int] = None,+ timestep: Any = None,) -> None:"""From ``callback_on_step_end``: the final step drops carried copies so nothing is on the card at decode (also- interrupted or unknown-length renders)."""+ interrupted or unknown-length renders). Two experts: carry only into the same expert's next forward."""pfs = [pffor pf in _pipe_prefetchers(pipe)@@ -1222,9 +1285,30 @@ def cross_forward_step(total = int(total) if total is not None else Noneexcept (TypeError, ValueError):total = None+ step_index = int(step_index)stop = total is None or bool(getattr(pipe, "_interrupt", False))+ router = (+ None+ if stop or step_index + 1 >= total+ else _expert_router(pipe, step_index, timestep, total)+ )+ if router is _UNKNOWN:+ stop = True+ nxt = nxt2 = None+ if router is not None and not stop:+ nxt = module_prefetcher(router(step_index + 1))+ if step_index + 2 < total:+ nxt2 = module_prefetcher(router(step_index + 2))for pf in pfs:- if stop or int(step_index) + 1 >= total:+ if stop or step_index + 1 >= total:_disarm(pf)- elif int(step_index) + 2 >= total:+ elif step_index + 2 >= total:pf.armed = False # the last step reads what this one carried, and carries nothing on+ if router is not None and pf is not nxt and not getattr(pf, "active", False):+ pf.drop_carried() # the last step runs another expert+ elif router is not None:+ if pf is not nxt:+ if not getattr(pf, "active", False):+ pf.drop_carried() # the next step runs another expert: these copies would not be read+ elif nxt2 is not pf:+ pf.armed = False # the expert switches after the next step: carry nothing past itdiff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex e7a59075bd7..f76cc5a1365 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -228,6 +228,7 @@ from .diffusion import (from .diffusion_native_quant import native_quant_reasonfrom .diffusion_batched import is_oom_errorfrom .diffusion_block_graph import compile_pipe_below_offload_hooks+from .diffusion_offload_prefetch import cross_forward_render, cross_forward_stepfrom .diffusion_precision import (effective_te_quant,normalize_te_quant,@@ -3312,6 +3313,12 @@ def _video_seed_stays_resident(return plan_keeps_transformer_resident(planned)+def _carried_call(pipe: Any, kwargs: dict, has_step_callback: bool) -> Any:+ """Armed on the render thread (the interleave check is per thread); needs the step callback to stop before decode."""+ with cross_forward_render(pipe, enabled = has_step_callback):+ return pipe(**kwargs)++def _render_under_no_grad(state: Any) -> bool:"""Render under no_grad: torchao weight subclasses fail an offload hook's in-place swap under inference_mode.Stored fp8 codes too: moved under inference_mode they become unversionable, re-copied to host every offload."""@@ -12538,12 +12545,14 @@ class VideoBackend:# the step's latent update is already submitted when the marker goes down.if cancel.is_set():p._interrupt = True+ cross_forward_step(p, step_index, timestep = timestep)return callback_kwargs_tick(step_index + 1)if previewer is not None:previewer.on_step(callback_kwargs.get("latents"), getattr(p, "scheduler", None))+ cross_forward_step(p, step_index, timestep = timestep)return callback_kwargsdef _on_scheduler_step_cancel(done: int) -> None:@@ -12676,7 +12685,7 @@ class VideoBackend:"video",ltx25_render_callif ltx25_render_call is not None- else (lambda: pipe(**kwargs)),+ else (lambda: _carried_call(pipe, kwargs, has_step_callback)),)except _VideoGenerationCancelled:# Unwinding by exception skips maybe_free_model_hooks(); under offload the onloaded modules woulddiff --git a/studio/backend/tests/test_video_offload_cross_forward.py b/studio/backend/tests/test_video_offload_cross_forward.pynew file mode 100644index 00000000000..37932bf5ad1--- /dev/null+++ b/studio/backend/tests/test_video_offload_cross_forward.py@@ -0,0 +1,251 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Cross-forward carry on the video path: two-expert (Wan2.2-A14B) routing and the render wiring."""++from __future__ import annotations++import copy+import os+import sys+import types++import pytest+import torch++import core.inference.diffusion_memory as dm+import core.inference.diffusion_offload_prefetch as op++_TESTS_DIR = os.path.dirname(os.path.abspath(__file__))+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++from test_diffusion_offload_cross_forward import (+ _Net,+ _clean_env, # noqa: F401 - autouse fixture+ _cpu_prefetcher,+ _cuda,+ _forward,+) # noqa: E402+++def _wan_class(module = "diffusers.pipelines.wan.pipeline_wan"):+ def call(self, **kwargs):+ return None++ return type("WanPipeline", (), {"__call__": call, "__module__": module})+++def _wan_pipe(+ first,+ second,+ timesteps,+ ratio = 0.875,+ cls = None,+):+ pipe = (cls or _wan_class())()+ pipe.transformer = first+ pipe.transformer_2 = second+ pipe.components = {"transformer": first, "transformer_2": second}+ pipe.config = types.SimpleNamespace(boundary_ratio = ratio)+ pipe.scheduler = types.SimpleNamespace(+ timesteps = torch.tensor(timesteps, dtype = torch.float32),+ config = types.SimpleNamespace(num_train_timesteps = 1000),+ )+ pipe._interrupt = False+ pipe._num_timesteps = len(timesteps)+ return pipe+++def _holder(pf):+ return types.SimpleNamespace(**{op.PREFETCHER_ATTR: pf})+++def _two_experts(depth = 2):+ a, ga, la = _cpu_prefetcher("abcdef", depth = depth)+ b, gb, lb = _cpu_prefetcher("uvwxyz", depth = depth)+ _forward(a, ga) # records the orders+ _forward(b, gb)+ return (a, ga, la), (b, gb, lb)+++def _wan_loop(+ pipe,+ experts,+ cfg = True,+ timesteps_seen = None,+):+ """diffusers' WanPipeline denoise loop: the expert picked per step by ``t >= boundary``."""+ boundary = pipe.config.boundary_ratio * 1000 if pipe.config.boundary_ratio is not None else None+ ran = []+ with op.cross_forward_render(pipe):+ seq = (+ torch.tensor(timesteps_seen, dtype = torch.float32)+ if timesteps_seen+ else pipe.scheduler.timesteps+ )+ for i, t in enumerate(seq):+ pf, groups = experts[0] if boundary is None or t >= boundary else experts[1]+ for _ in range(2 if cfg else 1):+ _forward(pf, groups)+ ran.append(pf)+ op.cross_forward_step(pipe, i, timestep = t)+ return ran++[email protected]("cfg", [False, True])[email protected]("switch_at", [1, 2, 4, 5])+def test_two_experts_never_carry_across_the_switch(cfg, switch_at):+ (a, ga, la), (b, gb, lb) = _two_experts()+ steps = 6+ ts = [999.0 - i for i in range(switch_at)] + [500.0 - i for i in range(steps - switch_at)]+ pipe = _wan_pipe(_holder(a), _holder(b), ts)+ la.clear()+ lb.clear()+ begun_b = b.begin++ def begin_b():+ assert (+ not a.ready and not a.carried+ ), "the high-noise expert's copies sat into the low-noise forward"+ begun_b()++ b.begin = begin_b+ ran = _wan_loop(pipe, [(a, ga), (b, gb)], cfg = cfg)+ assert ran == [a] * switch_at + [b] * (steps - switch_at)+ for pf in (a, b):+ # every carried copy was read by the next forward of the same expert, except a switch right after step 0:+ # no callback runs before step 0's carry, so step 0's callback drops it (before the other expert begins)+ wasted = 2 if pf is a and switch_at == 1 else 0+ assert pf.stats.get("carried", 0) == pf.stats.get("carried_hits", 0) + wasted+ assert pf.stats.get("carried_dropped", 0) == wasted+ assert not pf.ready and not pf.carried and pf.inflight_bytes == 0+ assert pf.stats["missed"] == 0+ # the low-noise expert carries within its own run of steps (none when it runs only the last step)+ assert (b.stats.get("carried", 0) > 0) == (steps - switch_at >= 2)+ if switch_at > 1 or cfg:+ assert a.stats.get("carried", 0) > 0+++def test_unreadable_schedule_carries_nothing_more():+ (a, ga, _), (b, gb, _) = _two_experts()+ ts = [999.0, 998.0, 997.0, 500.0]+ pipe = _wan_pipe(_holder(a), _holder(b), ts)+ # the timesteps the loop runs are not the scheduler's: the routing cannot be read back+ _wan_loop(pipe, [(a, ga), (b, gb)], cfg = False, timesteps_seen = [999.0, 600.0, 599.0, 500.0])+ assert a.stats.get("carried_hits", 0) == 0 and not a.armed and not b.armed+ assert not a.ready and not b.ready and a.inflight_bytes == 0 and b.inflight_bytes == 0+++def test_second_denoiser_under_another_loop_is_not_armed():+ (a, _, _), (b, _, _) = _two_experts()+ pipe = _wan_pipe(+ _holder(a), _holder(b), [999.0, 500.0], cls = _wan_class("diffusers.pipelines.lucy.x")+ )+ with op.cross_forward_render(pipe) as pfs:+ assert pfs == [] and not a.armed and not b.armed+++def test_subclass_overriding_the_loop_is_not_armed():+ base = _wan_class()+ sub = type("Sub", (base,), {"__call__": lambda self, **k: None, "__module__": "studio.x"})+ (a, _, _), (b, _, _) = _two_experts()+ pipe = _wan_pipe(_holder(a), _holder(b), [999.0, 500.0], cls = sub)+ with op.cross_forward_render(pipe) as pfs:+ assert pfs == []+ sub2 = type("Sub2", (base,), {"__module__": "studio.x"}) # inherits Wan's loop unchanged+ pipe = _wan_pipe(_holder(a), _holder(b), [999.0, 500.0], cls = sub2)+ with op.cross_forward_render(pipe) as pfs:+ assert len(pfs) == 2+++def test_no_boundary_runs_the_first_expert_throughout():+ (a, ga, _), (b, gb, _) = _two_experts()+ pipe = _wan_pipe(_holder(a), _holder(b), [999.0, 500.0, 100.0, 50.0], ratio = None)+ ran = _wan_loop(pipe, [(a, ga), (b, gb)], cfg = False)+ assert ran == [a] * 4+ assert (+ a.stats["carried"] == a.stats["carried_hits"] > 0 and a.stats.get("carried_dropped", 0) == 0+ )+++def test_video_render_arms_only_with_the_step_callback(monkeypatch):+ import core.inference.video as video++ pf, groups, _ = _cpu_prefetcher("abcdef")+ seen = []++ class _Pipe:+ def __init__(self):+ self.components = {"transformer": _holder(pf)}+ self._interrupt = False++ def __call__(self, **kwargs):+ seen.append(pf.armed)+ return "out"++ assert video._carried_call(_Pipe(), {}, True) == "out"+ assert video._carried_call(_Pipe(), {}, False) == "out"+ monkeypatch.setenv(op.CROSS_FORWARD_ENV, "0")+ assert video._carried_call(_Pipe(), {}, True) == "out"+ assert seen == [True, False, False] and not pf.armed+++def _streamed(net):+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net}, _interrupt = False)+ assert dm._apply_group_offload(pipe, "cuda", None)+ return op.module_prefetcher(net)++[email protected]("cfg", [False, True])+def test_streamed_two_experts_bit_identical(cfg):+ _cuda()+ torch.manual_seed(0)+ nets = [_Net(), _Net()]+ refs = [copy.deepcopy(n).cuda() for n in nets]+ pfs = [_streamed(n) for n in nets]+ pipe = _wan_pipe(nets[0], nets[1], [999.0, 950.0, 900.0, 600.0, 400.0, 200.0])+ pipe.scheduler.timesteps = (+ pipe.scheduler.timesteps.cuda()+ ) # the device schedule diffusers keeps+ ts = pipe.scheduler.timesteps+ xs = [torch.randn(4, 64, device = "cuda") for _ in range(2)]++ def run(models, on_step):+ x = xs[0]+ for i, t in enumerate(ts):+ model = models[0] if t >= 875.0 else models[1] # the Wan loop's own routing+ v = model(x)+ if cfg:+ v = v + 0.5 * model(xs[1] + x)+ x = x - 0.1 * v+ on_step(i, t)+ return x++ with torch.no_grad():+ want = run(refs, lambda i, t: None)+ for n in nets:+ n(xs[0]) # records the order+ with op.cross_forward_render(pipe):+ got = run(nets, lambda i, t: op.cross_forward_step(pipe, i, timestep = t))+ torch.cuda.synchronize()+ for n, pf in zip(nets, pfs):+ assert all(b.weight.device.type == "cpu" for b in n.blocks)+ assert not pf.ready and not pf.carried and pf.inflight_bytes == 0+ assert torch.equal(got, want)+ for pf in pfs:+ assert pf.stats["carried_hits"] == pf.stats["carried"] > 0+ assert pf.stats.get("carried_dropped", 0) == 0 and pf.stats["missed"] == 0+++def test_a_new_schedule_is_routed_afresh():+ (a, ga, _), (b, gb, _) = _two_experts()+ pipe = _wan_pipe(_holder(a), _holder(b), [999.0, 998.0, 997.0, 500.0, 499.0])+ assert _wan_loop(pipe, [(a, ga), (b, gb)], cfg = False) == [a] * 3 + [b] * 2+ pipe.scheduler.timesteps = torch.tensor([999.0, 500.0, 499.0, 498.0, 497.0])+ assert _wan_loop(pipe, [(a, ga), (b, gb)], cfg = False) == [a] + [b] * 4+ for pf in (a, b):+ assert pf.stats.get("carried", 0) == pf.stats.get("carried_hits", 0) + pf.stats.get(+ "carried_dropped", 0+ )+ assert not pf.ready and pf.inflight_bytes == 0