WorldPlay2: Extending Real-Time Interactive World Models in Control and Horizon
Python
# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.
"""WorldPlay2 chunked inference with compressed memory and temporal anchors,
built on Wan2.2 I2V-A14B.
Recipe (per HY WorldPlay reference, adapted to Wan2.2):
* Generate ``total_latent_frames`` HR latent frames in groups of
``chunk_length`` frames (default 4).
* For chunk 0, run a vanilla i2v denoising loop without context.
* For chunks > 0:
1. Streaming-VAE-decode the just-finished HR chunk.
2. Take ``frames[:, ::2]`` (every other frame) and bilinear
downsample to (H/4, W/4) in pixel space.
3. Streaming-VAE-encode the downsampled clip back to LR latents.
4. Accumulate (lr_latents_cache, hr_latents_cache, decoded_rgb_cache).
5. Build ``hr_temporal`` = last 2 hr latents before the generate
window with zero y, build hr_y / lr_y context (4-mask + 16-image
latent, image-only at t=0), then call
``model(forward_mode="compress", ...)`` to
produce ``memory_dict``.
6. Run the chunk's denoising loop with ``memory_dict`` plumbed in.
* After the final chunk, decode the last chunk (continuing the same
streaming cache) and concatenate with the cached RGB.
The pipeline builds the T5 encoder, VAE, and two ``WorldPlay2Model`` experts.
Each expert loads its complete DiT, action encoder, and memory compressor
weights from a separate checkpoint.
Sequence-parallel behavior is owned by ``WorldPlay2Model`` and
``WorldPlay2SelfAttention`` themselves. The pipeline only enables it through
``model.set_sequence_parallel``; no runtime monkey-patching is used.
"""
import gc
import logging
import math
import os
import random
import sys
from contextlib import contextmanager
from enum import Enum
from functools import partial
import torch
import torch.distributed as dist
import torch.nn.functional as F
import torchvision.transforms.functional as TF
from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
from tqdm import tqdm
from .checkpoint import load_expert_checkpoint
from .distributed.fsdp import shard_model
from .distributed.ops import get_world_size
from .models.attention import resolve_attention_backend
from .models.model import WorldPlay2Model
from .models.fp8 import (
convert_model_ffn_to_fp8,
convert_model_qkv_to_fp8,
fp8_model_stats,
)
from .pdd import FixedPDD4Scheduler
from .schedulers import (FlowDPMSolverMultistepScheduler,
FlowUniPCMultistepScheduler, get_sampling_sigmas,
retrieve_timesteps)
from .text.encoder import T5EncoderModel
from .vae.wan21 import Wan2_1_VAE
# ---------------------------------------------------------------------------
# small utilities (mirror the dataset/training conventions verbatim)
# ---------------------------------------------------------------------------
def _build_i2v_mask_4ch(t_lat: int, h_lat: int, w_lat: int,
device, dtype) -> torch.Tensor:
"""Build the 4-channel "1,0,0,..." mask used as the first 4 channels
of ``y``. Mirrors ``bi_camera_wan22_w_mem_dataset._build_i2v_mask``.
"""
m = torch.ones(1, t_lat * 4 - 3, h_lat, w_lat, device=device, dtype=dtype)
m[:, 1:] = 0
m = torch.cat([m[:, 0:1].repeat_interleave(4, dim=1), m[:, 1:]], dim=1)
m = m.view(1, m.shape[1] // 4, 4, h_lat, w_lat).transpose(1, 2)[0]
return m # [4, t_lat, h_lat, w_lat]
def _encode_i2v_clip(vae, img_pixel: torch.Tensor, t_lat: int,
h_pix: int, w_pix: int, device) -> torch.Tensor:
"""VAE-encode the I2V pixel clip ``[image, zeros..., zeros]`` into a
16-channel latent of temporal length ``t_lat``.
Mirrors the upstream :meth:`WanI2V.generate` recipe verbatim::
y = vae.encode([
cat([
F.interpolate(img[None], (h, w), mode='bicubic').transpose(0,1),
zeros(3, F_pix - 1, h, w),
], dim=1)
])[0]
where ``F_pix = t_lat * 4 - 3`` so the temporal-causal VAE produces
exactly ``t_lat`` latent frames. ``img_pixel`` is on ``device``,
shape ``[3, H_in, W_in]``, value range [-1, 1].
"""
F_pix = t_lat * 4 - 3
clip = torch.cat([
torch.nn.functional.interpolate(
img_pixel[None].cpu(), size=(h_pix, w_pix),
mode='bilinear').transpose(0, 1), # [3, 1, h, w]
torch.zeros(3, F_pix - 1, h_pix, w_pix),
], dim=1).to(device) # [3, F_pix, h, w]
return vae.encode([clip])[0] # [16, t_lat, h, w]
def _build_y(vae, img_pixel: torch.Tensor, t_lat: int,
h_pix: int, w_pix: int, device, dtype) -> torch.Tensor:
"""Construct the 20-channel y volume = [4-mask | 16-image-clip-latent]
using the same recipe as upstream :meth:`WanI2V.generate`.
"""
y_img = _encode_i2v_clip(vae, img_pixel, t_lat, h_pix, w_pix, device)
h_lat, w_lat = y_img.shape[2], y_img.shape[3]
msk = _build_i2v_mask_4ch(t_lat, h_lat, w_lat, device, y_img.dtype)
return torch.cat([msk, y_img], dim=0).to(dtype=dtype) # [20, t_lat, h_lat, w_lat]
@contextmanager
def _noop_no_sync():
yield
def _parameter_dtype_summary(module) -> dict[str, int]:
counts: dict[str, int] = {}
for parameter in module.parameters():
name = str(parameter.dtype).replace("torch.", "")
counts[name] = counts.get(name, 0) + parameter.numel()
return counts
def _fsdp_unit_count(module) -> int:
return sum(
1 for child in module.modules() if isinstance(child, FSDP)
)
class InferenceMode(str, Enum):
BI = "bi"
AR = "ar"
PDD = "pdd"
# ---------------------------------------------------------------------------
# main pipeline
# ---------------------------------------------------------------------------
class WorldPlay2Pipeline:
"""WorldPlay2 pipeline for chunked action-conditioned video generation.
Owns: T5 encoder, Wan2.1 VAE, two ``WorldPlay2Model`` experts (low / high
noise) with the action MLP + memory_compress submodule installed.
"""
def __init__(
self,
config,
checkpoint_dir: str,
low_noise_mem_ckpt: str,
high_noise_mem_ckpt: str,
device_id: int = 0,
rank: int = 0,
t5_fsdp: bool = False,
dit_fsdp: bool = False,
use_sp: bool = False,
compile_qk_rope: bool = True,
conditioning_cache: bool = True,
fuse_self_qkv: bool = True,
compile_block_glue: bool = True,
compile_ffn: bool = True,
fp8_ffn: bool = False,
fp8_start_block: int = 4,
fp8_end_block: int = 35,
fp8_qkv: bool = False,
fp8_qkv_start_block: int = 4,
fp8_qkv_end_block: int = 35,
init_on_cpu: bool = True,
inference_mode: InferenceMode | str = InferenceMode.BI,
vae_type: str = "wan2_1",
tae_ckpt: str = None,
attention_backend: str = "flash",
):
"""
Args:
config: model settings from ``configs.I2V_A14B_CONFIG``.
checkpoint_dir: directory holding the base T5 / VAE / DiT
checkpoints (same layout :class:`WanI2V` expects).
low_noise_mem_ckpt: path to a ``.safetensors``/``.pt`` file
holding the LOW-noise expert's trained ``action_in.*``,
``img_action_mlp.*`` and ``memory_compress.*`` weights.
high_noise_mem_ckpt: same, but for the HIGH-noise expert.
The two experts are trained separately (see
``run_bi_wan22_action_mem_stage_three.sh`` with /
without ``--low_noise``), so each carries its own
memory_compress + action weights and at inference we
must run TWO compressor passes per chunk and dispatch
by timestep just like the denoise step does.
device_id, rank, t5_fsdp, dit_fsdp, use_sp, init_on_cpu:
runtime placement options.
attention_backend: visual denoising attention backend, 'flash'
or 'sage'.
"""
assert low_noise_mem_ckpt is not None and high_noise_mem_ckpt is not None, (
"WorldPlay2Pipeline requires BOTH --low_noise_mem_ckpt and "
"--high_noise_mem_ckpt; the FastVideo training pipeline "
"produces a separate checkpoint for each expert (each with "
"its own memory_compress + action weights) and the inference "
"loop dispatches between them by timestep.")
self.inference_mode = InferenceMode(inference_mode)
self.attention_backend = resolve_attention_backend(attention_backend)
logging.info("Visual self-attention backend: %s", self.attention_backend)
self.device = torch.device(f"cuda:{device_id}")
self.config = config
self.rank = rank
self.init_on_cpu = init_on_cpu
self.denoise_forward_mode = (
"bi_denoise"
if self.inference_mode is InferenceMode.BI
else (
"pdd_denoise"
if self.inference_mode is InferenceMode.PDD
else "ar_denoise"
)
)
self.pdd_scheduler = (
FixedPDD4Scheduler()
if self.inference_mode is InferenceMode.PDD
else None
)
self.use_sp = use_sp
self.compile_qk_rope = compile_qk_rope
self.conditioning_cache_enabled = conditioning_cache
self.fuse_self_qkv = fuse_self_qkv
self.compile_block_glue = compile_block_glue
self.compile_ffn = compile_ffn
self.fp8_ffn_enabled = fp8_ffn
self.fp8_start_block = fp8_start_block
self.fp8_end_block = fp8_end_block
self.fp8_qkv_enabled = fp8_qkv
self.fp8_qkv_start_block = fp8_qkv_start_block
self.fp8_qkv_end_block = fp8_qkv_end_block
self.fp8_enabled = fp8_ffn or fp8_qkv
if self.fp8_enabled and dit_fsdp:
raise ValueError(
"DiT FP8 currently requires --dit_fsdp to be disabled")
if fp8_qkv and not fuse_self_qkv:
raise ValueError(
"QKV FP8 requires self-attention QKV fusion")
self.num_train_timesteps = config.num_train_timesteps
self.boundary = config.boundary
self.param_dtype = config.param_dtype
if t5_fsdp or dit_fsdp or use_sp:
self.init_on_cpu = False
shard_fn = partial(shard_model, device_id=device_id)
self.text_encoder = T5EncoderModel(
text_len=config.text_len,
dtype=config.t5_dtype,
device=self.device,
checkpoint_path=os.path.join(checkpoint_dir, config.t5_checkpoint),
tokenizer_path=os.path.join(checkpoint_dir, config.t5_tokenizer),
shard_fn=shard_fn if t5_fsdp else None,
)
self.vae_stride = config.vae_stride
self.patch_size = config.patch_size
# VAE is pluggable: the default Wan2.1 VAE, or the causal TAE
# (lightvae) which exposes the SAME encode/decode/encode_chunk/
# decode_chunk list interface via CausalTAEVAE, so nothing downstream
# changes. Both are temporally causal (T_pix = 4*T_lat - 3).
assert vae_type in ("wan2_1", "causal_tae"), (
f"vae_type must be 'wan2_1' or 'causal_tae', got {vae_type!r}")
if vae_type == "causal_tae":
from .vae.causal_tae import CausalTAEVAE
assert tae_ckpt is not None, (
"vae_type='causal_tae' requires --tae_ckpt "
"(the CausalTAE encoder+decoder weights).")
self.vae = CausalTAEVAE(
vae_pth=tae_ckpt,
z_dim=16,
device=self.device)
else:
self.vae = Wan2_1_VAE(
vae_pth=os.path.join(checkpoint_dir, config.vae_checkpoint),
device=self.device)
# Build the architecture from the base model config only. The supplied
# expert checkpoints contain the complete DiT + action + compressor
# state, so loading base-model parameters through from_pretrained would
# be redundant and would temporarily consume additional host memory.
self.low_noise_model = self._build_expert_from_config(
checkpoint_dir=checkpoint_dir,
config_subfolder=config.low_noise_checkpoint,
expert_ckpt=low_noise_mem_ckpt,
tag="low_noise_model",
dtype=self.param_dtype,
fuse_self_qkv=fuse_self_qkv,
pdd_scheduler=self.pdd_scheduler,
pdd_expert="low",
)
self.high_noise_model = self._build_expert_from_config(
checkpoint_dir=checkpoint_dir,
config_subfolder=config.high_noise_checkpoint,
expert_ckpt=high_noise_mem_ckpt,
tag="high_noise_model",
dtype=self.param_dtype,
fuse_self_qkv=fuse_self_qkv,
pdd_scheduler=self.pdd_scheduler,
pdd_expert="high",
)
# ---- configure (FSDP / SP / dtype / device) ----------------------
self.low_noise_model = self._configure_model(
model=self.low_noise_model,
use_sp=use_sp,
attention_backend=self.attention_backend,
compile_qk_rope=compile_qk_rope,
conditioning_cache=conditioning_cache,
compile_block_glue=compile_block_glue,
compile_ffn=compile_ffn,
fp8_ffn=fp8_ffn,
fp8_start_block=fp8_start_block,
fp8_end_block=fp8_end_block,
fp8_qkv=fp8_qkv,
fp8_qkv_start_block=fp8_qkv_start_block,
fp8_qkv_end_block=fp8_qkv_end_block,
dit_fsdp=dit_fsdp,
shard_fn=shard_fn)
self.high_noise_model = self._configure_model(
model=self.high_noise_model,
use_sp=use_sp,
attention_backend=self.attention_backend,
compile_qk_rope=compile_qk_rope,
conditioning_cache=conditioning_cache,
compile_block_glue=compile_block_glue,
compile_ffn=compile_ffn,
fp8_ffn=fp8_ffn,
fp8_start_block=fp8_start_block,
fp8_end_block=fp8_end_block,
fp8_qkv=fp8_qkv,
fp8_qkv_start_block=fp8_qkv_start_block,
fp8_qkv_end_block=fp8_qkv_end_block,
dit_fsdp=dit_fsdp,
shard_fn=shard_fn)
self.sp_size = get_world_size() if use_sp else 1
self.sample_neg_prompt = config.sample_neg_prompt
self.dtype_report = {
"attention_backend": self.attention_backend,
"configured_dit_compute_dtype": str(
self.param_dtype).replace("torch.", ""),
"t5_fsdp_requested": bool(t5_fsdp),
"dit_fsdp": bool(dit_fsdp),
"t5_fsdp_units": _fsdp_unit_count(self.text_encoder.model),
"low_noise_fsdp_units": _fsdp_unit_count(
self.low_noise_model),
"high_noise_fsdp_units": _fsdp_unit_count(
self.high_noise_model),
"compile_qk_rope": bool(compile_qk_rope),
"conditioning_cache": bool(conditioning_cache),
"self_qkv_fused": bool(fuse_self_qkv),
"compile_block_glue": bool(compile_block_glue),
"compile_ffn": bool(compile_ffn),
"fp8_ffn": bool(fp8_ffn),
"fp8_block_range": (
[fp8_start_block, fp8_end_block] if fp8_ffn else None
),
"fp8_qkv": bool(fp8_qkv),
"fp8_qkv_block_range": (
[fp8_qkv_start_block, fp8_qkv_end_block]
if fp8_qkv else None
),
"low_noise_parameter_storage": _parameter_dtype_summary(
self.low_noise_model),
"high_noise_parameter_storage": _parameter_dtype_summary(
self.high_noise_model),
"t5_configured_dtype": str(
config.t5_dtype).replace("torch.", ""),
"t5_parameter_storage": _parameter_dtype_summary(
self.text_encoder.model),
"vae_configured_dtype": str(
self.vae.dtype).replace("torch.", ""),
"vae_parameter_storage": _parameter_dtype_summary(
self.vae.model),
}
logging.info("Dtype report: %s", self.dtype_report)
# ------------------------------------------------------------------
# ckpt loading + model configuration
# ------------------------------------------------------------------
@staticmethod
def _build_expert_from_config(
checkpoint_dir: str,
config_subfolder: str,
expert_ckpt: str,
tag: str,
dtype: torch.dtype,
fuse_self_qkv: bool,
pdd_scheduler: FixedPDD4Scheduler | None = None,
pdd_expert: str | None = None,
) -> WorldPlay2Model:
"""Initialize architecture from config and strictly load a full expert."""
config_dir = os.path.join(checkpoint_dir, config_subfolder)
logging.info("Creating %s from config: %s", tag, config_dir)
model_config = WorldPlay2Model.load_config(config_dir)
model = WorldPlay2Model.from_config(model_config)
model.add_embedding_action_parameters()
model.add_memory_compress_model()
if pdd_scheduler is not None:
model.head.make_fixed_pdd(num_blocks=2)
# Keep the actual DiT/action/compressor parameter storage in BF16,
# rather than relying on autocast or FSDP's transient compute cast.
model.to(dtype=dtype)
if pdd_scheduler is None:
load_expert_checkpoint(model, expert_ckpt, tag=tag, strict=True)
else:
state = pdd_scheduler.prepare_checkpoint(
expert_ckpt, expert=pdd_expert, dtype=dtype
)
provided = set(state)
required_prefixes = (
"memory_compress.",
"action_in_cont.",
"action_in_disc.",
)
absent = [
prefix for prefix in required_prefixes
if not any(key.startswith(prefix) for key in provided)
]
if absent:
raise ValueError(
f"{tag} PDD checkpoint is missing required parameter "
f"groups: {absent}"
)
model.load_state_dict(state, strict=True)
logging.info(
"%s fixed-PDD checkpoint: loaded=%d expert=%s",
tag,
len(state),
pdd_expert,
)
# Install after strict checkpoint loading (which expects q/k/v keys)
# and before FSDP wrapping (which should only see the fused weights).
if fuse_self_qkv:
model.fuse_self_attention_qkv()
return model
def _configure_model(
self,
model,
use_sp,
attention_backend,
compile_qk_rope,
conditioning_cache,
compile_block_glue,
compile_ffn,
fp8_ffn,
fp8_start_block,
fp8_end_block,
fp8_qkv,
fp8_qkv_start_block,
fp8_qkv_end_block,
dit_fsdp,
shard_fn,
):
"""Configure dtype/device/FSDP and enable the model-owned SP path."""
model.eval().requires_grad_(False)
model.to(dtype=self.param_dtype)
model.set_sequence_parallel(use_sp)
model.set_self_attention_backend(attention_backend)
model.set_compiled_qk_rope(compile_qk_rope)
model.set_conditioning_cache(conditioning_cache)
model.set_compiled_regions(
block_glue=compile_block_glue,
ffn=compile_ffn,
)
if dist.is_initialized():
dist.barrier(device_ids=[self.device.index])
if fp8_ffn or fp8_qkv:
# TE primary FP8 weights must remain resident. This path
# intentionally does not support DiT FSDP or model offload.
model.to(self.device)
if fp8_qkv:
report = convert_model_qkv_to_fp8(
model,
start_block=fp8_qkv_start_block,
end_block=fp8_qkv_end_block,
)
logging.info("Installed self-attention QKV FP8: %s", report)
if fp8_ffn:
report = convert_model_ffn_to_fp8(
model,
start_block=fp8_start_block,
end_block=fp8_end_block,
)
logging.info("Installed FFN FP8: %s", report)
if dit_fsdp:
model = shard_fn(model)
else:
if not self.init_on_cpu:
model.to(self.device)
return model
@staticmethod
def _find_wan_model(model):
for module in model.modules():
if isinstance(module, WorldPlay2Model):
return module
raise RuntimeError("FSDP/model wrapper does not contain WorldPlay2Model")
def clear_conditioning_cache(self) -> None:
self._find_wan_model(
self.low_noise_model).clear_conditioning_cache()
self._find_wan_model(
self.high_noise_model).clear_conditioning_cache()
def fp8_stats(self) -> dict:
return {
"low_noise": fp8_model_stats(
self._find_wan_model(self.low_noise_model)),
"high_noise": fp8_model_stats(
self._find_wan_model(self.high_noise_model)),
}
def _prepare_model_for_timestep(self, t, boundary, offload_model):
"""Mirrors :meth:`WanI2V._prepare_model_for_timestep`."""
if t.item() >= boundary:
required_model_name = 'high_noise_model'
offload_model_name = 'low_noise_model'
else:
required_model_name = 'low_noise_model'
offload_model_name = 'high_noise_model'
if offload_model or self.init_on_cpu:
other = getattr(self, offload_model_name)
active = getattr(self, required_model_name)
if next(other.parameters()).device.type == 'cuda':
# Cached tensors are plain inference state and are not moved by
# Module.to(). Drop them before offloading the expert.
self._find_wan_model(other).clear_conditioning_cache()
other.to('cpu')
if next(active.parameters()).device.type == 'cpu':
active.to(self.device)
return getattr(self, required_model_name)
def _prepare_named_expert(self, expert: str, offload_model: bool):
required = f"{expert}_noise_model"
other = "low_noise_model" if expert == "high" else "high_noise_model"
if offload_model or self.init_on_cpu:
other_model = getattr(self, other)
active_model = getattr(self, required)
if next(other_model.parameters()).device.type == "cuda":
self._find_wan_model(other_model).clear_conditioning_cache()
other_model.to("cpu")
if next(active_model.parameters()).device.type == "cpu":
active_model.to(self.device)
return getattr(self, required)
def _run_pdd_chunk(
self,
*,
latent,
y_chunk,
hr_action_chunk,
context,
memory_dict_high,
memory_dict_low,
kv_high,
kv_low,
chunk_start,
seed_g,
):
for block in self.pdd_scheduler.ordered_blocks:
model = self._prepare_named_expert(
block.expert,
self.offload_model,
)
kv_cache = kv_high if block.expert == "high" else kv_low
memory_dict = (
memory_dict_high
if block.expert == "high"
else memory_dict_low
)
timestep = self.pdd_scheduler.timesteps[block.start:block.start + 1].to(self.device)
args = dict(
context=[context[0]],
seq_len=self.seq_len_chunk,
y=[y_chunk],
hr_action=hr_action_chunk,
forward_mode="pdd_denoise",
pdd_block_index=block.local_index,
kv_cache=kv_cache,
ar_chunk_start=chunk_start,
memory_dict=memory_dict,
)
output = model(
[latent.to(self.device)],
t=timestep,
**args,
)[0]
if not isinstance(output, tuple) or len(output) != 2:
raise RuntimeError(
"fixed PDD model must return "
"(displacement, endpoint_velocity)"
)
displacement, endpoint_velocity = output
sigma_endpoint = self.pdd_scheduler.sigmas[block.end - 1].to(device=latent.device, dtype=torch.float32)
state_endpoint = latent.float() + displacement.float()
x0 = (state_endpoint - sigma_endpoint * endpoint_velocity.float()).to(latent.dtype)
sigma_end = self.pdd_scheduler.sigmas[block.end].to(device=latent.device, dtype=torch.float32)
if block.end == self.pdd_scheduler.num_intervals:
latent = x0
else:
noise = torch.randn(x0.shape, generator=seed_g, device=x0.device, dtype=x0.dtype)
latent = ((1.0 - sigma_end) * x0.float() + sigma_end * noise.float()).to(x0.dtype)
return latent
def _build_chunk_scheduler(
self,
sample_solver: str,
sampling_steps: int,
shift: float,
):
if sample_solver == "unipc":
scheduler = FlowUniPCMultistepScheduler(
num_train_timesteps=self.num_train_timesteps,
shift=1,
use_dynamic_shifting=False,
)
scheduler.set_timesteps(
sampling_steps,
device=self.device,
shift=shift,
)
return scheduler, scheduler.timesteps
if sample_solver == "dpm++":
scheduler = FlowDPMSolverMultistepScheduler(
num_train_timesteps=self.num_train_timesteps,
shift=1,
use_dynamic_shifting=False,
)
sampling_sigmas = get_sampling_sigmas(sampling_steps, shift)
timesteps, _ = retrieve_timesteps(
scheduler,
device=self.device,
sigmas=sampling_sigmas,
)
return scheduler, timesteps
raise NotImplementedError(f"Unsupported solver {sample_solver!r}")
def _predict_chunk_noise(
self,
*,
latent,
timestep_value,
boundary,
context,
context_null,
memory_dict_high,
memory_dict_low,
kv_high,
kv_high_neg,
kv_low,
kv_low_neg,
y_chunk,
hr_action_chunk,
ar_cache,
chunk_start,
):
latent_model_input = [latent.to(self.device)]
timestep = torch.stack([timestep_value]).to(self.device)
model = self._prepare_model_for_timestep(
timestep_value,
boundary,
self.offload_model,
)
if timestep_value.item() >= boundary:
memory_dict = memory_dict_high
kv_cache, kv_cache_neg = kv_high, kv_high_neg
sample_guide_scale = self.guide_scale[1]
use_cfg = self.use_cfg_high
else:
memory_dict = memory_dict_low
kv_cache, kv_cache_neg = kv_low, kv_low_neg
sample_guide_scale = self.guide_scale[0]
use_cfg = self.use_cfg_low
model_args = {
"context": [context[0]],
"seq_len": self.seq_len_chunk,
"y": [y_chunk],
"hr_action": hr_action_chunk,
"forward_mode": self.denoise_forward_mode,
}
if ar_cache:
model_args.update(
kv_cache=kv_cache,
ar_chunk_start=chunk_start,
)
else:
model_args["memory_dict"] = memory_dict
noise_pred_cond = model(
latent_model_input,
t=timestep,
**model_args,
)[0]
if use_cfg:
null_args = {
"context": context_null,
"seq_len": self.seq_len_chunk,
"y": [y_chunk],
"hr_action": hr_action_chunk,
"forward_mode": self.denoise_forward_mode,
}
if ar_cache:
null_args.update(
kv_cache=kv_cache_neg,
ar_chunk_start=chunk_start,
)
else:
null_args["memory_dict"] = memory_dict
noise_pred_uncond = model(
latent_model_input,
t=timestep,
**null_args,
)[0]
if self.offload_model:
torch.cuda.empty_cache()
noise_pred_cond = noise_pred_uncond + sample_guide_scale * (
noise_pred_cond - noise_pred_uncond
)
return noise_pred_cond
def _run_solver_chunk(
self,
*,
latent,
sample_solver,
sampling_steps,
shift,
seed_g,
boundary,
context,
context_null,
memory_dict_high,
memory_dict_low,
kv_high,
kv_high_neg,
kv_low,
kv_low_neg,
y_chunk,
hr_action_chunk,
ar_cache,
chunk_start,
chunk_index,
):
scheduler, timesteps = self._build_chunk_scheduler(
sample_solver,
sampling_steps,
shift,
)
if self.offload_model:
torch.cuda.empty_cache()
for timestep_value in tqdm(timesteps, desc=f"chunk {chunk_index}"):
noise_pred = self._predict_chunk_noise(
latent=latent,
timestep_value=timestep_value,
boundary=boundary,
context=context,
context_null=context_null,
memory_dict_high=memory_dict_high,
memory_dict_low=memory_dict_low,
kv_high=kv_high,
kv_high_neg=kv_high_neg,
kv_low=kv_low,
kv_low_neg=kv_low_neg,
y_chunk=y_chunk,
hr_action_chunk=hr_action_chunk,
ar_cache=ar_cache,
chunk_start=chunk_start,
)
latent = scheduler.step(
noise_pred.unsqueeze(0),
timestep_value,
latent.unsqueeze(0),
return_dict=False,
generator=seed_g,
)[0].squeeze(0)
return latent
def _prefill_kv_cache(
self,
model,
memory_dict,
kv_cache,
kv_cache_neg,
context,
context_null,
use_cfg,
):
"""Prefill conditional cache and the optional CFG cache.
The ctx self-attn K/V pass through cross-attention (text), so the
cond and uncond ctx K/V differ -> two separate caches. When
``memory_dict`` is None (first chunk) prefill is a no-op and the
caches stay empty (denoise then runs pred-only attention).
Returns the (possibly mutated) ``(kv_cache, kv_cache_neg)`` so the
caller rebinds explicitly rather than relying on in-place mutation.
"""
if memory_dict is None:
return kv_cache, kv_cache_neg
kv_cache = model(
forward_mode="ar_prefill",
context=[context[0]],
memory_dict=memory_dict,
kv_cache=kv_cache,
)
if use_cfg:
if context_null is None or kv_cache_neg is None:
raise ValueError("CFG prefill requires null context and cache")
kv_cache_neg = model(
forward_mode="ar_prefill",
context=context_null,
memory_dict=memory_dict,
kv_cache=kv_cache_neg,
)
return kv_cache, kv_cache_neg
# ------------------------------------------------------------------
# streaming VAE helpers
# ------------------------------------------------------------------
def _decode_chunk_and_lr_encode(
self,
hr_latent_chunk: torch.Tensor,
is_first_vae_chunk: bool,
):
"""Decode one HR latent chunk via streaming VAE, downsample
(T::2, HW//4 in pixel space), then re-encode via streaming VAE.
Returns (lr_latent, rgb_frames) where:
lr_latent : [16, T_lr_lat, h//4, w//4]
rgb_frames : [3, T_pix, H_pix, W_pix] in [-1, 1]
"""
rgb = self.vae.decode_chunk([hr_latent_chunk],
is_first_chunk=is_first_vae_chunk)[0]
lr_pix = rgb[:, ::2, :, :] # T::2
h_pix, w_pix = lr_pix.shape[2:]
target_size = (h_pix // 4, w_pix // 4)
lr_pix = lr_pix.permute(1, 0, 2, 3).contiguous() # [T, C, H, W]
lr_pix = F.interpolate(lr_pix, size=target_size, mode='bilinear',
align_corners=False)
lr_pix = lr_pix.permute(1, 0, 2, 3).contiguous() # [C, T, H, W]
lr_lat = self.vae.encode_chunk([lr_pix],
is_first_chunk=is_first_vae_chunk)[0]
return lr_lat, rgb
def _build_lr_image_pixel(self, img_pixel_hr: torch.Tensor,
h_hr_pix: int, w_hr_pix: int) -> torch.Tensor:
"""Bicubic-downsample the HR pixel image to the LR resolution
``(h_hr_pix//4, w_hr_pix//4)`` -- matches the (H//4, W//4) target
used by :meth:`_decode_chunk_and_lr_encode`. Returns ``[3, H_lr, W_lr]``
on ``self.device``.
"""
return F.interpolate(
img_pixel_hr[None].cpu(),
size=(h_hr_pix // 4, w_hr_pix // 4),
mode='bilinear')[0].to(self.device)
def _prepare_chunk_memory(
self,
*,
chunk_index,
chunk_end,
latents,
hr_y_full,
lr_y_full,
hr_action,
lr_action,
context,
context_null,
kv_high,
kv_high_neg,
kv_low,
kv_low_neg,
):
"""Decode a generated chunk and prepare memory for the next chunk."""
chunk_start = chunk_end - self.chunk_length
generated_latents = latents[:, chunk_start:chunk_end].contiguous()
new_lr_latents, decoded_rgb = self._decode_chunk_and_lr_encode(
generated_latents,
is_first_vae_chunk=chunk_index == 0,
)
if self.lr_latents_cache is None:
self.lr_latents_cache = new_lr_latents
else:
self.lr_latents_cache = torch.cat(
[self.lr_latents_cache, new_lr_latents],
dim=1,
)
hr_context_latents = latents[:, :chunk_end].contiguous()
hr_context_y = hr_y_full[:, :chunk_end]
lr_context_y = lr_y_full[:, :chunk_end // 2]
hr_input = torch.cat([hr_context_latents, hr_context_y], dim=0).to(dtype=self.param_dtype).unsqueeze(0)
lr_input = torch.cat(
[self.lr_latents_cache, lr_context_y],
dim=0,
).to(dtype=self.param_dtype).unsqueeze(0)
temporal_latents = hr_context_latents[:, -self.temporal_size:].to(dtype=self.param_dtype)
temporal_y = hr_y_full[:, chunk_end - self.temporal_size:chunk_end].to(dtype=self.param_dtype)
temporal_input = torch.cat([temporal_latents, temporal_y], dim=0).unsqueeze(0)
compressor_args = {
"forward_mode": "compress",
"hr_latents_input": hr_input,
"lr_latents_input": lr_input,
"lr_action": lr_action[:chunk_end // 2],
"memory_t": torch.zeros(1, device=self.device, dtype=self.param_dtype),
"hr_temporal_latents_input": temporal_input,
"hr_temporal_action": hr_action[chunk_end - self.temporal_size:chunk_end],
"sink_action": (
hr_action[:self.sink_size]
if self.sink_size > 0
else None
),
}
if self.offload_model:
if next(self.high_noise_model.parameters()).device.type != "cuda":
self.high_noise_model.to(self.device)
if next(self.low_noise_model.parameters()).device.type == "cuda":
self.low_noise_model.to("cpu")
memory_dict_high = self.high_noise_model(**compressor_args)
if self.ar_cache:
kv_high, kv_high_neg = self._prefill_kv_cache(
self.high_noise_model,
memory_dict_high,
kv_high,
kv_high_neg,
context=context,
context_null=context_null,
use_cfg=self.use_cfg_high,
)
if self.offload_model:
if next(self.low_noise_model.parameters()).device.type != "cuda":
self.low_noise_model.to(self.device)
if next(self.high_noise_model.parameters()).device.type == "cuda":
self.high_noise_model.to("cpu")
memory_dict_low = self.low_noise_model(**compressor_args)
if self.ar_cache:
kv_low, kv_low_neg = self._prefill_kv_cache(
self.low_noise_model,
memory_dict_low,
kv_low,
kv_low_neg,
context=context,
context_null=context_null,
use_cfg=self.use_cfg_low,
)
return (
decoded_rgb.cpu(),
memory_dict_high,
memory_dict_low,
kv_high,
kv_high_neg,
kv_low,
kv_low_neg,
)
# ------------------------------------------------------------------
# public entry: chunked autoregressive generation
# ------------------------------------------------------------------
def generate_chunked(
self,
prompts: dict[str, str],
chunk_prompts: tuple[str, ...],
img,
hr_action: torch.Tensor,
total_latent_frames: int = 16,
chunk_length: int = 4,
shift: float = 5.0,
sample_solver: str = 'unipc',
sampling_steps: int = 40,
guide_scale=5.0,
n_prompt: str = "",
seed: int = -1,
offload_model: bool = True,
sink_size: int = 0,
temporal_size: int = 2,
height: int = 448,
width: int = 832,
):
"""Run stage-3 chunked inference. See module docstring for
the recipe. Returns ``[3, T_pix, H_pix, W_pix]`` on rank 0,
``None`` on other ranks.
``sink_size`` (default 0): number of leading hr latent frames used as
the context "sink" (``hr_latents_input[:, :, :sink_size]``, prepended
to the front of the context). 0 disables the sink.
``temporal_size`` (default 2): number of trailing hr latent frames
used as the temporal anchor (``hr_ctx_lat[:, -temporal_size:]``),
appended after the compressed lr+hr context.
"""
assert total_latent_frames % chunk_length == 0, (
f"total_latent_frames ({total_latent_frames}) must be a multiple "
f"of chunk_length ({chunk_length})")
num_chunks = total_latent_frames // chunk_length
if len(chunk_prompts) != num_chunks:
raise ValueError(
f"chunk_prompts must contain {num_chunks} entries, "
f"got {len(chunk_prompts)}"
)
# Conditioning is sample-local. Never carry projected text or
# cross-attention K/V into the next JSON item.
self.clear_conditioning_cache()
if isinstance(guide_scale, (int, float)):
guide_scale = (float(guide_scale), float(guide_scale))
else:
guide_scale = tuple(float(scale) for scale in guide_scale)
if len(guide_scale) != 2:
raise ValueError(
"guide_scale must be a scalar or (low_noise, high_noise)"
)
use_cfg_low = not math.isclose(
guide_scale[0], 1.0, rel_tol=0.0, abs_tol=1e-8)
use_cfg_high = not math.isclose(
guide_scale[1], 1.0, rel_tol=0.0, abs_tol=1e-8)
use_any_cfg = use_cfg_low or use_cfg_high
if self.inference_mode is InferenceMode.PDD and use_any_cfg:
raise ValueError("PDD consistency inference requires guide_scale=1")
self.chunk_length = chunk_length
self.temporal_size = temporal_size
self.sink_size = sink_size
self.offload_model = offload_model
self.ar_cache = (
self.denoise_forward_mode in {"ar_denoise", "pdd_denoise"}
)
self.guide_scale = guide_scale
self.use_cfg_high = use_cfg_high
self.use_cfg_low = use_cfg_low
self.lr_latents_cache = None
logging.info(
"CFG enabled: low_noise=%s high_noise=%s",
use_cfg_low,
use_cfg_high,
)
# ---- preprocess image ----
img = TF.to_tensor(img).sub_(0.5).div_(0.5).to(self.device)
h, w = height, width
lat_h = h // self.vae_stride[1] # 56
lat_w = w // self.vae_stride[2] # 104
img_pixel = F.interpolate(img[None].cpu(), size=(h, w),
mode='bicubic')[0].to(self.device)
# ---- text encode ----
if n_prompt == "":
n_prompt = self.sample_neg_prompt
prompt_names = tuple(dict.fromkeys(chunk_prompts))
prompt_texts = [prompts[name] for name in prompt_names]
encoded_prompts = self.text_encoder(prompt_texts, self.device)
context_null = (
self.text_encoder([n_prompt], self.device)
if use_any_cfg else None
)
prompt_contexts = {
name: [encoded]
for name, encoded in zip(prompt_names, encoded_prompts)
}
# ---- pre-build hr_y_full / lr_y_full (upstream WanI2V recipe:
# encode the full pixel clip [image, zeros..., zeros] in one shot
# so the temporally-causal VAE produces the correct latent volume).
with torch.no_grad():
hr_y_full = _build_y(
self.vae, img_pixel,
t_lat=total_latent_frames,
h_pix=h, w_pix=w,
device=self.device, dtype=self.param_dtype)
img_pixel_lr = self._build_lr_image_pixel(img_pixel, h, w)
lr_y_full = _build_y(
self.vae, img_pixel_lr,
t_lat=total_latent_frames // 2,
h_pix=h // 4, w_pix=w // 4,
device=self.device, dtype=self.param_dtype)
# ---- noise allocation for the whole HR latent volume ----
seed = seed if seed >= 0 else random.randint(0, sys.maxsize)
seed_g = torch.Generator(device=self.device)
seed_g.manual_seed(seed)
noise_full = torch.randn(
16, total_latent_frames, lat_h, lat_w,
dtype=torch.float32, generator=seed_g, device=self.device)
latents = noise_full.clone()
# ---- hr_action sanity ----
hr_action = hr_action.to(self.device)
assert hr_action.shape[0] >= total_latent_frames, (
f"hr_action must have >= total_latent_frames ({total_latent_frames}) "
f"rows; got {hr_action.shape[0]}.")
lr_action = hr_action[::2][:total_latent_frames // 2]
decoded_rgb_cache: list = []
# Chunk-AR always keeps conditional caches. Negative caches are only
# allocated for experts whose guide scale actually enables CFG.
ar_cache = self.ar_cache
# PERSISTENT KV caches, created ONCE and reused across all chunks.
# The unified prefill path is incremental: each chunk only pushes the
# new cmp block + recomputed tmp anchor. Each layer stores one
# contiguous [sink | accumulated cmp | latest tmp] K/V pair instead
# of re-prefilling the whole growing ctx. When AR but no KV cache,
# ar_cache=False and the denoise falls back to the fused [ctx|pred] AR
# path (memory_dict). BI leaves them None.
if ar_cache:
kv_high = self.high_noise_model.init_kv_cache()
kv_low = self.low_noise_model.init_kv_cache()
kv_high_neg = (
self.high_noise_model.init_kv_cache()
if use_cfg_high else None
)
kv_low_neg = (
self.low_noise_model.init_kv_cache()
if use_cfg_low else None
)
else:
kv_high = kv_high_neg = kv_low = kv_low_neg = None
memory_dict_high = None
memory_dict_low = None
boundary = self.boundary * self.num_train_timesteps
no_sync_low = getattr(self.low_noise_model, 'no_sync', _noop_no_sync)
no_sync_high = getattr(self.high_noise_model, 'no_sync', _noop_no_sync)
with (torch.amp.autocast('cuda', dtype=self.param_dtype),
torch.no_grad(), no_sync_low(), no_sync_high()):
# Exact gen token count -- no rounding to a multiple of sp_size.
# The unified SP forwards (BI + AR) chunk the sequence UNEVENLY
# (uneven all_to_all) and attend with SDPA over the true
# sequence, so any extra zero-padded rows would leak into the
# attention (BI: as real tokens; AR: into the chunk-causal mask).
seq_len_chunk = chunk_length * lat_h * lat_w // (
self.patch_size[1] * self.patch_size[2])
self.seq_len_chunk = seq_len_chunk
for chunk_i in range(num_chunks):
s = chunk_i * chunk_length
e = (chunk_i + 1) * chunk_length
prompt_name = chunk_prompts[chunk_i]
context = prompt_contexts[prompt_name]
logging.info(f"[stage3] chunk {chunk_i}/{num_chunks - 1} "
f"latent frames [{s}, {e}) prompt={prompt_name}")
# Per-chunk denoise inputs (also needed by AR prefill below).
latent = latents[:, s:e].contiguous()
hr_action_chunk = hr_action[s:e]
y_chunk = hr_y_full[:, s:e]
chunk_denoise_args = dict(
latent=latent,
seed_g=seed_g,
boundary=boundary,
context=context,
context_null=context_null,
memory_dict_high=memory_dict_high,
memory_dict_low=memory_dict_low,
kv_high=kv_high,
kv_high_neg=kv_high_neg,
kv_low=kv_low,
kv_low_neg=kv_low_neg,
y_chunk=y_chunk,
hr_action_chunk=hr_action_chunk,
chunk_start=s,
)
if self.inference_mode is InferenceMode.PDD:
latent = self._run_pdd_chunk(
latent=latent,
y_chunk=y_chunk,
hr_action_chunk=hr_action_chunk,
context=context,
memory_dict_high=memory_dict_high,
memory_dict_low=memory_dict_low,
kv_high=kv_high,
kv_low=kv_low,
chunk_start=s,
seed_g=seed_g,
)
elif self.inference_mode is InferenceMode.BI:
latent = self._run_solver_chunk(
sample_solver=sample_solver,
sampling_steps=sampling_steps,
shift=shift,
ar_cache=False,
chunk_index=chunk_i,
**chunk_denoise_args,
)
elif self.inference_mode is InferenceMode.AR:
latent = self._run_solver_chunk(
sample_solver=sample_solver,
sampling_steps=sampling_steps,
shift=shift,
ar_cache=True,
chunk_index=chunk_i,
**chunk_denoise_args,
)
else:
raise RuntimeError(
f"Unsupported inference mode: {self.inference_mode}"
)
latents[:, s:e] = latent.to(latents.dtype)
if chunk_i < num_chunks - 1:
(
decoded_rgb,
memory_dict_high,
memory_dict_low,
kv_high,
kv_high_neg,
kv_low,
kv_low_neg,
) = self._prepare_chunk_memory(
chunk_index=chunk_i,
chunk_end=e,
latents=latents,
hr_y_full=hr_y_full,
lr_y_full=lr_y_full,
hr_action=hr_action,
lr_action=lr_action,
context=context,
context_null=context_null,
kv_high=kv_high,
kv_high_neg=kv_high_neg,
kv_low=kv_low,
kv_low_neg=kv_low_neg,
)
decoded_rgb_cache.append(decoded_rgb)
elif self.rank == 0:
final_rgb = self.vae.decode_chunk(
[latent],
is_first_chunk=chunk_i == 0,
)[0].cpu()
decoded_rgb_cache.append(final_rgb)
if self.rank == 0:
video = torch.cat(decoded_rgb_cache, dim=1)
else:
video = None
if offload_model:
self.clear_conditioning_cache()
self.low_noise_model.cpu()
self.high_noise_model.cpu()
torch.cuda.empty_cache()
if offload_model:
gc.collect()
torch.cuda.synchronize()
if dist.is_initialized():
dist.barrier(device_ids=[self.device.index])
return video