WorldPlay2: Extending Real-Time Interactive World Models in Control and Horizon
Python
from __future__ import annotations
import logging
from functools import lru_cache
import torch
import torch.nn.functional as F
try:
import triton
import triton.language as tl
except ImportError:
triton = None
tl = None
_QK_ROPE_STATS = {
"compiled_calls": 0,
"eager_calls": 0,
"compile_failures": 0,
}
_REGION_STATS = {
"layer_norm_modulation": {
"triton_calls": 0,
"eager_calls": 0,
"failures": 0,
},
"modulation": {"compiled_calls": 0, "eager_calls": 0, "failures": 0},
"gated_residual": {
"compiled_calls": 0,
"eager_calls": 0,
"failures": 0,
},
"action_mlp": {
"compiled_calls": 0,
"eager_calls": 0,
"failures": 0,
},
"ffn": {"compiled_calls": 0, "eager_calls": 0, "failures": 0},
}
_DISABLED_REGIONS: set[str] = set()
if triton is not None:
@triton.jit
def _layer_norm_modulation_kernel(
x_ptr,
scale_ptr,
shift_ptr,
output_ptr,
sequence_length: tl.constexpr,
hidden_size: tl.constexpr,
x_stride_b: tl.constexpr,
x_stride_s: tl.constexpr,
scale_stride_b: tl.constexpr,
scale_stride_s: tl.constexpr,
shift_stride_b: tl.constexpr,
shift_stride_s: tl.constexpr,
eps: tl.constexpr,
INPUT_DTYPE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""One Triton program normalizes and modulates one token row."""
row = tl.program_id(0)
batch_index = row // sequence_length
sequence_index = row - batch_index * sequence_length
columns = tl.arange(0, BLOCK_SIZE)
mask = columns < hidden_size
x_offsets = (
batch_index * x_stride_b
+ sequence_index * x_stride_s
+ columns
)
x = tl.load(x_ptr + x_offsets, mask=mask, other=0.0).to(tl.float32)
mean = tl.sum(x, axis=0) / hidden_size
centered = tl.where(mask, x - mean, 0.0)
variance = tl.sum(centered * centered, axis=0) / hidden_size
normalized = centered * tl.rsqrt(variance + eps)
# Match WorldPlay2LayerNorm exactly: FP32 normalization is rounded back to
# the activation dtype before adaptive modulation evaluates in FP32.
if INPUT_DTYPE == 1:
normalized = normalized.to(tl.bfloat16).to(tl.float32)
elif INPUT_DTYPE == 0:
normalized = normalized.to(tl.float16).to(tl.float32)
scale_offsets = (
batch_index * scale_stride_b
+ sequence_index * scale_stride_s
+ columns
)
shift_offsets = (
batch_index * shift_stride_b
+ sequence_index * shift_stride_s
+ columns
)
scale = tl.load(
scale_ptr + scale_offsets, mask=mask, other=0.0
).to(tl.float32)
shift = tl.load(
shift_ptr + shift_offsets, mask=mask, other=0.0
).to(tl.float32)
output = normalized * (1.0 + scale) + shift
output_offsets = row * hidden_size + columns
tl.store(output_ptr + output_offsets, output, mask=mask)
def qk_norm_rope_eager(
q: torch.Tensor,
k: torch.Tensor,
norm_q_weight: torch.Tensor,
norm_k_weight: torch.Tensor,
freqs_real: torch.Tensor,
freqs_imag: torch.Tensor,
*,
eps: float,
num_heads: int,
head_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""RMS-normalize Q/K and apply real-valued RoPE in FP32."""
def normalize_and_rotate(x, weight):
normalized = x.float()
normalized = normalized * torch.rsqrt(
normalized.square().mean(dim=-1, keepdim=True) + eps
)
# Preserve the original WorldPlay2RMSNorm order: normalize in FP32, cast
# back to projection dtype, then apply the learned norm weight.
normalized = normalized.to(x.dtype) * weight
batch, sequence, _ = normalized.shape
pairs = normalized.float().view(
batch, sequence, num_heads, head_dim // 2, 2)
even = pairs[..., 0]
odd = pairs[..., 1]
cos = freqs_real.float().view(
1, sequence, 1, head_dim // 2)
sin = freqs_imag.float().view(
1, sequence, 1, head_dim // 2)
return torch.stack(
[even * cos - odd * sin, even * sin + odd * cos],
dim=-1,
).flatten(-2)
return (
normalize_and_rotate(q, norm_q_weight),
normalize_and_rotate(k, norm_k_weight),
)
@lru_cache(maxsize=16)
def _compiled_kernel(num_heads: int, head_dim: int, eps: float):
def kernel(q, k, norm_q_weight, norm_k_weight, freqs_real, freqs_imag):
return qk_norm_rope_eager(
q,
k,
norm_q_weight,
norm_k_weight,
freqs_real,
freqs_imag,
eps=eps,
num_heads=num_heads,
head_dim=head_dim,
)
return torch.compile(
kernel,
fullgraph=True,
dynamic=True,
mode="max-autotune-no-cudagraphs",
)
def qk_norm_rope(
q: torch.Tensor,
k: torch.Tensor,
norm_q_weight: torch.Tensor,
norm_k_weight: torch.Tensor,
freqs_flat: torch.Tensor,
*,
eps: float,
num_heads: int,
head_dim: int,
compile_enabled: bool,
) -> tuple[torch.Tensor, torch.Tensor, bool]:
"""Run compiled QK norm + RoPE, falling back to the same eager kernel."""
args = (
q,
k,
norm_q_weight,
norm_k_weight,
freqs_flat.real,
freqs_flat.imag,
)
if compile_enabled:
try:
output = _compiled_kernel(num_heads, head_dim, eps)(*args)
_QK_ROPE_STATS["compiled_calls"] += 1
return output[0], output[1], True
except Exception:
_QK_ROPE_STATS["compile_failures"] += 1
logging.exception(
"Compiled QK norm + RoPE failed; using eager fallback")
output = qk_norm_rope_eager(
*args,
eps=eps,
num_heads=num_heads,
head_dim=head_dim,
)
_QK_ROPE_STATS["eager_calls"] += 1
return output[0], output[1], False
def get_qk_rope_compile_stats() -> dict:
return dict(_QK_ROPE_STATS)
# ---------------------------------------------------------------------------
# Repeated Wan block regions
# ---------------------------------------------------------------------------
def layer_norm_modulation_eager(
hidden_states,
scale,
shift,
*,
eps,
):
"""Reference Wan LayerNorm followed by adaptive scale and shift."""
normalized = F.layer_norm(
hidden_states.float(),
(hidden_states.shape[-1],),
weight=None,
bias=None,
eps=eps,
).to(hidden_states.dtype)
return normalized.float() * (1.0 + scale.float()) + shift.float()
def layer_norm_modulation(
hidden_states,
scale,
shift,
*,
eps,
enabled,
):
"""Fuse non-affine LayerNorm and Wan modulation in one Triton kernel."""
name = "layer_norm_modulation"
can_use_triton = (
enabled
and name not in _DISABLED_REGIONS
and triton is not None
and hidden_states.is_cuda
and hidden_states.ndim == 3
and scale.ndim == 3
and shift.ndim == 3
and hidden_states.shape == scale.shape == shift.shape
and hidden_states.dtype in (
torch.bfloat16,
torch.float16,
torch.float32,
)
and hidden_states.stride(-1) == 1
and scale.stride(-1) == 1
and shift.stride(-1) == 1
)
if can_use_triton:
try:
batch, sequence, hidden_size = hidden_states.shape
output = torch.empty(
hidden_states.shape,
device=hidden_states.device,
dtype=torch.float32,
)
block_size = triton.next_power_of_2(hidden_size)
if block_size > 65536:
raise ValueError(
f"LayerNorm hidden size {hidden_size} is too large")
num_warps = 8 if block_size >= 4096 else 4
_layer_norm_modulation_kernel[(batch * sequence,)](
hidden_states,
scale,
shift,
output,
sequence_length=sequence,
hidden_size=hidden_size,
x_stride_b=hidden_states.stride(0),
x_stride_s=hidden_states.stride(1),
scale_stride_b=scale.stride(0),
scale_stride_s=scale.stride(1),
shift_stride_b=shift.stride(0),
shift_stride_s=shift.stride(1),
eps=eps,
INPUT_DTYPE=(
1 if hidden_states.dtype == torch.bfloat16
else 0 if hidden_states.dtype == torch.float16
else 2
),
BLOCK_SIZE=block_size,
num_warps=num_warps,
)
_REGION_STATS[name]["triton_calls"] += 1
return output, True
except Exception:
_REGION_STATS[name]["failures"] += 1
_DISABLED_REGIONS.add(name)
logging.exception(
"Triton fused LayerNorm + modulation failed; "
"disabling it and using eager fallback"
)
_REGION_STATS[name]["eager_calls"] += 1
return layer_norm_modulation_eager(
hidden_states,
scale,
shift,
eps=eps,
), False
def modulation_eager(normalized, scale, shift):
"""Exact block modulation expression, evaluated in FP32."""
return normalized.float() * (1.0 + scale.float()) + shift.float()
def gated_residual_eager(hidden_states, branch, gate):
"""Exact gated residual expression used by WorldPlay2AttentionBlock."""
return hidden_states + branch * gate
def action_mlp_eager(
hidden_states,
action,
input_weight,
input_bias,
output_weight,
output_bias,
):
"""Action injection: concat, MLP, and residual add."""
branch = torch.cat([hidden_states, action], dim=-1)
branch = F.linear(branch, input_weight, input_bias)
branch = F.silu(branch)
branch = F.linear(branch, output_weight, output_bias)
return hidden_states + branch
def ffn_eager(
hidden_states,
input_weight,
input_bias,
output_weight,
output_bias,
):
hidden_states = F.linear(
hidden_states, input_weight, input_bias)
hidden_states = F.gelu(hidden_states, approximate="tanh")
return F.linear(hidden_states, output_weight, output_bias)
@lru_cache(maxsize=1)
def _compiled_modulation():
return torch.compile(
modulation_eager,
fullgraph=True,
dynamic=True,
mode="max-autotune-no-cudagraphs",
)
@lru_cache(maxsize=1)
def _compiled_gated_residual():
return torch.compile(
gated_residual_eager,
fullgraph=True,
dynamic=True,
mode="max-autotune-no-cudagraphs",
)
@lru_cache(maxsize=1)
def _compiled_action_mlp():
return torch.compile(
action_mlp_eager,
fullgraph=True,
dynamic=True,
mode="max-autotune-no-cudagraphs",
)
@lru_cache(maxsize=1)
def _compiled_ffn():
return torch.compile(
ffn_eager,
fullgraph=True,
dynamic=True,
mode="max-autotune-no-cudagraphs",
)
def _run_region(name, eager, compiled_factory, args, enabled):
if enabled and name not in _DISABLED_REGIONS:
try:
output = compiled_factory()(*args)
_REGION_STATS[name]["compiled_calls"] += 1
return output, True
except Exception:
_REGION_STATS[name]["failures"] += 1
_DISABLED_REGIONS.add(name)
logging.exception(
"Compiled region %s failed; disabling it and using eager",
name,
)
_REGION_STATS[name]["eager_calls"] += 1
return eager(*args), False
def modulation(normalized, scale, shift, *, enabled):
return _run_region(
"modulation",
modulation_eager,
_compiled_modulation,
(normalized, scale, shift),
enabled,
)
def gated_residual(hidden_states, branch, gate, *, enabled):
return _run_region(
"gated_residual",
gated_residual_eager,
_compiled_gated_residual,
(hidden_states, branch, gate),
enabled,
)
def action_mlp(
hidden_states,
action,
input_weight,
input_bias,
output_weight,
output_bias,
*,
enabled,
):
return _run_region(
"action_mlp",
action_mlp_eager,
_compiled_action_mlp,
(
hidden_states,
action,
input_weight,
input_bias,
output_weight,
output_bias,
),
enabled,
)
def ffn(
hidden_states,
input_weight,
input_bias,
output_weight,
output_bias,
*,
enabled,
):
return _run_region(
"ffn",
ffn_eager,
_compiled_ffn,
(
hidden_states,
input_weight,
input_bias,
output_weight,
output_bias,
),
enabled,
)
def get_compiled_region_stats() -> dict:
return {
name: dict(values)
for name, values in _REGION_STATS.items()
} | {
"disabled_after_failure": sorted(_DISABLED_REGIONS),
}