WorldPlay2: Extending Real-Time Interactive World Models in Control and Horizon
Python
# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.
from functools import lru_cache
import torch
try:
import flash_attn_interface
FLASH_ATTN_3_AVAILABLE = True
except ModuleNotFoundError:
flash_attn_interface = None
FLASH_ATTN_3_AVAILABLE = False
try:
import flash_attn
FLASH_ATTN_2_AVAILABLE = True
except ModuleNotFoundError:
flash_attn = None
FLASH_ATTN_2_AVAILABLE = False
import warnings
__all__ = [
'flash_attention',
'flash_attention_dense',
'sage_attention',
'resolve_attention_backend',
'attention',
]
@lru_cache(maxsize=1)
def _load_sage_attention():
"""Load the optional CUDA backend only when it is selected."""
try:
from sageattention import sageattn
except (ImportError, OSError) as exc:
raise RuntimeError(
"attention_backend='sage' requires a working SageAttention "
"installation. Install it with `pip install sageattention "
"--no-build-isolation` in your CUDA environment, or use "
"attention_backend='flash'."
) from exc
return sageattn
def resolve_attention_backend(backend: str) -> str:
"""Normalize the backend name and check optional dependencies early."""
backend = backend.lower()
if backend not in ('flash', 'sage'):
raise ValueError(
"attention_backend must be 'flash' or 'sage', "
f"got {backend!r}"
)
if backend == 'sage':
_load_sage_attention()
return backend
def sage_attention(
q,
k,
v,
*,
k_lens=None,
softmax_scale=None,
causal=False,
window_size=(-1, -1),
):
"""Dense SageAttention in the model's ``[B, S, H, D]`` layout.
Only full key sequences are supported here. Masked history prefill and
text cross-attention keep their existing SDPA/FlashAttention paths.
"""
if tuple(window_size) != (-1, -1):
raise NotImplementedError(
"SageAttention does not support sliding-window attention in "
"WorldPlay2; use attention_backend='flash'."
)
if k_lens is not None and not torch.all(k_lens == k.shape[1]):
raise NotImplementedError(
"SageAttention requires dense, equal-length keys; "
"use attention_backend='flash' for padded/variable lengths."
)
return _load_sage_attention()(
q.contiguous(),
k.contiguous(),
v.contiguous(),
tensor_layout='NHD',
is_causal=causal,
sm_scale=softmax_scale,
)
def flash_attention_dense(
q,
k,
v,
*,
softmax_scale=None,
causal=False,
):
"""Dense attention using FA3 when available, otherwise PyTorch SDPA.
Inputs and output use the model-native layout ``[B, S, H, D]``. This is
preferable to the varlen wrapper when every batch item has the same
sequence length because it avoids flattening and constructing cu_seqlens.
"""
if FLASH_ATTN_3_AVAILABLE:
return flash_attn_interface.flash_attn_func(
q=q.contiguous(),
k=k.contiguous(),
v=v.contiguous(),
softmax_scale=softmax_scale,
causal=causal,
)
out = torch.nn.functional.scaled_dot_product_attention(
q.transpose(1, 2),
k.transpose(1, 2),
v.transpose(1, 2),
is_causal=causal,
scale=softmax_scale,
)
return out.transpose(1, 2).contiguous()
def flash_attention(
q,
k,
v,
q_lens=None,
k_lens=None,
dropout_p=0.,
softmax_scale=None,
q_scale=None,
causal=False,
window_size=(-1, -1),
deterministic=False,
dtype=torch.bfloat16,
version=None,
):
"""
q: [B, Lq, Nq, C1].
k: [B, Lk, Nk, C1].
v: [B, Lk, Nk, C2]. Nq must be divisible by Nk.
q_lens: [B].
k_lens: [B].
dropout_p: float. Dropout probability.
softmax_scale: float. The scaling of QK^T before applying softmax.
causal: bool. Whether to apply causal attention mask.
window_size: (left right). If not (-1, -1), apply sliding window local attention.
deterministic: bool. If True, slightly slower and uses more memory.
dtype: torch.dtype. Apply when dtype of q/k/v is not float16/bfloat16.
"""
half_dtypes = (torch.float16, torch.bfloat16)
assert dtype in half_dtypes
assert q.device.type == 'cuda' and q.size(-1) <= 256
# params
b, lq, lk, out_dtype = q.size(0), q.size(1), k.size(1), q.dtype
def half(x):
return x if x.dtype in half_dtypes else x.to(dtype)
# preprocess query
if q_lens is None:
q = half(q.flatten(0, 1))
q_lens = torch.tensor(
[lq] * b, dtype=torch.int32).to(
device=q.device, non_blocking=True)
else:
q = half(torch.cat([u[:v] for u, v in zip(q, q_lens)]))
# preprocess key, value
if k_lens is None:
k = half(k.flatten(0, 1))
v = half(v.flatten(0, 1))
k_lens = torch.tensor(
[lk] * b, dtype=torch.int32).to(
device=k.device, non_blocking=True)
else:
k = half(torch.cat([u[:v] for u, v in zip(k, k_lens)]))
v = half(torch.cat([u[:v] for u, v in zip(v, k_lens)]))
q = q.to(v.dtype)
k = k.to(v.dtype)
if q_scale is not None:
q = q * q_scale
if version is not None and version == 3 and not FLASH_ATTN_3_AVAILABLE:
warnings.warn(
'Flash attention 3 is not available, use flash attention 2 instead.'
)
# apply attention
if (version is None or version == 3) and FLASH_ATTN_3_AVAILABLE:
# Note: dropout_p, window_size are not supported in FA3 now.
x = flash_attn_interface.flash_attn_varlen_func(
q=q,
k=k,
v=v,
cu_seqlens_q=torch.cat([q_lens.new_zeros([1]), q_lens]).cumsum(
0, dtype=torch.int32).to(q.device, non_blocking=True),
cu_seqlens_k=torch.cat([k_lens.new_zeros([1]), k_lens]).cumsum(
0, dtype=torch.int32).to(q.device, non_blocking=True),
seqused_q=None,
seqused_k=None,
max_seqlen_q=lq,
max_seqlen_k=lk,
softmax_scale=softmax_scale,
causal=causal,
deterministic=deterministic).unflatten(0, (b, lq))
else:
assert FLASH_ATTN_2_AVAILABLE
x = flash_attn.flash_attn_varlen_func(
q=q,
k=k,
v=v,
cu_seqlens_q=torch.cat([q_lens.new_zeros([1]), q_lens]).cumsum(
0, dtype=torch.int32).to(q.device, non_blocking=True),
cu_seqlens_k=torch.cat([k_lens.new_zeros([1]), k_lens]).cumsum(
0, dtype=torch.int32).to(q.device, non_blocking=True),
max_seqlen_q=lq,
max_seqlen_k=lk,
dropout_p=dropout_p,
softmax_scale=softmax_scale,
causal=causal,
window_size=window_size,
deterministic=deterministic).unflatten(0, (b, lq))
# output
return x.type(out_dtype)
def attention(
q,
k,
v,
q_lens=None,
k_lens=None,
dropout_p=0.,
softmax_scale=None,
q_scale=None,
causal=False,
window_size=(-1, -1),
deterministic=False,
dtype=torch.bfloat16,
fa_version=None,
):
if FLASH_ATTN_2_AVAILABLE or FLASH_ATTN_3_AVAILABLE:
return flash_attention(
q=q,
k=k,
v=v,
q_lens=q_lens,
k_lens=k_lens,
dropout_p=dropout_p,
softmax_scale=softmax_scale,
q_scale=q_scale,
causal=causal,
window_size=window_size,
deterministic=deterministic,
dtype=dtype,
version=fa_version,
)
else:
if q_lens is not None or k_lens is not None:
warnings.warn(
'Padding mask is disabled when using scaled_dot_product_attention. It can have a significant impact on performance.'
)
attn_mask = None
q = q.transpose(1, 2).to(dtype)
k = k.transpose(1, 2).to(dtype)
v = v.transpose(1, 2).to(dtype)
out = torch.nn.functional.scaled_dot_product_attention(
q, k, v, attn_mask=attn_mask, is_causal=causal, dropout_p=dropout_p)
out = out.transpose(1, 2).contiguous()
return out