Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[feature] impl cached pos embedding (#1)

Hongxin Liu8d5278b99a30
diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.py
index 24e6416..b8824ce 100644
--- a/opensora/models/layers/blocks.py
+++ b/opensora/models/layers/blocks.py
@@ -9,7 +9,9 @@
# MAE: https://github.com/facebookresearch/mae/blob/main/models_mae.py
# --------------------------------------------------------
+import functools
import math
+from typing import Optional
import numpy as np
import torch
@@ -27,7 +29,9 @@ from opensora.acceleration.parallel_states import get_sequence_parallel_group
approx_gelu = lambda: nn.GELU(approximate="tanh")
-def get_layernorm(hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool):
+def get_layernorm(
+ hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool
+):
if use_kernel:
try:
from apex.normalization import FusedLayerNorm
@@ -82,7 +86,9 @@ class PatchEmbed3D(nn.Module):
self.in_chans = in_chans
self.embed_dim = embed_dim
- self.proj = nn.Conv3d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+ self.proj = nn.Conv3d(
+ in_chans, embed_dim, kernel_size=patch_size, stride=patch_size
+ )
if norm_layer is not None:
self.norm = norm_layer(embed_dim)
else:
@@ -215,7 +221,9 @@ class SeqParallelAttention(Attention):
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
- B, N, C = x.shape # for sequence parallel here, the N is a local sequence length
+ B, N, C = (
+ x.shape
+ ) # for sequence parallel here, the N is a local sequence length
qkv = self.qkv(x)
qkv_shape = (B, N, 3, self.num_heads, self.head_dim)
@@ -228,9 +236,21 @@ class SeqParallelAttention(Attention):
qkv = all_to_all(qkv, sp_group, scatter_dim=3, gather_dim=1)
if self.enable_flashattn:
- qkv_permute_shape = (2, 0, 1, 3, 4) # [3, B, N, NUM_HEAD_PER_DEVICE, HEAD_DIM]
+ qkv_permute_shape = (
+ 2,
+ 0,
+ 1,
+ 3,
+ 4,
+ ) # [3, B, N, NUM_HEAD_PER_DEVICE, HEAD_DIM]
else:
- qkv_permute_shape = (2, 0, 3, 1, 4) # [3, B, NUM_HEAD_PER_DEVICE, N, HEAD_DIM]
+ qkv_permute_shape = (
+ 2,
+ 0,
+ 3,
+ 1,
+ 4,
+ ) # [3, B, NUM_HEAD_PER_DEVICE, N, HEAD_DIM]
qkv = qkv.permute(qkv_permute_shape)
q, k, v = qkv.unbind(0)
@@ -296,7 +316,9 @@ class MultiHeadCrossAttention(nn.Module):
attn_bias = None
if mask is not None:
attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)
- x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)
+ x = xformers.ops.memory_efficient_attention(
+ q, k, v, p=self.attn_drop.p, attn_bias=attn_bias
+ )
x = x.view(B, -1, C)
x = self.proj(x)
@@ -312,7 +334,12 @@ class SeqParallelMultiHeadCrossAttention(MultiHeadCrossAttention):
attn_drop=0.0,
proj_drop=0.0,
):
- super().__init__(d_model=d_model, num_heads=num_heads, attn_drop=attn_drop, proj_drop=proj_drop)
+ super().__init__(
+ d_model=d_model,
+ num_heads=num_heads,
+ attn_drop=attn_drop,
+ proj_drop=proj_drop,
+ )
def forward(self, x, cond, mask=None):
# query/value: img tokens; key: condition; mask: if padding tokens
@@ -330,8 +357,12 @@ class SeqParallelMultiHeadCrossAttention(MultiHeadCrossAttention):
# apply all_to_all to gather sequence and split attention heads
q = all_to_all(q, sp_group, scatter_dim=2, gather_dim=1)
- k = split_forward_gather_backward(k, get_sequence_parallel_group(), dim=2, grad_scale="down")
- v = split_forward_gather_backward(v, get_sequence_parallel_group(), dim=2, grad_scale="down")
+ k = split_forward_gather_backward(
+ k, get_sequence_parallel_group(), dim=2, grad_scale="down"
+ )
+ v = split_forward_gather_backward(
+ v, get_sequence_parallel_group(), dim=2, grad_scale="down"
+ )
q = q.view(1, -1, self.num_heads // sp_size, self.head_dim)
k = k.view(1, -1, self.num_heads // sp_size, self.head_dim)
@@ -341,7 +372,9 @@ class SeqParallelMultiHeadCrossAttention(MultiHeadCrossAttention):
attn_bias = None
if mask is not None:
attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)
- x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)
+ x = xformers.ops.memory_efficient_attention(
+ q, k, v, p=self.attn_drop.p, attn_bias=attn_bias
+ )
# apply all to all to gather back attention heads and scatter sequence
x = x.view(B, -1, self.num_heads // sp_size, self.head_dim)
@@ -363,7 +396,9 @@ class FinalLayer(nn.Module):
super().__init__()
self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)
- self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True))
+ self.adaLN_modulation = nn.Sequential(
+ nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)
+ )
def forward(self, x, c):
shift, scale = self.adaLN_modulation(c).chunk(2, dim=1)
@@ -381,7 +416,9 @@ class T2IFinalLayer(nn.Module):
super().__init__()
self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)
- self.scale_shift_table = nn.Parameter(torch.randn(2, hidden_size) / hidden_size**0.5)
+ self.scale_shift_table = nn.Parameter(
+ torch.randn(2, hidden_size) / hidden_size**0.5
+ )
self.out_channels = out_channels
self.d_t = d_t
self.d_s = d_s
@@ -404,7 +441,9 @@ class T2IFinalLayer(nn.Module):
shift, scale = (self.scale_shift_table[None] + t[:, None]).chunk(2, dim=1)
x = t2i_modulate(self.norm_final(x), shift, scale)
if x_mask is not None:
- shift_zero, scale_zero = (self.scale_shift_table[None] + t0[:, None]).chunk(2, dim=1)
+ shift_zero, scale_zero = (self.scale_shift_table[None] + t0[:, None]).chunk(
+ 2, dim=1
+ )
x_zero = t2i_modulate(self.norm_final(x), shift_zero, scale_zero)
x = self.t_mask_select(x_mask, x, x_zero, T, S)
x = self.linear(x)
@@ -442,12 +481,18 @@ class TimestepEmbedder(nn.Module):
"""
# https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
half = dim // 2
- freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half)
+ freqs = torch.exp(
+ -math.log(max_period)
+ * torch.arange(start=0, end=half, dtype=torch.float32)
+ / half
+ )
freqs = freqs.to(device=t.device)
args = t[:, None].float() * freqs[None]
embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
if dim % 2:
- embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
+ embedding = torch.cat(
+ [embedding, torch.zeros_like(embedding[:, :1])], dim=-1
+ )
return embedding
def forward(self, t, dtype):
@@ -466,7 +511,9 @@ class LabelEmbedder(nn.Module):
def __init__(self, num_classes, hidden_size, dropout_prob):
super().__init__()
use_cfg_embedding = dropout_prob > 0
- self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)
+ self.embedding_table = nn.Embedding(
+ num_classes + use_cfg_embedding, hidden_size
+ )
self.num_classes = num_classes
self.dropout_prob = dropout_prob
@@ -494,7 +541,9 @@ class SizeEmbedder(TimestepEmbedder):
"""
def __init__(self, hidden_size, frequency_embedding_size=256):
- super().__init__(hidden_size=hidden_size, frequency_embedding_size=frequency_embedding_size)
+ super().__init__(
+ hidden_size=hidden_size, frequency_embedding_size=frequency_embedding_size
+ )
self.mlp = nn.Sequential(
nn.Linear(frequency_embedding_size, hidden_size, bias=True),
nn.SiLU(),
@@ -512,7 +561,9 @@ class SizeEmbedder(TimestepEmbedder):
assert s.shape[0] == bs
b, dims = s.shape[0], s.shape[1]
s = rearrange(s, "b d -> (b d)")
- s_freq = self.timestep_embedding(s, self.frequency_embedding_size).to(self.dtype)
+ s_freq = self.timestep_embedding(s, self.frequency_embedding_size).to(
+ self.dtype
+ )
s_emb = self.mlp(s_freq)
s_emb = rearrange(s_emb, "(b d) d2 -> b (d d2)", b=b, d=dims, d2=self.outdim)
return s_emb
@@ -527,12 +578,26 @@ class CaptionEmbedder(nn.Module):
Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.
"""
- def __init__(self, in_channels, hidden_size, uncond_prob, act_layer=nn.GELU(approximate="tanh"), token_num=120):
+ def __init__(
+ self,
+ in_channels,
+ hidden_size,
+ uncond_prob,
+ act_layer=nn.GELU(approximate="tanh"),
+ token_num=120,
+ ):
super().__init__()
self.y_proj = Mlp(
- in_features=in_channels, hidden_features=hidden_size, out_features=hidden_size, act_layer=act_layer, drop=0
+ in_features=in_channels,
+ hidden_features=hidden_size,
+ out_features=hidden_size,
+ act_layer=act_layer,
+ drop=0,
+ )
+ self.register_buffer(
+ "y_embedding",
+ nn.Parameter(torch.randn(token_num, in_channels) / in_channels**0.5),
)
- self.register_buffer("y_embedding", nn.Parameter(torch.randn(token_num, in_channels) / in_channels**0.5))
self.uncond_prob = uncond_prob
def token_drop(self, caption, force_drop_ids=None):
@@ -556,13 +621,65 @@ class CaptionEmbedder(nn.Module):
return caption
+class PositionEmbedding2D(nn.Module):
+ def __init__(
+ self, dim: int, max_position_embeddings: int = 512, scale: float = 1.0
+ ) -> None:
+ super().__init__()
+ self.dim = dim
+ self.max_position_embeddings = max_position_embeddings
+ self.scale = scale
+ assert dim % 4 == 0, "dim must be divisible by 4"
+ half_dim = dim // 2
+ inv_freq = 1.0 / (10000 ** (torch.arange(0, half_dim, 2).float() / half_dim))
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+ def _get_sin_cos_emb(self, t: torch.Tensor):
+ out = torch.einsum("i,d->id", t, self.inv_freq)
+ emb_cos = torch.cos(out)
+ emb_sin = torch.sin(out)
+ return torch.cat((emb_sin, emb_cos), dim=-1)
+
+ @functools.lru_cache(maxsize=128)
+ def _get_cached_emb(
+ self,
+ device: torch.device,
+ dtype: torch.dtype,
+ h: int,
+ w: int,
+ base_size: Optional[int] = None,
+ ):
+ grid_h = torch.arange(h, device=device) / self.scale
+ grid_w = torch.arange(w, device=device) / self.scale
+ if base_size is not None:
+ grid_h *= base_size / h
+ grid_w *= base_size / w
+ grid_h, grid_w = torch.meshgrid(
+ grid_w,
+ grid_h,
+ indexing="ij",
+ ) # here w goes first
+ grid_h = grid_h.t().reshape(-1)
+ grid_w = grid_w.t().reshape(-1)
+ emb_h = self._get_sin_cos_emb(grid_h)
+ emb_w = self._get_sin_cos_emb(grid_w)
+ return torch.concat([emb_h, emb_w], dim=-1).unsqueeze(0).to(dtype)
+
+ def forward(
+ self, x: torch.Tensor, h: int, w: int, base_size: Optional[int] = None
+ ) -> torch.Tensor:
+ return self._get_cached_emb(x.device, x.dtype, h, w, base_size)
+
+
# ===============================================
# Sine/Cosine Positional Embedding Functions
# ===============================================
# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
-def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0, scale=1.0, base_size=None):
+def get_2d_sincos_pos_embed(
+ embed_dim, grid_size, cls_token=False, extra_tokens=0, scale=1.0, base_size=None
+):
"""
grid_size: int of the grid height and width
return:
@@ -582,7 +699,9 @@ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=
grid = grid.reshape([2, 1, grid_size[1], grid_size[0]])
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
if cls_token and extra_tokens > 0:
- pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)
+ pos_embed = np.concatenate(
+ [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0
+ )
return pos_embed
diff --git a/tests/test_pos_emb.py b/tests/test_pos_emb.py
new file mode 100644
index 0000000..c409acb
--- /dev/null
+++ b/tests/test_pos_emb.py
@@ -0,0 +1,45 @@
+import pytest
+import torch
+
+from opensora.models.layers.blocks import PositionEmbedding2D, get_2d_sincos_pos_embed
+
+D = 8
+SCALE = 2.0
+from torch.testing import assert_close
+
+
+def get_spatial_pos_embed(x, hidden_size, h, w, scale, base_size=None):
+ pos_embed = get_2d_sincos_pos_embed(
+ hidden_size,
+ (h, w),
+ scale=scale,
+ base_size=base_size,
+ )
+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)
+ return pos_embed.to(device=x.device, dtype=x.dtype)
+
+
[email protected]("dtype", [torch.float, torch.float16])
[email protected]("device", ["cpu", "cuda"])
+def test_pos_emb(dtype, device):
+ # just a placeholder to get the device and dtype
+ x = torch.empty(1, dtype=dtype, device=device)
+ pos_embedder = PositionEmbedding2D(
+ D,
+ max_position_embeddings=8,
+ scale=SCALE,
+ ).to(device=device, dtype=dtype)
+ output = pos_embedder(x, 8, 7)
+ target = get_spatial_pos_embed(x, D, 8, 7, SCALE)
+ assert_close(output, target)
+ output = pos_embedder(x, 15, 16)
+ target = get_spatial_pos_embed(x, D, 15, 16, SCALE)
+ assert_close(output, target)
+ output = pos_embedder(x, 30, 20, base_size=2)
+ target = get_spatial_pos_embed(x, D, 30, 20, SCALE, base_size=2)
+ assert_close(output, target)
+ # test cache
+ output = pos_embedder(x, 30, 20, base_size=2)
+ target = get_spatial_pos_embed(x, D, 30, 20, SCALE, base_size=2)
+ assert_close(output, target)
+ assert pos_embedder._get_cached_emb.cache_info().hits >= 1