Open-Sora: Democratizing Efficient Video Production for All
Python
[hotfix] fix attn impl (#17)
Hongxin Liubdb9b7ccbfa9
diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.pyindex f7c67b1..0a9ebb1 100644--- a/opensora/models/layers/blocks.py+++ b/opensora/models/layers/blocks.py@@ -23,13 +23,18 @@ import xformers.opsfrom einops import rearrangefrom timm.models.vision_transformer import Mlp-from opensora.acceleration.communications import all_to_all, split_forward_gather_backward+from opensora.acceleration.communications import (+ all_to_all,+ split_forward_gather_backward,+)from opensora.acceleration.parallel_states import get_sequence_parallel_groupapprox_gelu = lambda: nn.GELU(approximate="tanh")-def get_layernorm(hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool):+def get_layernorm(+ hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool+):if use_kernel:try:from apex.normalization import FusedLayerNorm@@ -84,7 +89,9 @@ class PatchEmbed3D(nn.Module):self.in_chans = in_chansself.embed_dim = embed_dim- self.proj = nn.Conv3d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)+ self.proj = nn.Conv3d(+ in_chans, embed_dim, kernel_size=patch_size, stride=patch_size+ )if norm_layer is not None:self.norm = norm_layer(embed_dim)else:@@ -147,6 +154,8 @@ class Attention(nn.Module):def forward(self, x: torch.Tensor) -> torch.Tensor:B, N, C = x.shape+ # flash attn is not memory efficient for small sequences, this is empirical+ enable_flashattn = self.enable_flashattn and (N > B)qkv = self.qkv(x)qkv_shape = (B, N, 3, self.num_heads, self.head_dim)@@ -155,14 +164,14 @@ class Attention(nn.Module):if self.rope:q = self.rotary_emb(q)k = self.rotary_emb(k)- if self.enable_flashattn:+ if enable_flashattn:# (B, #heads, N, #dim) -> (B, N, #heads, #dim)q = q.permute(0, 2, 1, 3)k = k.permute(0, 2, 1, 3)v = v.permute(0, 2, 1, 3)q, k = self.q_norm(q), self.k_norm(k)- if self.enable_flashattn:+ if enable_flashattn:from flash_attn import flash_attn_funcx = flash_attn_func(@@ -183,7 +192,7 @@ class Attention(nn.Module):x = attn @ vx_output_shape = (B, N, C)- if not self.enable_flashattn:+ if not enable_flashattn:x = x.transpose(1, 2)x = x.reshape(x_output_shape)x = self.proj(x)@@ -217,7 +226,9 @@ class SeqParallelAttention(Attention):)def forward(self, x: torch.Tensor) -> torch.Tensor:- B, N, C = x.shape # for sequence parallel here, the N is a local sequence length+ B, N, C = (+ x.shape+ ) # for sequence parallel here, the N is a local sequence lengthqkv = self.qkv(x)qkv_shape = (B, N, 3, self.num_heads, self.head_dim)@@ -310,7 +321,9 @@ class MultiHeadCrossAttention(nn.Module):attn_bias = Noneif mask is not None:attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)- x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)+ x = xformers.ops.memory_efficient_attention(+ q, k, v, p=self.attn_drop.p, attn_bias=attn_bias+ )x = x.view(B, -1, C)x = self.proj(x)@@ -349,8 +362,12 @@ class SeqParallelMultiHeadCrossAttention(MultiHeadCrossAttention):# apply all_to_all to gather sequence and split attention headsq = all_to_all(q, sp_group, scatter_dim=2, gather_dim=1)- k = split_forward_gather_backward(k, get_sequence_parallel_group(), dim=2, grad_scale="down")- v = split_forward_gather_backward(v, get_sequence_parallel_group(), dim=2, grad_scale="down")+ k = split_forward_gather_backward(+ k, get_sequence_parallel_group(), dim=2, grad_scale="down"+ )+ v = split_forward_gather_backward(+ v, get_sequence_parallel_group(), dim=2, grad_scale="down"+ )q = q.view(1, -1, self.num_heads // sp_size, self.head_dim)k = k.view(1, -1, self.num_heads // sp_size, self.head_dim)@@ -360,7 +377,9 @@ class SeqParallelMultiHeadCrossAttention(MultiHeadCrossAttention):attn_bias = Noneif mask is not None:attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)- x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)+ x = xformers.ops.memory_efficient_attention(+ q, k, v, p=self.attn_drop.p, attn_bias=attn_bias+ )# apply all to all to gather back attention heads and scatter sequencex = x.view(B, -1, self.num_heads // sp_size, self.head_dim)@@ -382,7 +401,9 @@ class FinalLayer(nn.Module):super().__init__()self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)- self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True))+ self.adaLN_modulation = nn.Sequential(+ nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)+ )def forward(self, x, c):shift, scale = self.adaLN_modulation(c).chunk(2, dim=1)@@ -400,7 +421,9 @@ class T2IFinalLayer(nn.Module):super().__init__()self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)- self.scale_shift_table = nn.Parameter(torch.randn(2, hidden_size) / hidden_size**0.5)+ self.scale_shift_table = nn.Parameter(+ torch.randn(2, hidden_size) / hidden_size**0.5+ )self.out_channels = out_channelsself.d_t = d_tself.d_s = d_s@@ -423,7 +446,9 @@ class T2IFinalLayer(nn.Module):shift, scale = (self.scale_shift_table[None] + t[:, None]).chunk(2, dim=1)x = t2i_modulate(self.norm_final(x), shift, scale)if x_mask is not None:- shift_zero, scale_zero = (self.scale_shift_table[None] + t0[:, None]).chunk(2, dim=1)+ shift_zero, scale_zero = (self.scale_shift_table[None] + t0[:, None]).chunk(+ 2, dim=1+ )x_zero = t2i_modulate(self.norm_final(x), shift_zero, scale_zero)x = self.t_mask_select(x_mask, x, x_zero, T, S)x = self.linear(x)@@ -461,12 +486,18 @@ class TimestepEmbedder(nn.Module):"""# https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.pyhalf = dim // 2- freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half)+ freqs = torch.exp(+ -math.log(max_period)+ * torch.arange(start=0, end=half, dtype=torch.float32)+ / half+ )freqs = freqs.to(device=t.device)args = t[:, None].float() * freqs[None]embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)if dim % 2:- embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)+ embedding = torch.cat(+ [embedding, torch.zeros_like(embedding[:, :1])], dim=-1+ )return embeddingdef forward(self, t, dtype):@@ -485,7 +516,9 @@ class LabelEmbedder(nn.Module):def __init__(self, num_classes, hidden_size, dropout_prob):super().__init__()use_cfg_embedding = dropout_prob > 0- self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)+ self.embedding_table = nn.Embedding(+ num_classes + use_cfg_embedding, hidden_size+ )self.num_classes = num_classesself.dropout_prob = dropout_prob@@ -513,7 +546,9 @@ class SizeEmbedder(TimestepEmbedder):"""def __init__(self, hidden_size, frequency_embedding_size=256):- super().__init__(hidden_size=hidden_size, frequency_embedding_size=frequency_embedding_size)+ super().__init__(+ hidden_size=hidden_size, frequency_embedding_size=frequency_embedding_size+ )self.mlp = nn.Sequential(nn.Linear(frequency_embedding_size, hidden_size, bias=True),nn.SiLU(),@@ -531,7 +566,9 @@ class SizeEmbedder(TimestepEmbedder):assert s.shape[0] == bsb, dims = s.shape[0], s.shape[1]s = rearrange(s, "b d -> (b d)")- s_freq = self.timestep_embedding(s, self.frequency_embedding_size).to(self.dtype)+ s_freq = self.timestep_embedding(s, self.frequency_embedding_size).to(+ self.dtype+ )s_emb = self.mlp(s_freq)s_emb = rearrange(s_emb, "(b d) d2 -> b (d d2)", b=b, d=dims, d2=self.outdim)return s_emb@@ -631,7 +668,12 @@ class PositionEmbedding2D(nn.Module):return torch.concat([emb_h, emb_w], dim=-1).unsqueeze(0).to(dtype)def forward(- self, x: torch.Tensor, h: int, w: int, scale: Optional[float] = 1.0, base_size: Optional[int] = None+ self,+ x: torch.Tensor,+ h: int,+ w: int,+ scale: Optional[float] = 1.0,+ base_size: Optional[int] = None,) -> torch.Tensor:return self._get_cached_emb(x.device, x.dtype, h, w, scale, base_size)@@ -642,7 +684,9 @@ class PositionEmbedding2D(nn.Module):# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py-def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0, scale=1.0, base_size=None):+def get_2d_sincos_pos_embed(+ embed_dim, grid_size, cls_token=False, extra_tokens=0, scale=1.0, base_size=None+):"""grid_size: int of the grid height and widthreturn:@@ -662,7 +706,9 @@ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=grid = grid.reshape([2, 1, grid_size[1], grid_size[0]])pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)if cls_token and extra_tokens > 0:- pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)+ pos_embed = np.concatenate(+ [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0+ )return pos_embeddiff --git a/tests/test_attn.py b/tests/test_attn.pynew file mode 100644index 0000000..22444a1--- /dev/null+++ b/tests/test_attn.py@@ -0,0 +1,36 @@+import torch+from colossalai.accelerator import get_accelerator+from colossalai.utils import get_current_device+from rotary_embedding_torch import RotaryEmbedding++from opensora.models.layers.blocks import Attention++# B, S, H = 7488, 1, 1152+# B, S, H = 32, 234, 1152+B, S, H = 128, 32, 1152+N, D = 16, 72+++def run_attn(enable_flashattn: bool):+ get_accelerator().reset_peak_memory_stats()+ rope = RotaryEmbedding(D).to(device=get_current_device(), dtype=torch.bfloat16)+ attn = Attention(+ H,+ N,+ qkv_bias=True,+ rope=rope.rotate_queries_or_keys,+ enable_flashattn=enable_flashattn,+ ).to(device=get_current_device(), dtype=torch.bfloat16)+ x = torch.randn(+ B, S, H, device=get_current_device(), dtype=torch.bfloat16+ ).requires_grad_()+ y = attn(x)+ y.mean().backward()+ print(f"Peak memory: {get_accelerator().max_memory_allocated() / 1024**2:.2f} MB")+++if __name__ == "__main__":+ print("Use flashattn")+ run_attn(True)+ print("No flashattn")+ run_attn(False)