Open-Sora: Democratizing Efficient Video Production for All
Python
Merge branch 'main' into dev/v1.2
zhengzangwf551321e49cd
diff --cc opensora/models/layers/blocks.pyindex b104439,58dfd38..705a4aa--- a/opensora/models/layers/blocks.py+++ b/opensora/models/layers/blocks.py@@@ -215,6 -214,154 +215,152 @@@ class Attention(nn.Module)x = x.reshape(x_output_shape)x = self.proj(x)x = self.proj_drop(x)+ return x+++ class KVCompressAttention(nn.Module):+ def __init__(+ self,+ dim: int,+ num_heads: int = 8,+ qkv_bias: bool = False,+ qk_norm: bool = False,+ attn_drop: float = 0.0,+ proj_drop: float = 0.0,+ norm_layer: nn.Module = LlamaRMSNorm,+ enable_flashattn: bool = False,+ rope=None,+ sampling="conv",+ sr_ratio=1,+ mem_eff_attention=False,- attn_half=False++ attn_half=False,+ ) -> None:+ super().__init__()+ assert dim % num_heads == 0, "dim should be divisible by num_heads"+ self.dim = dim+ self.num_heads = num_heads+ self.head_dim = dim // num_heads+ self.scale = self.head_dim**-0.5+ self.enable_flashattn = enable_flashattn++ self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)++ self.sr_ratio = sr_ratio+ self.sampling = sampling- if sr_ratio > 1 and sampling == 'conv':++ if sr_ratio > 1 and sampling == "conv":+ # Avg Conv Init.+ self.sr = nn.Conv2d(dim, dim, groups=dim, kernel_size=sr_ratio, stride=sr_ratio)- self.sr.weight.data.fill_(1 / sr_ratio ** 2)++ self.sr.weight.data.fill_(1 / sr_ratio**2)+ self.sr.bias.data.zero_()+ self.norm = nn.LayerNorm(dim)++ self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()+ self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()+ self.attn_drop = nn.Dropout(attn_drop)+ self.proj = nn.Linear(dim, dim)+ self.proj_drop = nn.Dropout(proj_drop)++ self.rope = False+ if rope is not None:+ self.rope = True+ self.rotary_emb = rope++ self.mem_eff_attention = mem_eff_attention+ self.attn_half = attn_half++ def downsample_2d(self, tensor, H, W, scale_factor, sampling=None):+ if sampling is None or scale_factor == 1:+ return tensor+ B, N, C = tensor.shape+- if sampling == 'uniform_every':++ if sampling == "uniform_every":+ return tensor[:, ::scale_factor], int(N // scale_factor)++ tensor = tensor.reshape(B, H, W, C).permute(0, 3, 1, 2)+ new_H, new_W = int(H / scale_factor), int(W / scale_factor)+ new_N = new_H * new_W+- if sampling == 'ave':- tensor = F.interpolate(- tensor, scale_factor=1 / scale_factor, mode='nearest'- ).permute(0, 2, 3, 1)- elif sampling == 'uniform':++ if sampling == "ave":++ tensor = F.interpolate(tensor, scale_factor=1 / scale_factor, mode="nearest").permute(0, 2, 3, 1)++ elif sampling == "uniform":+ tensor = tensor[:, :, ::scale_factor, ::scale_factor].permute(0, 2, 3, 1)- elif sampling == 'conv':++ elif sampling == "conv":+ tensor = self.sr(tensor).reshape(B, C, -1).permute(0, 2, 1)+ tensor = self.norm(tensor)+ else:+ raise ValueError++ return tensor.reshape(B, new_N, C).contiguous(), new_N++ def forward(self, x: torch.Tensor, mask=None, HW=None, block_id=None, **kwargs) -> torch.Tensor:+ B, N, C = x.shape++ new_N = N+ if HW is None:- H = W = int(N ** 0.5)++ H = W = int(N**0.5)+ else:+ H, W = HW++ # flash attn is not memory efficient for small sequences, this is empirical+ enable_flashattn = self.enable_flashattn and (N > B)+ qkv = self.qkv(x)+ qkv_shape = (B, N, 3, self.num_heads, self.head_dim)++ qkv = qkv.view(qkv_shape).permute(2, 0, 3, 1, 4)+ q, k, v = qkv.unbind(0)+ # WARNING: this may be a bug+ if self.rope:+ q = self.rotary_emb(q)+ k = self.rotary_emb(k)+ q, k = self.q_norm(q), self.k_norm(k)++ if self.sr_ratio > 1:+ k, new_N = self.downsample_2d(k, H, W, self.sr_ratio, sampling=self.sampling)+ v, new_N = self.downsample_2d(v, H, W, self.sr_ratio, sampling=self.sampling)++ q = q.reshape(B, N, self.num_heads, C // self.num_heads)+ k = k.reshape(B, new_N, self.num_heads, C // self.num_heads)+ v = v.reshape(B, new_N, self.num_heads, C // self.num_heads)++ if enable_flashattn:+ from flash_attn import flash_attn_func++ # (B, #heads, N, #dim) -> (B, N, #heads, #dim)+ q = q.permute(0, 2, 1, 3)+ k = k.permute(0, 2, 1, 3)+ v = v.permute(0, 2, 1, 3)+ x = flash_attn_func(+ q,+ k,+ v,+ dropout_p=self.attn_drop.p if self.training else 0.0,+ softmax_scale=self.scale,+ )++ elif self.mem_eff_attention:+ attn_bias = None+ if mask is not None:+ attn_bias = torch.zeros([B * self.num_heads, q.shape[1], k.shape[1]], dtype=q.dtype, device=q.device)- attn_bias.masked_fill_(mask.squeeze(1).repeat(self.num_heads, 1, 1) == 0, float('-inf'))++ attn_bias.masked_fill_(mask.squeeze(1).repeat(self.num_heads, 1, 1) == 0, float("-inf"))+ x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)++ else:+ dtype = q.dtype+ q = q * self.scale+ attn = q @ k.transpose(-2, -1) # translate attn to float32+ if not self.attn_half:+ attn = attn.to(torch.float32)+ attn = attn.softmax(dim=-1)+ attn = attn.to(dtype) # cast back attn to original dtype+ attn = self.attn_drop(attn)+ x = attn @ v++ x_output_shape = (B, N, C)+ if not enable_flashattn:+ x = x.transpose(1, 2)+ x = x.reshape(x_output_shape)+ x = self.proj(x)+ x = self.proj_drop(x)return x