Open-Sora: Democratizing Efficient Video Production for All
Python
Added causal mask in Attention forward pass
HangXu8f239c87bf12
diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.pyindex 8bc7e72..73a0162 100644--- a/opensora/models/layers/blocks.py+++ b/opensora/models/layers/blocks.py@@ -163,6 +163,8 @@ class Attention(nn.Module):if rope is not None:self.rope = Trueself.rotary_emb = rope++ self.is_causal = Falsedef forward(self, x: torch.Tensor) -> torch.Tensor:B, N, C = x.shape@@ -198,12 +200,17 @@ class Attention(nn.Module):v,dropout_p=self.attn_drop.p if self.training else 0.0,softmax_scale=self.scale,+ causal=self.is_causal,)else:dtype = q.dtypeq = q * self.scaleattn = q @ k.transpose(-2, -1) # translate attn to float32attn = attn.to(torch.float32)+ if self.is_causal:+ causal_mask = torch.tril(torch.ones_like(attn), diagonal=0)+ causal_mask = torch.where(causal_mask.bool(), 0, float('-inf'))+ attn += causal_maskattn = attn.softmax(dim=-1)attn = attn.to(dtype) # cast back attn to original dtypeattn = self.attn_drop(attn)