Open-Sora: Democratizing Efficient Video Production for All
Python
[feature] impl fastseq-style seq parallel (#21)
Hongxin Liu91275b2b5e2d
diff --git a/benchmark.py b/benchmark.pyindex dbe9855..5343fad 100644--- a/benchmark.py+++ b/benchmark.py@@ -11,6 +11,7 @@ import argparseimport timeimport torch+import torch.distributed as distfrom colossalai import launch_from_torchfrom colossalai.accelerator import get_acceleratorfrom colossalai.booster import Booster@@ -23,6 +24,7 @@ from tqdm import tqdmfrom open_sora.diffusion import create_diffusionfrom open_sora.modeling import DiT_models+from open_sora.modeling.dit import SUPPORTED_SEQ_PARALLEL_MODESfrom open_sora.utils.data import create_video_compressor, preprocess_batchfrom open_sora.utils.plugin import ZeroSeqParallelPlugin@@ -55,6 +57,8 @@ def main(args):model_kwargs = {"in_channels": video_compressor.out_channels,"seq_parallel_group": plugin.sp_group,+ "seq_parallel_mode": args.sp_mode,+ "seq_parallel_overlap": args.sp_overlap,}# Create DiT and EMA@@ -93,7 +97,7 @@ def main(args):video_inputs = batch.pop("video_latent_states")mask = batch.pop("video_padding_mask")logger.info(- f"Num patches: {video_inputs.shape[1]}, num_tokens: {batch['text_latent_states'].shape[1]}",+ f"Num patches: {video_inputs.shape[1]}, num tokens: {batch['text_latent_states'].shape[1]}",ranks=[0],)@@ -129,6 +133,11 @@ def main(args):if i >= args.warmup_steps:total_samples += args.batch_size * coordinator.world_sizetotal_duration += time_per_iter+ total_duration = torch.tensor([total_duration], device=get_current_device())+ dist.all_reduce(total_duration)+ total_duration = total_duration / coordinator.world_size+ total_duration = total_duration.item()+ total_samples *= coordinator.world_size // args.sp_sizethroughput = total_samples / total_durationlogger.info(@@ -136,7 +145,7 @@ def main(args):ranks=[0],)logger.info(- f"Throughput per device: {throughput:.2f} samples/s",+ f"Throughput: {throughput:.2f} samples/s",ranks=[0],)@@ -151,6 +160,10 @@ if __name__ == "__main__":"-p", "--plugin", type=str, default="zero2", choices=["ddp", "zero2"])parser.add_argument("--sp_size", type=int, default=1)+ parser.add_argument(+ "--sp_mode", type=str, default="ulysses", choices=SUPPORTED_SEQ_PARALLEL_MODES+ )+ parser.add_argument("--sp_overlap", action="store_true", default=False)parser.add_argument("-w", "--warmup_steps", type=int, default=2)parser.add_argument("-s", "--steps", type=int, default=3)parser.add_argument("-b", "--batch_size", type=int, default=4)diff --git a/open_sora/modeling/attn.py b/open_sora/modeling/attn.pynew file mode 100644index 0000000..f81d36b--- /dev/null+++ b/open_sora/modeling/attn.py@@ -0,0 +1,330 @@+from typing import Optional++import torch+import torch.distributed as dist+import torch.nn as nn+import torch.nn.functional as F+from colossalai.logging import get_dist_logger+from colossalai.shardformer.layer._operation import gather_forward_split_backward++from open_sora.utils.comm import all_to_all, async_all_gather_proj_for_two+++class CrossAttention(nn.Module):+ r"""+ A cross attention layer.++ Parameters:+ query_dim (`int`): The number of channels in the query.+ cross_attention_dim (`int`, *optional*):+ The number of channels in the context. If not given, defaults to `query_dim`.+ num_heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.+ head_dim (`int`, *optional*, defaults to 64): The number of channels in each head.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ bias (`bool`, *optional*, defaults to False):+ Set to `True` for the query, key, and value linear layers to contain a bias parameter.+ """++ def __init__(+ self,+ query_dim: int,+ cross_attention_dim: Optional[int] = None,+ num_heads: int = 8,+ head_dim: int = 64,+ dropout: float = 0.0,+ bias=False,+ sdpa=True,+ ):+ super().__init__()+ self.hidden_size = head_dim * num_heads+ cross_attention_dim = (+ cross_attention_dim if cross_attention_dim is not None else query_dim+ )++ self.scale = head_dim**-0.5+ self.num_heads = num_heads+ self.head_dim = head_dim+ self.sdpa = sdpa++ self.to_q = nn.Linear(query_dim, self.hidden_size, bias=bias)+ self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)+ self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)++ self.to_out = nn.Sequential(+ nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)+ )++ def forward(self, hidden_states, context=None, mask=None):+ bsz, q_len, _ = hidden_states.shape++ query = self.to_q(hidden_states)+ context = context if context is not None else hidden_states+ kv_seq_len = context.shape[1]+ key = self.to_k(context)+ value = self.to_v(context)++ # [B, S, H, D]+ query = query.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)+ key = key.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)+ value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(+ 1, 2+ )++ if mask is not None:+ assert mask.shape == (bsz, 1, q_len, kv_seq_len)+ if self.sdpa:+ attn_output = F.scaled_dot_product_attention(+ query, key, value, attn_mask=mask, scale=self.scale+ )+ else:+ attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale+ assert attn_weights.shape == (bsz, self.num_heads, q_len, kv_seq_len)+ if mask is not None:+ attn_weights = attn_weights + mask+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(+ query.dtype+ )+ attn_output = torch.matmul(attn_weights, value)+ assert attn_output.shape == (bsz, self.num_heads, q_len, self.head_dim)+ attn_output = attn_output.transpose(1, 2).contiguous()+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)+ attn_output = self.to_out(attn_output)+ return attn_output+++class SeqParallelCrossAttention(CrossAttention):+ r"""+ A cross attention layer.++ Parameters:+ query_dim (`int`): The number of channels in the query.+ cross_attention_dim (`int`, *optional*):+ The number of channels in the context. If not given, defaults to `query_dim`.+ num_heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.+ head_dim (`int`, *optional*, defaults to 64): The number of channels in each head.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ bias (`bool`, *optional*, defaults to False):+ Set to `True` for the query, key, and value linear layers to contain a bias parameter.+ """++ def __init__(+ self,+ query_dim: int,+ cross_attention_dim: Optional[int] = None,+ num_heads: int = 8,+ head_dim: int = 64,+ dropout: float = 0.0,+ bias=False,+ sdpa=True,+ seq_parallel_group=None,+ ):+ super().__init__(+ query_dim,+ cross_attention_dim,+ num_heads,+ head_dim,+ dropout,+ bias,+ sdpa,+ )+ self.seq_parallel_group = seq_parallel_group+ self.seq_parallel_size = (+ dist.get_world_size(self.seq_parallel_group)+ if seq_parallel_group is not None+ else 1+ )+ assert self.num_heads % self.seq_parallel_size == 0++ def forward(self, hidden_states, context=None, mask=None):+ bsz, q_len, _ = hidden_states.shape++ query = self.to_q(hidden_states)+ context = context if context is not None else hidden_states+ kv_seq_len = context.shape[1]+ key = self.to_k(context)+ value = self.to_v(context)++ # [B, S/P, H] -> [B, S, H/P]+ num_heads_parallel = self.num_heads // self.seq_parallel_size+ hidden_size_parallel = self.hidden_size // self.seq_parallel_size+ if self.seq_parallel_size > 1:+ query = all_to_all(+ query, self.seq_parallel_group, scatter_dim=2, gather_dim=1+ )+ key = all_to_all(key, self.seq_parallel_group, scatter_dim=2, gather_dim=1)+ value = all_to_all(+ value, self.seq_parallel_group, scatter_dim=2, gather_dim=1+ )++ q_len *= self.seq_parallel_size+ kv_seq_len *= self.seq_parallel_size++ # [B, S, H/P] -> [B, S, N/P, D] -> [B, N/P, S, D]+ query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(+ 1, 2+ )+ key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(+ 1, 2+ )+ value = value.view(+ bsz, kv_seq_len, num_heads_parallel, self.head_dim+ ).transpose(1, 2)++ if mask is not None:+ assert mask.shape == (bsz, 1, q_len, kv_seq_len)+ if self.sdpa:+ attn_output = F.scaled_dot_product_attention(+ query, key, value, attn_mask=mask, scale=self.scale+ )+ else:+ attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale+ assert attn_weights.shape == (bsz, num_heads_parallel, q_len, kv_seq_len)+ if mask is not None:+ attn_weights = attn_weights + mask+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(+ query.dtype+ )+ attn_output = torch.matmul(attn_weights, value)+ assert attn_output.shape == (bsz, num_heads_parallel, q_len, self.head_dim)+ attn_output = attn_output.transpose(1, 2).contiguous()+ attn_output = attn_output.reshape(bsz, q_len, hidden_size_parallel)+ # [B, S, H/P] -> [B, S/P, H]+ if self.seq_parallel_size > 1:+ attn_output = all_to_all(+ attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2+ )+ attn_output = self.to_out(attn_output)+ return attn_output+++class FastSeqParallelCrossAttention(SeqParallelCrossAttention):+ def __init__(+ self,+ query_dim: int,+ cross_attention_dim: Optional[int] = None,+ num_heads: int = 8,+ head_dim: int = 64,+ dropout: float = 0,+ bias=False,+ sdpa=True,+ seq_parallel_group=None,+ overlap=False,+ ):+ super().__init__(+ query_dim,+ cross_attention_dim,+ num_heads,+ head_dim,+ dropout,+ bias,+ sdpa,+ seq_parallel_group,+ )+ self.seq_parallel_rank = (+ dist.get_rank(self.seq_parallel_group)+ if seq_parallel_group is not None+ else 0+ )+ self.sequence_parallel_param_slice = slice(+ self.hidden_size // self.seq_parallel_size * self.seq_parallel_rank,+ self.hidden_size // self.seq_parallel_size * (self.seq_parallel_rank + 1),+ )+ if overlap and self.seq_parallel_size != 2:+ logger = get_dist_logger()+ logger.warning(+ "FastSeqParallelCrossAttention only supports overlap with seq_parallel_size=2. Fallback to non-overlap",+ ranks=[0],+ )+ overlap = False+ self.overlap = overlap++ def _get_sliced_params(self, proj_layer: nn.Linear):+ bias = bias = (+ proj_layer.bias[self.sequence_parallel_param_slice]+ if proj_layer.bias is not None+ else None+ )+ return proj_layer.weight[self.sequence_parallel_param_slice], bias++ def _proj(self, x: torch.Tensor, proj_layer: nn.Linear):+ return F.linear(+ x,+ *self._get_sliced_params(proj_layer),+ )++ def forward(self, hidden_states, context=None, mask=None):+ bsz, q_len, _ = hidden_states.shape++ context = context if context is not None else hidden_states+ kv_seq_len = context.shape[1]+ if self.seq_parallel_size > 1:+ if self.overlap and self.seq_parallel_size == 2:+ query, key, value = async_all_gather_proj_for_two(+ hidden_states,+ context,+ *self._get_sliced_params(self.to_q),+ *self._get_sliced_params(self.to_k),+ *self._get_sliced_params(self.to_v),+ dim=1,+ process_group=self.seq_parallel_group,+ sp_size=self.seq_parallel_size,+ sp_rank=self.seq_parallel_rank,+ )+ else:+ # [B, S/P, H] -> [B, S, H]+ hidden_states = gather_forward_split_backward(+ hidden_states, 1, self.seq_parallel_group+ )+ context = gather_forward_split_backward(+ context, 1, self.seq_parallel_group+ )+ query = self._proj(hidden_states, self.to_q)+ key = self._proj(context, self.to_k)+ value = self._proj(context, self.to_v)+ else:+ query = self.to_q(hidden_states)+ key = self.to_k(context)+ value = self.to_v(context)+ # output is [B, S, H/P]++ num_heads_parallel = self.num_heads // self.seq_parallel_size+ hidden_size_parallel = self.hidden_size // self.seq_parallel_size+ q_len *= self.seq_parallel_size+ kv_seq_len *= self.seq_parallel_size++ # [B, S, H/P] -> [B, S, N/P, D] -> [B, N/P, S, D]+ query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(+ 1, 2+ )+ key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(+ 1, 2+ )+ value = value.view(+ bsz, kv_seq_len, num_heads_parallel, self.head_dim+ ).transpose(1, 2)++ if mask is not None:+ assert mask.shape == (bsz, 1, q_len, kv_seq_len)+ if self.sdpa:+ attn_output = F.scaled_dot_product_attention(+ query, key, value, attn_mask=mask, scale=self.scale+ )+ else:+ attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale+ assert attn_weights.shape == (bsz, num_heads_parallel, q_len, kv_seq_len)+ if mask is not None:+ attn_weights = attn_weights + mask+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(+ query.dtype+ )+ attn_output = torch.matmul(attn_weights, value)+ assert attn_output.shape == (bsz, num_heads_parallel, q_len, self.head_dim)+ attn_output = attn_output.transpose(1, 2).contiguous()+ attn_output = attn_output.reshape(bsz, q_len, hidden_size_parallel)+ # [B, S, H/P] -> [B, S/P, H]+ if self.seq_parallel_size > 1:+ attn_output = all_to_all(+ attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2+ )+ attn_output = self.to_out(attn_output)+ return attn_outputdiff --git a/open_sora/modeling/dit.py b/open_sora/modeling/dit.pyindex ac9ce8a..776f736 100644--- a/open_sora/modeling/dit.py+++ b/open_sora/modeling/dit.py@@ -19,203 +19,15 @@ import torch.nn as nnimport torch.nn.functional as Ffrom timm.models.vision_transformer import Mlp-from open_sora.utils.comm import all_to_all, gather_seq, split_seq---class CrossAttention(nn.Module):- r"""- A cross attention layer.-- Parameters:- query_dim (`int`): The number of channels in the query.- cross_attention_dim (`int`, *optional*):- The number of channels in the context. If not given, defaults to `query_dim`.- num_heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.- head_dim (`int`, *optional*, defaults to 64): The number of channels in each head.- dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.- bias (`bool`, *optional*, defaults to False):- Set to `True` for the query, key, and value linear layers to contain a bias parameter.- """-- def __init__(- self,- query_dim: int,- cross_attention_dim: Optional[int] = None,- num_heads: int = 8,- head_dim: int = 64,- dropout: float = 0.0,- bias=False,- sdpa=True,- ):- super().__init__()- self.hidden_size = head_dim * num_heads- cross_attention_dim = (- cross_attention_dim if cross_attention_dim is not None else query_dim- )-- self.scale = head_dim**-0.5- self.num_heads = num_heads- self.head_dim = head_dim- self.sdpa = sdpa-- self.to_q = nn.Linear(query_dim, self.hidden_size, bias=bias)- self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)- self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)-- self.to_out = nn.Sequential(- nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)- )-- def forward(self, hidden_states, context=None, mask=None):- bsz, q_len, _ = hidden_states.shape-- query = self.to_q(hidden_states)- context = context if context is not None else hidden_states- kv_seq_len = context.shape[1]- key = self.to_k(context)- value = self.to_v(context)+from open_sora.utils.comm import gather_seq, split_seq- # [B, S, H, D]- query = query.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)- key = key.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)- value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(- 1, 2- )+from .attn import (+ CrossAttention,+ FastSeqParallelCrossAttention,+ SeqParallelCrossAttention,+)- if mask is not None:- assert mask.shape == (bsz, 1, q_len, kv_seq_len)- if self.sdpa:- attn_output = F.scaled_dot_product_attention(- query, key, value, attn_mask=mask, scale=self.scale- )- else:- attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale- assert attn_weights.shape == (bsz, self.num_heads, q_len, kv_seq_len)- if mask is not None:- attn_weights = attn_weights + mask- attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(- query.dtype- )- attn_output = torch.matmul(attn_weights, value)- assert attn_output.shape == (bsz, self.num_heads, q_len, self.head_dim)- attn_output = attn_output.transpose(1, 2).contiguous()- attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)- attn_output = self.to_out(attn_output)- return attn_output---class SeqParallelCrossAttention(nn.Module):- r"""- A cross attention layer.-- Parameters:- query_dim (`int`): The number of channels in the query.- cross_attention_dim (`int`, *optional*):- The number of channels in the context. If not given, defaults to `query_dim`.- num_heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.- head_dim (`int`, *optional*, defaults to 64): The number of channels in each head.- dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.- bias (`bool`, *optional*, defaults to False):- Set to `True` for the query, key, and value linear layers to contain a bias parameter.- """-- def __init__(- self,- query_dim: int,- cross_attention_dim: Optional[int] = None,- num_heads: int = 8,- head_dim: int = 64,- dropout: float = 0.0,- bias=False,- sdpa=True,- seq_parallel_group=None,- ):- super().__init__()- self.hidden_size = head_dim * num_heads- cross_attention_dim = (- cross_attention_dim if cross_attention_dim is not None else query_dim- )-- self.scale = head_dim**-0.5- self.num_heads = num_heads- self.head_dim = head_dim- self.sdpa = sdpa- self.seq_parallel_group = seq_parallel_group- self.seq_parallel_size = (- dist.get_world_size(self.seq_parallel_group)- if seq_parallel_group is not None- else 1- )- assert self.num_heads % self.seq_parallel_size == 0-- self.to_q = nn.Linear(query_dim, self.hidden_size, bias=bias)- self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)- self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)-- self.to_out = nn.Sequential(- nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)- )-- def forward(self, hidden_states, context=None, mask=None):- bsz, q_len, _ = hidden_states.shape-- query = self.to_q(hidden_states)- context = context if context is not None else hidden_states- kv_seq_len = context.shape[1]- key = self.to_k(context)- value = self.to_v(context)-- # [B, S/P, H] -> [B, S, H/P]- num_heads_parallel = self.num_heads // self.seq_parallel_size- hidden_size_parallel = self.hidden_size // self.seq_parallel_size- if self.seq_parallel_group is not None and self.seq_parallel_size > 1:- query = all_to_all(- query, self.seq_parallel_group, scatter_dim=2, gather_dim=1- )- key = all_to_all(key, self.seq_parallel_group, scatter_dim=2, gather_dim=1)- value = all_to_all(- value, self.seq_parallel_group, scatter_dim=2, gather_dim=1- )-- q_len *= self.seq_parallel_size- kv_seq_len *= self.seq_parallel_size-- # [B, S, H/P] -> [B, S, N/P, D] -> [B, N/P, S, D]- query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(- 1, 2- )- key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(- 1, 2- )- value = value.view(- bsz, kv_seq_len, num_heads_parallel, self.head_dim- ).transpose(1, 2)-- if mask is not None:- assert mask.shape == (bsz, 1, q_len, kv_seq_len)- if self.sdpa:- attn_output = F.scaled_dot_product_attention(- query, key, value, attn_mask=mask, scale=self.scale- )- else:- attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale- assert attn_weights.shape == (bsz, num_heads_parallel, q_len, kv_seq_len)- if mask is not None:- attn_weights = attn_weights + mask- attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(- query.dtype- )- attn_output = torch.matmul(attn_weights, value)- assert attn_output.shape == (bsz, num_heads_parallel, q_len, self.head_dim)- attn_output = attn_output.transpose(1, 2).contiguous()- attn_output = attn_output.reshape(bsz, q_len, hidden_size_parallel)- # [B, S, H/P] -> [B, S/P, H]- if self.seq_parallel_group is not None and self.seq_parallel_size > 1:- attn_output = all_to_all(- attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2- )- attn_output = self.to_out(attn_output)- return attn_output+SUPPORTED_SEQ_PARALLEL_MODES = ["ulysses", "fastseq"]def modulate(x, shift, scale):@@ -408,17 +220,33 @@ class DiTBlock(nn.Module):cross_attention_dim=None,mlp_ratio=4.0,seq_parallel_group=None,+ seq_parallel_mode="ulysses",+ seq_parallel_overlap=False,):super().__init__()self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)- self.attn = SeqParallelCrossAttention(+ attn_kwargs = {}+ if seq_parallel_group is not None:+ attn_kwargs["seq_parallel_group"] = seq_parallel_group+ if seq_parallel_mode == "ulysses":+ attn_cls = SeqParallelCrossAttention+ elif seq_parallel_mode == "fastseq":+ attn_cls = FastSeqParallelCrossAttention+ attn_kwargs["overlap"] = seq_parallel_overlap+ else:+ raise ValueError(+ f"seq_parallel_mode must be one of {SUPPORTED_SEQ_PARALLEL_MODES}"+ )+ else:+ attn_cls = CrossAttention+ self.attn = attn_cls(query_dim=hidden_size,cross_attention_dim=cross_attention_dim,num_heads=num_heads,head_dim=hidden_size // num_heads,bias=True,sdpa=True,- seq_parallel_group=seq_parallel_group,+ **attn_kwargs,)self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)@@ -500,6 +328,8 @@ class DiT(nn.Module):learn_sigma=True,use_cross_attn=True,seq_parallel_group=None,+ seq_parallel_mode="ulysses",+ seq_parallel_overlap=False,):super().__init__()self.grad_checkpointing = False@@ -546,6 +376,8 @@ class DiT(nn.Module):cross_attn_dim,mlp_ratio=mlp_ratio,seq_parallel_group=seq_parallel_group,+ seq_parallel_mode=seq_parallel_mode,+ seq_parallel_overlap=seq_parallel_overlap,)for _ in range(depth)]diff --git a/open_sora/utils/comm.py b/open_sora/utils/comm.pyindex 00703d3..9b947ff 100644--- a/open_sora/utils/comm.py+++ b/open_sora/utils/comm.py@@ -1,7 +1,11 @@+from typing import Optional, Tuple+import torchimport torch.distributed as dist+import torch.nn.functional as Ffrom colossalai.moe._operation import MoeInGradScaler, MoeOutGradScalerfrom colossalai.shardformer.layer._operation import gather_forward_split_backward+from torch.distributed.distributed_c10d import get_global_rankdef _all_to_all(@@ -94,3 +98,195 @@ def gather_seq("""input_ = gather_forward_split_backward(input_, dim, sp_group)return MoeInGradScaler.apply(input_, sp_size)+++class AsyncAllGatherProjForTwo(torch.autograd.Function):+ @staticmethod+ def forward(+ ctx,+ hidden_states: torch.Tensor,+ context: torch.Tensor,+ q_proj_weight: torch.Tensor,+ q_proj_bias: Optional[torch.Tensor],+ k_proj_weight: torch.Tensor,+ k_proj_bias: Optional[torch.Tensor],+ v_proj_weight: torch.Tensor,+ v_proj_bias: Optional[torch.Tensor],+ dim: int,+ process_group: dist.ProcessGroup,+ sp_size: int,+ sp_rank: int,+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ assert sp_size == 2+ ctx.process_group = process_group+ ctx.sp_size = sp_size+ ctx.sp_rank = sp_rank+ ctx.dim = dim++ is_cross_attn = not (hidden_states is context)+ ctx.is_cross_attn = is_cross_attn++ recv_hidden_states = torch.empty_like(hidden_states)+ if is_cross_attn:+ recv_context = torch.empty_like(context)+ else:+ recv_context = recv_hidden_states++ peer_global_rank = get_global_rank(process_group, 1 - sp_rank)+ ops = [+ dist.P2POp(dist.isend, hidden_states, peer_global_rank, process_group),+ dist.P2POp(dist.irecv, recv_hidden_states, peer_global_rank, process_group),+ ]+ if is_cross_attn:+ ops.extend(+ [+ dist.P2POp(dist.isend, context, peer_global_rank, process_group),+ dist.P2POp(+ dist.irecv, recv_context, peer_global_rank, process_group+ ),+ ]+ )++ reqs = dist.batch_isend_irecv(ops)++ # [B, S/P, H]+ q = F.linear(hidden_states, q_proj_weight, q_proj_bias)+ k = F.linear(context, k_proj_weight, k_proj_bias)+ v = F.linear(context, v_proj_weight, v_proj_bias)++ for req in reqs:+ req.wait()++ q_other = F.linear(recv_hidden_states, q_proj_weight, q_proj_bias)+ k_other = F.linear(recv_context, k_proj_weight, k_proj_bias)+ v_other = F.linear(recv_context, v_proj_weight, v_proj_bias)++ if sp_rank == 0:+ q = torch.cat([q, q_other], dim=dim)+ k = torch.cat([k, k_other], dim=dim)+ v = torch.cat([v, v_other], dim=dim)+ else:+ q = torch.cat([q_other, q], dim=dim)+ k = torch.cat([k_other, k], dim=dim)+ v = torch.cat([v_other, v], dim=dim)++ ctx.save_for_backward(+ hidden_states,+ context,+ q_proj_weight,+ q_proj_bias,+ k_proj_weight,+ k_proj_bias,+ v_proj_weight,+ v_proj_bias,+ recv_hidden_states,+ recv_context,+ )++ return q, k, v++ @staticmethod+ def backward(+ ctx: torch.Any, q_grad: torch.Tensor, k_grad: torch.Tensor, v_grad: torch.Tensor+ ) -> torch.Any:+ (+ hidden_states,+ context,+ q_proj_weight,+ q_proj_bias,+ k_proj_weight,+ k_proj_bias,+ v_proj_weight,+ v_proj_bias,+ recv_hidden_states,+ recv_context,+ ) = ctx.saved_tensors++ # compute param grads+ if ctx.sp_rank == 0:+ hidden_states = torch.cat([hidden_states, recv_hidden_states], dim=ctx.dim)+ if ctx.is_cross_attn:+ context = torch.cat([context, recv_context], dim=ctx.dim)+ else:+ context = hidden_states+ else:+ hidden_states = torch.cat([recv_hidden_states, hidden_states], dim=ctx.dim)+ if ctx.is_cross_attn:+ context = torch.cat([recv_context, context], dim=ctx.dim)+ else:+ context = hidden_states+ q_proj_weight_grad = q_grad.transpose(-1, -2).matmul(hidden_states).sum(dim=0)+ q_proj_bias_grad = (+ q_grad.sum(dim=0).sum(dim=0) if q_proj_bias is not None else None+ )+ k_proj_weight_grad = k_grad.transpose(-1, -2).matmul(context).sum(dim=0)+ k_proj_bias_grad = (+ k_grad.sum(dim=0).sum(dim=0) if k_proj_bias is not None else None+ )+ v_proj_weight_grad = v_grad.transpose(-1, -2).matmul(context).sum(dim=0)+ v_proj_bias_grad = (+ v_grad.sum(dim=0).sum(dim=0) if v_proj_bias is not None else None+ )++ # split grads+ q_grad = q_grad.chunk(ctx.sp_size, dim=ctx.dim)[ctx.sp_rank].clone()+ k_grad = k_grad.chunk(ctx.sp_size, dim=ctx.dim)[ctx.sp_rank].clone()+ v_grad = v_grad.chunk(ctx.sp_size, dim=ctx.dim)[ctx.sp_rank].clone()++ if ctx.is_cross_attn:+ hidden_states_grad = torch.matmul(q_grad, q_proj_weight)+ context_grad = torch.matmul(k_grad, k_proj_weight) + torch.matmul(+ v_grad, v_proj_weight+ )+ else:+ hidden_states_grad = (+ torch.matmul(q_grad, q_proj_weight)+ + torch.matmul(k_grad, k_proj_weight)+ + torch.matmul(v_grad, v_proj_weight)+ )+ context_grad = hidden_states_grad++ return (+ hidden_states_grad,+ context_grad,+ q_proj_weight_grad,+ q_proj_bias_grad,+ k_proj_weight_grad,+ k_proj_bias_grad,+ v_proj_weight_grad,+ v_proj_bias_grad,+ None,+ None,+ None,+ None,+ )+++def async_all_gather_proj_for_two(+ hidden_states: torch.Tensor,+ context: torch.Tensor,+ q_proj_weight: torch.Tensor,+ q_proj_bias: Optional[torch.Tensor],+ k_proj_weight: torch.Tensor,+ k_proj_bias: Optional[torch.Tensor],+ v_proj_weight: torch.Tensor,+ v_proj_bias: Optional[torch.Tensor],+ dim: int,+ process_group: dist.ProcessGroup,+ sp_size: int,+ sp_rank: int,+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ return AsyncAllGatherProjForTwo.apply(+ hidden_states,+ context,+ q_proj_weight,+ q_proj_bias,+ k_proj_weight,+ k_proj_bias,+ v_proj_weight,+ v_proj_bias,+ dim,+ process_group,+ sp_size,+ sp_rank,+ )diff --git a/tests/test_model.py b/tests/test_model.pyindex 62b3172..1c2462f 100644--- a/tests/test_model.py+++ b/tests/test_model.py@@ -15,16 +15,40 @@ from open_sora.utils.plugin import ZeroSeqParallelPlugin@parameterize("sp_size", [2, 4])+@parameterize("sp_mode", ["ulysses", "fastseq"])+@parameterize("sp_overlap", [True, False])+@parameterize("cross_attn", [True, False])def check_dit_model_fwd_bwd(- sp_size: int, video_latent_states, text_latent_states, t, mask+ sp_size: int,+ sp_mode: str,+ sp_overlap: bool,+ cross_attn: bool,+ video_latent_states,+ text_latent_states,+ t,+ mask,):+ if sp_overlap and sp_mode == "ulysses":+ return+ if not cross_attn:+ mask = mask.new_ones(+ *mask.shape[:2],+ mask.shape[2] + mask.shape[3],+ mask.shape[2] + mask.shape[3],+ )plugin = ZeroSeqParallelPlugin(sp_size=sp_size, stage=2, precision="fp32", master_weights=False)booster = Booster(plugin=plugin)- model = DiT_models["DiT-B/8"](text_dropout_prob=0.0).to(get_current_device())+ model = DiT_models["DiT-B/8"](text_dropout_prob=0.0, use_cross_attn=cross_attn).to(+ get_current_device()+ )parallel_model = DiT_models["DiT-B/8"](- text_dropout_prob=0.0, seq_parallel_group=plugin.sp_group+ text_dropout_prob=0.0,+ use_cross_attn=cross_attn,+ seq_parallel_group=plugin.sp_group,+ seq_parallel_mode=sp_mode,+ seq_parallel_overlap=sp_overlap,).to(get_current_device())parallel_model.load_state_dict(model.state_dict())opt = HybridAdam(parallel_model.parameters(), lr=1e-3)diff --git a/tests/test_sp_attn.py b/tests/test_sp_attn.pyindex 58d3b7d..d62a4ab 100644--- a/tests/test_sp_attn.py+++ b/tests/test_sp_attn.py@@ -3,15 +3,27 @@ import pytestimport torchimport torch.distributed as distfrom colossalai.logging import disable_existing_loggers-from colossalai.testing import rerun_if_address_is_in_use, spawn+from colossalai.testing import parameterize, rerun_if_address_is_in_use, spawnfrom colossalai.utils import get_current_devicefrom torch.testing import assert_close-from open_sora.modeling.dit import CrossAttention, SeqParallelCrossAttention+from open_sora.modeling.dit import (+ CrossAttention,+ FastSeqParallelCrossAttention,+ SeqParallelCrossAttention,+)from open_sora.utils.comm import gather_seq, split_seq-def check_sp_attn():+@parameterize("layer_cls", [SeqParallelCrossAttention, FastSeqParallelCrossAttention])+@parameterize("overlap", [True, False])+@parameterize("cross_attn", [True, False])+def check_sp_attn(layer_cls, overlap, cross_attn):+ if overlap and layer_cls == SeqParallelCrossAttention:+ return+ model_kwargs = {}+ if layer_cls == FastSeqParallelCrossAttention:+ model_kwargs["overlap"] = overlapsp_size = dist.get_world_size()sp_rank = dist.get_rank()q_dim, context_dim = 8, 4@@ -20,19 +32,33 @@ def check_sp_attn():bs = 2sq = 8skv = 4+ if not cross_attn:+ context_dim = None+ skv = sqattn = CrossAttention(q_dim, context_dim, num_heads, head_dim).to(get_current_device())- parallel_attn = SeqParallelCrossAttention(- q_dim, context_dim, num_heads, head_dim, seq_parallel_group=dist.group.WORLD+ parallel_attn = layer_cls(+ q_dim,+ context_dim,+ num_heads,+ head_dim,+ seq_parallel_group=dist.group.WORLD,+ **model_kwargs,).to(get_current_device())parallel_attn.load_state_dict(attn.state_dict())hidden_states = torch.rand(bs, sq, q_dim, device=get_current_device())- context = torch.rand(bs, skv, context_dim, device=get_current_device())+ if cross_attn:+ context = torch.rand(bs, skv, context_dim, device=get_current_device())+ else:+ context = Nonemask = torch.zeros(bs, 1, sq, skv, device=get_current_device())target = attn(hidden_states, context, mask)hidden_states_parallel = split_seq(hidden_states, sp_size, sp_rank)- context_parallel = split_seq(context, sp_size, sp_rank)+ if cross_attn:+ context_parallel = split_seq(context, sp_size, sp_rank)+ else:+ context_parallel = Noneoutput_parallel = parallel_attn(hidden_states_parallel,context_parallel,@@ -49,8 +75,8 @@ def check_sp_attn():p.grad.data.div_(sp_size)dist.all_reduce(p.grad.data)- for p1, p2 in zip(attn.parameters(), parallel_attn.parameters()):- assert_close(p1.grad, p2.grad)+ for (n, p1), p2 in zip(attn.named_parameters(), parallel_attn.parameters()):+ assert_close(p1.grad, p2.grad, msg=lambda m: f"Check {n}\n{m}")def run_dist(rank, world_size, port):diff --git a/train.py b/train.pyindex 1f4da90..7ddd738 100644--- a/train.py+++ b/train.py@@ -27,6 +27,7 @@ from tqdm import tqdmfrom open_sora.diffusion import create_diffusionfrom open_sora.modeling import DiT_models+from open_sora.modeling.dit import SUPPORTED_SEQ_PARALLEL_MODESfrom open_sora.utils.data import (create_video_compressor,load_datasets,@@ -111,6 +112,8 @@ def main(args):model_kwargs = {"in_channels": video_compressor.out_channels,"seq_parallel_group": plugin.sp_group,+ "seq_parallel_mode": args.sp_mode,+ "seq_parallel_overlap": args.sp_overlap,}# Step 4: Create DiT and EMA@@ -247,6 +250,10 @@ if __name__ == "__main__":parser.add_argument("-g", "--grad_checkpoint", action="store_true", default=False)parser.add_argument("-a", "--accumulation_steps", default=1, type=int)parser.add_argument("--sp_size", type=int, default=1)+ parser.add_argument(+ "--sp_mode", type=str, default="ulysses", choices=SUPPORTED_SEQ_PARALLEL_MODES+ )+ parser.add_argument("--sp_overlap", action="store_true", default=False)parser.add_argument("--lr", type=float, default=1e-4)parser.add_argument("--save_interval", type=int, default=20)parser.add_argument("--checkpoint_dir", type=str, default="checkpoints")