Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[feature] impl fastseq-style seq parallel (#21)

Hongxin Liu91275b2b5e2d
diff --git a/benchmark.py b/benchmark.py
index dbe9855..5343fad 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -11,6 +11,7 @@ import argparse
import time
import torch
+import torch.distributed as dist
from colossalai import launch_from_torch
from colossalai.accelerator import get_accelerator
from colossalai.booster import Booster
@@ -23,6 +24,7 @@ from tqdm import tqdm
from open_sora.diffusion import create_diffusion
from open_sora.modeling import DiT_models
+from open_sora.modeling.dit import SUPPORTED_SEQ_PARALLEL_MODES
from open_sora.utils.data import create_video_compressor, preprocess_batch
from open_sora.utils.plugin import ZeroSeqParallelPlugin
@@ -55,6 +57,8 @@ def main(args):
model_kwargs = {
"in_channels": video_compressor.out_channels,
"seq_parallel_group": plugin.sp_group,
+ "seq_parallel_mode": args.sp_mode,
+ "seq_parallel_overlap": args.sp_overlap,
}
# Create DiT and EMA
@@ -93,7 +97,7 @@ def main(args):
video_inputs = batch.pop("video_latent_states")
mask = batch.pop("video_padding_mask")
logger.info(
- f"Num patches: {video_inputs.shape[1]}, num_tokens: {batch['text_latent_states'].shape[1]}",
+ f"Num patches: {video_inputs.shape[1]}, num tokens: {batch['text_latent_states'].shape[1]}",
ranks=[0],
)
@@ -129,6 +133,11 @@ def main(args):
if i >= args.warmup_steps:
total_samples += args.batch_size * coordinator.world_size
total_duration += time_per_iter
+ total_duration = torch.tensor([total_duration], device=get_current_device())
+ dist.all_reduce(total_duration)
+ total_duration = total_duration / coordinator.world_size
+ total_duration = total_duration.item()
+ total_samples *= coordinator.world_size // args.sp_size
throughput = total_samples / total_duration
logger.info(
@@ -136,7 +145,7 @@ def main(args):
ranks=[0],
)
logger.info(
- f"Throughput per device: {throughput:.2f} samples/s",
+ f"Throughput: {throughput:.2f} samples/s",
ranks=[0],
)
@@ -151,6 +160,10 @@ if __name__ == "__main__":
"-p", "--plugin", type=str, default="zero2", choices=["ddp", "zero2"]
)
parser.add_argument("--sp_size", type=int, default=1)
+ parser.add_argument(
+ "--sp_mode", type=str, default="ulysses", choices=SUPPORTED_SEQ_PARALLEL_MODES
+ )
+ parser.add_argument("--sp_overlap", action="store_true", default=False)
parser.add_argument("-w", "--warmup_steps", type=int, default=2)
parser.add_argument("-s", "--steps", type=int, default=3)
parser.add_argument("-b", "--batch_size", type=int, default=4)
diff --git a/open_sora/modeling/attn.py b/open_sora/modeling/attn.py
new file mode 100644
index 0000000..f81d36b
--- /dev/null
+++ b/open_sora/modeling/attn.py
@@ -0,0 +1,330 @@
+from typing import Optional
+
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+import torch.nn.functional as F
+from colossalai.logging import get_dist_logger
+from colossalai.shardformer.layer._operation import gather_forward_split_backward
+
+from open_sora.utils.comm import all_to_all, async_all_gather_proj_for_two
+
+
+class CrossAttention(nn.Module):
+ r"""
+ A cross attention layer.
+
+ Parameters:
+ query_dim (`int`): The number of channels in the query.
+ cross_attention_dim (`int`, *optional*):
+ The number of channels in the context. If not given, defaults to `query_dim`.
+ num_heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.
+ head_dim (`int`, *optional*, defaults to 64): The number of channels in each head.
+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
+ bias (`bool`, *optional*, defaults to False):
+ Set to `True` for the query, key, and value linear layers to contain a bias parameter.
+ """
+
+ def __init__(
+ self,
+ query_dim: int,
+ cross_attention_dim: Optional[int] = None,
+ num_heads: int = 8,
+ head_dim: int = 64,
+ dropout: float = 0.0,
+ bias=False,
+ sdpa=True,
+ ):
+ super().__init__()
+ self.hidden_size = head_dim * num_heads
+ cross_attention_dim = (
+ cross_attention_dim if cross_attention_dim is not None else query_dim
+ )
+
+ self.scale = head_dim**-0.5
+ self.num_heads = num_heads
+ self.head_dim = head_dim
+ self.sdpa = sdpa
+
+ self.to_q = nn.Linear(query_dim, self.hidden_size, bias=bias)
+ self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)
+ self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)
+
+ self.to_out = nn.Sequential(
+ nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)
+ )
+
+ def forward(self, hidden_states, context=None, mask=None):
+ bsz, q_len, _ = hidden_states.shape
+
+ query = self.to_q(hidden_states)
+ context = context if context is not None else hidden_states
+ kv_seq_len = context.shape[1]
+ key = self.to_k(context)
+ value = self.to_v(context)
+
+ # [B, S, H, D]
+ query = query.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+ key = key.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)
+ value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(
+ 1, 2
+ )
+
+ if mask is not None:
+ assert mask.shape == (bsz, 1, q_len, kv_seq_len)
+ if self.sdpa:
+ attn_output = F.scaled_dot_product_attention(
+ query, key, value, attn_mask=mask, scale=self.scale
+ )
+ else:
+ attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale
+ assert attn_weights.shape == (bsz, self.num_heads, q_len, kv_seq_len)
+ if mask is not None:
+ attn_weights = attn_weights + mask
+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ query.dtype
+ )
+ attn_output = torch.matmul(attn_weights, value)
+ assert attn_output.shape == (bsz, self.num_heads, q_len, self.head_dim)
+ attn_output = attn_output.transpose(1, 2).contiguous()
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+ attn_output = self.to_out(attn_output)
+ return attn_output
+
+
+class SeqParallelCrossAttention(CrossAttention):
+ r"""
+ A cross attention layer.
+
+ Parameters:
+ query_dim (`int`): The number of channels in the query.
+ cross_attention_dim (`int`, *optional*):
+ The number of channels in the context. If not given, defaults to `query_dim`.
+ num_heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.
+ head_dim (`int`, *optional*, defaults to 64): The number of channels in each head.
+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
+ bias (`bool`, *optional*, defaults to False):
+ Set to `True` for the query, key, and value linear layers to contain a bias parameter.
+ """
+
+ def __init__(
+ self,
+ query_dim: int,
+ cross_attention_dim: Optional[int] = None,
+ num_heads: int = 8,
+ head_dim: int = 64,
+ dropout: float = 0.0,
+ bias=False,
+ sdpa=True,
+ seq_parallel_group=None,
+ ):
+ super().__init__(
+ query_dim,
+ cross_attention_dim,
+ num_heads,
+ head_dim,
+ dropout,
+ bias,
+ sdpa,
+ )
+ self.seq_parallel_group = seq_parallel_group
+ self.seq_parallel_size = (
+ dist.get_world_size(self.seq_parallel_group)
+ if seq_parallel_group is not None
+ else 1
+ )
+ assert self.num_heads % self.seq_parallel_size == 0
+
+ def forward(self, hidden_states, context=None, mask=None):
+ bsz, q_len, _ = hidden_states.shape
+
+ query = self.to_q(hidden_states)
+ context = context if context is not None else hidden_states
+ kv_seq_len = context.shape[1]
+ key = self.to_k(context)
+ value = self.to_v(context)
+
+ # [B, S/P, H] -> [B, S, H/P]
+ num_heads_parallel = self.num_heads // self.seq_parallel_size
+ hidden_size_parallel = self.hidden_size // self.seq_parallel_size
+ if self.seq_parallel_size > 1:
+ query = all_to_all(
+ query, self.seq_parallel_group, scatter_dim=2, gather_dim=1
+ )
+ key = all_to_all(key, self.seq_parallel_group, scatter_dim=2, gather_dim=1)
+ value = all_to_all(
+ value, self.seq_parallel_group, scatter_dim=2, gather_dim=1
+ )
+
+ q_len *= self.seq_parallel_size
+ kv_seq_len *= self.seq_parallel_size
+
+ # [B, S, H/P] -> [B, S, N/P, D] -> [B, N/P, S, D]
+ query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(
+ 1, 2
+ )
+ key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(
+ 1, 2
+ )
+ value = value.view(
+ bsz, kv_seq_len, num_heads_parallel, self.head_dim
+ ).transpose(1, 2)
+
+ if mask is not None:
+ assert mask.shape == (bsz, 1, q_len, kv_seq_len)
+ if self.sdpa:
+ attn_output = F.scaled_dot_product_attention(
+ query, key, value, attn_mask=mask, scale=self.scale
+ )
+ else:
+ attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale
+ assert attn_weights.shape == (bsz, num_heads_parallel, q_len, kv_seq_len)
+ if mask is not None:
+ attn_weights = attn_weights + mask
+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ query.dtype
+ )
+ attn_output = torch.matmul(attn_weights, value)
+ assert attn_output.shape == (bsz, num_heads_parallel, q_len, self.head_dim)
+ attn_output = attn_output.transpose(1, 2).contiguous()
+ attn_output = attn_output.reshape(bsz, q_len, hidden_size_parallel)
+ # [B, S, H/P] -> [B, S/P, H]
+ if self.seq_parallel_size > 1:
+ attn_output = all_to_all(
+ attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2
+ )
+ attn_output = self.to_out(attn_output)
+ return attn_output
+
+
+class FastSeqParallelCrossAttention(SeqParallelCrossAttention):
+ def __init__(
+ self,
+ query_dim: int,
+ cross_attention_dim: Optional[int] = None,
+ num_heads: int = 8,
+ head_dim: int = 64,
+ dropout: float = 0,
+ bias=False,
+ sdpa=True,
+ seq_parallel_group=None,
+ overlap=False,
+ ):
+ super().__init__(
+ query_dim,
+ cross_attention_dim,
+ num_heads,
+ head_dim,
+ dropout,
+ bias,
+ sdpa,
+ seq_parallel_group,
+ )
+ self.seq_parallel_rank = (
+ dist.get_rank(self.seq_parallel_group)
+ if seq_parallel_group is not None
+ else 0
+ )
+ self.sequence_parallel_param_slice = slice(
+ self.hidden_size // self.seq_parallel_size * self.seq_parallel_rank,
+ self.hidden_size // self.seq_parallel_size * (self.seq_parallel_rank + 1),
+ )
+ if overlap and self.seq_parallel_size != 2:
+ logger = get_dist_logger()
+ logger.warning(
+ "FastSeqParallelCrossAttention only supports overlap with seq_parallel_size=2. Fallback to non-overlap",
+ ranks=[0],
+ )
+ overlap = False
+ self.overlap = overlap
+
+ def _get_sliced_params(self, proj_layer: nn.Linear):
+ bias = bias = (
+ proj_layer.bias[self.sequence_parallel_param_slice]
+ if proj_layer.bias is not None
+ else None
+ )
+ return proj_layer.weight[self.sequence_parallel_param_slice], bias
+
+ def _proj(self, x: torch.Tensor, proj_layer: nn.Linear):
+ return F.linear(
+ x,
+ *self._get_sliced_params(proj_layer),
+ )
+
+ def forward(self, hidden_states, context=None, mask=None):
+ bsz, q_len, _ = hidden_states.shape
+
+ context = context if context is not None else hidden_states
+ kv_seq_len = context.shape[1]
+ if self.seq_parallel_size > 1:
+ if self.overlap and self.seq_parallel_size == 2:
+ query, key, value = async_all_gather_proj_for_two(
+ hidden_states,
+ context,
+ *self._get_sliced_params(self.to_q),
+ *self._get_sliced_params(self.to_k),
+ *self._get_sliced_params(self.to_v),
+ dim=1,
+ process_group=self.seq_parallel_group,
+ sp_size=self.seq_parallel_size,
+ sp_rank=self.seq_parallel_rank,
+ )
+ else:
+ # [B, S/P, H] -> [B, S, H]
+ hidden_states = gather_forward_split_backward(
+ hidden_states, 1, self.seq_parallel_group
+ )
+ context = gather_forward_split_backward(
+ context, 1, self.seq_parallel_group
+ )
+ query = self._proj(hidden_states, self.to_q)
+ key = self._proj(context, self.to_k)
+ value = self._proj(context, self.to_v)
+ else:
+ query = self.to_q(hidden_states)
+ key = self.to_k(context)
+ value = self.to_v(context)
+ # output is [B, S, H/P]
+
+ num_heads_parallel = self.num_heads // self.seq_parallel_size
+ hidden_size_parallel = self.hidden_size // self.seq_parallel_size
+ q_len *= self.seq_parallel_size
+ kv_seq_len *= self.seq_parallel_size
+
+ # [B, S, H/P] -> [B, S, N/P, D] -> [B, N/P, S, D]
+ query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(
+ 1, 2
+ )
+ key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(
+ 1, 2
+ )
+ value = value.view(
+ bsz, kv_seq_len, num_heads_parallel, self.head_dim
+ ).transpose(1, 2)
+
+ if mask is not None:
+ assert mask.shape == (bsz, 1, q_len, kv_seq_len)
+ if self.sdpa:
+ attn_output = F.scaled_dot_product_attention(
+ query, key, value, attn_mask=mask, scale=self.scale
+ )
+ else:
+ attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale
+ assert attn_weights.shape == (bsz, num_heads_parallel, q_len, kv_seq_len)
+ if mask is not None:
+ attn_weights = attn_weights + mask
+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ query.dtype
+ )
+ attn_output = torch.matmul(attn_weights, value)
+ assert attn_output.shape == (bsz, num_heads_parallel, q_len, self.head_dim)
+ attn_output = attn_output.transpose(1, 2).contiguous()
+ attn_output = attn_output.reshape(bsz, q_len, hidden_size_parallel)
+ # [B, S, H/P] -> [B, S/P, H]
+ if self.seq_parallel_size > 1:
+ attn_output = all_to_all(
+ attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2
+ )
+ attn_output = self.to_out(attn_output)
+ return attn_output
diff --git a/open_sora/modeling/dit.py b/open_sora/modeling/dit.py
index ac9ce8a..776f736 100644
--- a/open_sora/modeling/dit.py
+++ b/open_sora/modeling/dit.py
@@ -19,203 +19,15 @@ import torch.nn as nn
import torch.nn.functional as F
from timm.models.vision_transformer import Mlp
-from open_sora.utils.comm import all_to_all, gather_seq, split_seq
-
-
-class CrossAttention(nn.Module):
- r"""
- A cross attention layer.
-
- Parameters:
- query_dim (`int`): The number of channels in the query.
- cross_attention_dim (`int`, *optional*):
- The number of channels in the context. If not given, defaults to `query_dim`.
- num_heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.
- head_dim (`int`, *optional*, defaults to 64): The number of channels in each head.
- dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
- bias (`bool`, *optional*, defaults to False):
- Set to `True` for the query, key, and value linear layers to contain a bias parameter.
- """
-
- def __init__(
- self,
- query_dim: int,
- cross_attention_dim: Optional[int] = None,
- num_heads: int = 8,
- head_dim: int = 64,
- dropout: float = 0.0,
- bias=False,
- sdpa=True,
- ):
- super().__init__()
- self.hidden_size = head_dim * num_heads
- cross_attention_dim = (
- cross_attention_dim if cross_attention_dim is not None else query_dim
- )
-
- self.scale = head_dim**-0.5
- self.num_heads = num_heads
- self.head_dim = head_dim
- self.sdpa = sdpa
-
- self.to_q = nn.Linear(query_dim, self.hidden_size, bias=bias)
- self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)
- self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)
-
- self.to_out = nn.Sequential(
- nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)
- )
-
- def forward(self, hidden_states, context=None, mask=None):
- bsz, q_len, _ = hidden_states.shape
-
- query = self.to_q(hidden_states)
- context = context if context is not None else hidden_states
- kv_seq_len = context.shape[1]
- key = self.to_k(context)
- value = self.to_v(context)
+from open_sora.utils.comm import gather_seq, split_seq
- # [B, S, H, D]
- query = query.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
- key = key.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)
- value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(
- 1, 2
- )
+from .attn import (
+ CrossAttention,
+ FastSeqParallelCrossAttention,
+ SeqParallelCrossAttention,
+)
- if mask is not None:
- assert mask.shape == (bsz, 1, q_len, kv_seq_len)
- if self.sdpa:
- attn_output = F.scaled_dot_product_attention(
- query, key, value, attn_mask=mask, scale=self.scale
- )
- else:
- attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale
- assert attn_weights.shape == (bsz, self.num_heads, q_len, kv_seq_len)
- if mask is not None:
- attn_weights = attn_weights + mask
- attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
- query.dtype
- )
- attn_output = torch.matmul(attn_weights, value)
- assert attn_output.shape == (bsz, self.num_heads, q_len, self.head_dim)
- attn_output = attn_output.transpose(1, 2).contiguous()
- attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
- attn_output = self.to_out(attn_output)
- return attn_output
-
-
-class SeqParallelCrossAttention(nn.Module):
- r"""
- A cross attention layer.
-
- Parameters:
- query_dim (`int`): The number of channels in the query.
- cross_attention_dim (`int`, *optional*):
- The number of channels in the context. If not given, defaults to `query_dim`.
- num_heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.
- head_dim (`int`, *optional*, defaults to 64): The number of channels in each head.
- dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
- bias (`bool`, *optional*, defaults to False):
- Set to `True` for the query, key, and value linear layers to contain a bias parameter.
- """
-
- def __init__(
- self,
- query_dim: int,
- cross_attention_dim: Optional[int] = None,
- num_heads: int = 8,
- head_dim: int = 64,
- dropout: float = 0.0,
- bias=False,
- sdpa=True,
- seq_parallel_group=None,
- ):
- super().__init__()
- self.hidden_size = head_dim * num_heads
- cross_attention_dim = (
- cross_attention_dim if cross_attention_dim is not None else query_dim
- )
-
- self.scale = head_dim**-0.5
- self.num_heads = num_heads
- self.head_dim = head_dim
- self.sdpa = sdpa
- self.seq_parallel_group = seq_parallel_group
- self.seq_parallel_size = (
- dist.get_world_size(self.seq_parallel_group)
- if seq_parallel_group is not None
- else 1
- )
- assert self.num_heads % self.seq_parallel_size == 0
-
- self.to_q = nn.Linear(query_dim, self.hidden_size, bias=bias)
- self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)
- self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)
-
- self.to_out = nn.Sequential(
- nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)
- )
-
- def forward(self, hidden_states, context=None, mask=None):
- bsz, q_len, _ = hidden_states.shape
-
- query = self.to_q(hidden_states)
- context = context if context is not None else hidden_states
- kv_seq_len = context.shape[1]
- key = self.to_k(context)
- value = self.to_v(context)
-
- # [B, S/P, H] -> [B, S, H/P]
- num_heads_parallel = self.num_heads // self.seq_parallel_size
- hidden_size_parallel = self.hidden_size // self.seq_parallel_size
- if self.seq_parallel_group is not None and self.seq_parallel_size > 1:
- query = all_to_all(
- query, self.seq_parallel_group, scatter_dim=2, gather_dim=1
- )
- key = all_to_all(key, self.seq_parallel_group, scatter_dim=2, gather_dim=1)
- value = all_to_all(
- value, self.seq_parallel_group, scatter_dim=2, gather_dim=1
- )
-
- q_len *= self.seq_parallel_size
- kv_seq_len *= self.seq_parallel_size
-
- # [B, S, H/P] -> [B, S, N/P, D] -> [B, N/P, S, D]
- query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(
- 1, 2
- )
- key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(
- 1, 2
- )
- value = value.view(
- bsz, kv_seq_len, num_heads_parallel, self.head_dim
- ).transpose(1, 2)
-
- if mask is not None:
- assert mask.shape == (bsz, 1, q_len, kv_seq_len)
- if self.sdpa:
- attn_output = F.scaled_dot_product_attention(
- query, key, value, attn_mask=mask, scale=self.scale
- )
- else:
- attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale
- assert attn_weights.shape == (bsz, num_heads_parallel, q_len, kv_seq_len)
- if mask is not None:
- attn_weights = attn_weights + mask
- attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
- query.dtype
- )
- attn_output = torch.matmul(attn_weights, value)
- assert attn_output.shape == (bsz, num_heads_parallel, q_len, self.head_dim)
- attn_output = attn_output.transpose(1, 2).contiguous()
- attn_output = attn_output.reshape(bsz, q_len, hidden_size_parallel)
- # [B, S, H/P] -> [B, S/P, H]
- if self.seq_parallel_group is not None and self.seq_parallel_size > 1:
- attn_output = all_to_all(
- attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2
- )
- attn_output = self.to_out(attn_output)
- return attn_output
+SUPPORTED_SEQ_PARALLEL_MODES = ["ulysses", "fastseq"]
def modulate(x, shift, scale):
@@ -408,17 +220,33 @@ class DiTBlock(nn.Module):
cross_attention_dim=None,
mlp_ratio=4.0,
seq_parallel_group=None,
+ seq_parallel_mode="ulysses",
+ seq_parallel_overlap=False,
):
super().__init__()
self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
- self.attn = SeqParallelCrossAttention(
+ attn_kwargs = {}
+ if seq_parallel_group is not None:
+ attn_kwargs["seq_parallel_group"] = seq_parallel_group
+ if seq_parallel_mode == "ulysses":
+ attn_cls = SeqParallelCrossAttention
+ elif seq_parallel_mode == "fastseq":
+ attn_cls = FastSeqParallelCrossAttention
+ attn_kwargs["overlap"] = seq_parallel_overlap
+ else:
+ raise ValueError(
+ f"seq_parallel_mode must be one of {SUPPORTED_SEQ_PARALLEL_MODES}"
+ )
+ else:
+ attn_cls = CrossAttention
+ self.attn = attn_cls(
query_dim=hidden_size,
cross_attention_dim=cross_attention_dim,
num_heads=num_heads,
head_dim=hidden_size // num_heads,
bias=True,
sdpa=True,
- seq_parallel_group=seq_parallel_group,
+ **attn_kwargs,
)
self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
@@ -500,6 +328,8 @@ class DiT(nn.Module):
learn_sigma=True,
use_cross_attn=True,
seq_parallel_group=None,
+ seq_parallel_mode="ulysses",
+ seq_parallel_overlap=False,
):
super().__init__()
self.grad_checkpointing = False
@@ -546,6 +376,8 @@ class DiT(nn.Module):
cross_attn_dim,
mlp_ratio=mlp_ratio,
seq_parallel_group=seq_parallel_group,
+ seq_parallel_mode=seq_parallel_mode,
+ seq_parallel_overlap=seq_parallel_overlap,
)
for _ in range(depth)
]
diff --git a/open_sora/utils/comm.py b/open_sora/utils/comm.py
index 00703d3..9b947ff 100644
--- a/open_sora/utils/comm.py
+++ b/open_sora/utils/comm.py
@@ -1,7 +1,11 @@
+from typing import Optional, Tuple
+
import torch
import torch.distributed as dist
+import torch.nn.functional as F
from colossalai.moe._operation import MoeInGradScaler, MoeOutGradScaler
from colossalai.shardformer.layer._operation import gather_forward_split_backward
+from torch.distributed.distributed_c10d import get_global_rank
def _all_to_all(
@@ -94,3 +98,195 @@ def gather_seq(
"""
input_ = gather_forward_split_backward(input_, dim, sp_group)
return MoeInGradScaler.apply(input_, sp_size)
+
+
+class AsyncAllGatherProjForTwo(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ hidden_states: torch.Tensor,
+ context: torch.Tensor,
+ q_proj_weight: torch.Tensor,
+ q_proj_bias: Optional[torch.Tensor],
+ k_proj_weight: torch.Tensor,
+ k_proj_bias: Optional[torch.Tensor],
+ v_proj_weight: torch.Tensor,
+ v_proj_bias: Optional[torch.Tensor],
+ dim: int,
+ process_group: dist.ProcessGroup,
+ sp_size: int,
+ sp_rank: int,
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ assert sp_size == 2
+ ctx.process_group = process_group
+ ctx.sp_size = sp_size
+ ctx.sp_rank = sp_rank
+ ctx.dim = dim
+
+ is_cross_attn = not (hidden_states is context)
+ ctx.is_cross_attn = is_cross_attn
+
+ recv_hidden_states = torch.empty_like(hidden_states)
+ if is_cross_attn:
+ recv_context = torch.empty_like(context)
+ else:
+ recv_context = recv_hidden_states
+
+ peer_global_rank = get_global_rank(process_group, 1 - sp_rank)
+ ops = [
+ dist.P2POp(dist.isend, hidden_states, peer_global_rank, process_group),
+ dist.P2POp(dist.irecv, recv_hidden_states, peer_global_rank, process_group),
+ ]
+ if is_cross_attn:
+ ops.extend(
+ [
+ dist.P2POp(dist.isend, context, peer_global_rank, process_group),
+ dist.P2POp(
+ dist.irecv, recv_context, peer_global_rank, process_group
+ ),
+ ]
+ )
+
+ reqs = dist.batch_isend_irecv(ops)
+
+ # [B, S/P, H]
+ q = F.linear(hidden_states, q_proj_weight, q_proj_bias)
+ k = F.linear(context, k_proj_weight, k_proj_bias)
+ v = F.linear(context, v_proj_weight, v_proj_bias)
+
+ for req in reqs:
+ req.wait()
+
+ q_other = F.linear(recv_hidden_states, q_proj_weight, q_proj_bias)
+ k_other = F.linear(recv_context, k_proj_weight, k_proj_bias)
+ v_other = F.linear(recv_context, v_proj_weight, v_proj_bias)
+
+ if sp_rank == 0:
+ q = torch.cat([q, q_other], dim=dim)
+ k = torch.cat([k, k_other], dim=dim)
+ v = torch.cat([v, v_other], dim=dim)
+ else:
+ q = torch.cat([q_other, q], dim=dim)
+ k = torch.cat([k_other, k], dim=dim)
+ v = torch.cat([v_other, v], dim=dim)
+
+ ctx.save_for_backward(
+ hidden_states,
+ context,
+ q_proj_weight,
+ q_proj_bias,
+ k_proj_weight,
+ k_proj_bias,
+ v_proj_weight,
+ v_proj_bias,
+ recv_hidden_states,
+ recv_context,
+ )
+
+ return q, k, v
+
+ @staticmethod
+ def backward(
+ ctx: torch.Any, q_grad: torch.Tensor, k_grad: torch.Tensor, v_grad: torch.Tensor
+ ) -> torch.Any:
+ (
+ hidden_states,
+ context,
+ q_proj_weight,
+ q_proj_bias,
+ k_proj_weight,
+ k_proj_bias,
+ v_proj_weight,
+ v_proj_bias,
+ recv_hidden_states,
+ recv_context,
+ ) = ctx.saved_tensors
+
+ # compute param grads
+ if ctx.sp_rank == 0:
+ hidden_states = torch.cat([hidden_states, recv_hidden_states], dim=ctx.dim)
+ if ctx.is_cross_attn:
+ context = torch.cat([context, recv_context], dim=ctx.dim)
+ else:
+ context = hidden_states
+ else:
+ hidden_states = torch.cat([recv_hidden_states, hidden_states], dim=ctx.dim)
+ if ctx.is_cross_attn:
+ context = torch.cat([recv_context, context], dim=ctx.dim)
+ else:
+ context = hidden_states
+ q_proj_weight_grad = q_grad.transpose(-1, -2).matmul(hidden_states).sum(dim=0)
+ q_proj_bias_grad = (
+ q_grad.sum(dim=0).sum(dim=0) if q_proj_bias is not None else None
+ )
+ k_proj_weight_grad = k_grad.transpose(-1, -2).matmul(context).sum(dim=0)
+ k_proj_bias_grad = (
+ k_grad.sum(dim=0).sum(dim=0) if k_proj_bias is not None else None
+ )
+ v_proj_weight_grad = v_grad.transpose(-1, -2).matmul(context).sum(dim=0)
+ v_proj_bias_grad = (
+ v_grad.sum(dim=0).sum(dim=0) if v_proj_bias is not None else None
+ )
+
+ # split grads
+ q_grad = q_grad.chunk(ctx.sp_size, dim=ctx.dim)[ctx.sp_rank].clone()
+ k_grad = k_grad.chunk(ctx.sp_size, dim=ctx.dim)[ctx.sp_rank].clone()
+ v_grad = v_grad.chunk(ctx.sp_size, dim=ctx.dim)[ctx.sp_rank].clone()
+
+ if ctx.is_cross_attn:
+ hidden_states_grad = torch.matmul(q_grad, q_proj_weight)
+ context_grad = torch.matmul(k_grad, k_proj_weight) + torch.matmul(
+ v_grad, v_proj_weight
+ )
+ else:
+ hidden_states_grad = (
+ torch.matmul(q_grad, q_proj_weight)
+ + torch.matmul(k_grad, k_proj_weight)
+ + torch.matmul(v_grad, v_proj_weight)
+ )
+ context_grad = hidden_states_grad
+
+ return (
+ hidden_states_grad,
+ context_grad,
+ q_proj_weight_grad,
+ q_proj_bias_grad,
+ k_proj_weight_grad,
+ k_proj_bias_grad,
+ v_proj_weight_grad,
+ v_proj_bias_grad,
+ None,
+ None,
+ None,
+ None,
+ )
+
+
+def async_all_gather_proj_for_two(
+ hidden_states: torch.Tensor,
+ context: torch.Tensor,
+ q_proj_weight: torch.Tensor,
+ q_proj_bias: Optional[torch.Tensor],
+ k_proj_weight: torch.Tensor,
+ k_proj_bias: Optional[torch.Tensor],
+ v_proj_weight: torch.Tensor,
+ v_proj_bias: Optional[torch.Tensor],
+ dim: int,
+ process_group: dist.ProcessGroup,
+ sp_size: int,
+ sp_rank: int,
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ return AsyncAllGatherProjForTwo.apply(
+ hidden_states,
+ context,
+ q_proj_weight,
+ q_proj_bias,
+ k_proj_weight,
+ k_proj_bias,
+ v_proj_weight,
+ v_proj_bias,
+ dim,
+ process_group,
+ sp_size,
+ sp_rank,
+ )
diff --git a/tests/test_model.py b/tests/test_model.py
index 62b3172..1c2462f 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -15,16 +15,40 @@ from open_sora.utils.plugin import ZeroSeqParallelPlugin
@parameterize("sp_size", [2, 4])
+@parameterize("sp_mode", ["ulysses", "fastseq"])
+@parameterize("sp_overlap", [True, False])
+@parameterize("cross_attn", [True, False])
def check_dit_model_fwd_bwd(
- sp_size: int, video_latent_states, text_latent_states, t, mask
+ sp_size: int,
+ sp_mode: str,
+ sp_overlap: bool,
+ cross_attn: bool,
+ video_latent_states,
+ text_latent_states,
+ t,
+ mask,
):
+ if sp_overlap and sp_mode == "ulysses":
+ return
+ if not cross_attn:
+ mask = mask.new_ones(
+ *mask.shape[:2],
+ mask.shape[2] + mask.shape[3],
+ mask.shape[2] + mask.shape[3],
+ )
plugin = ZeroSeqParallelPlugin(
sp_size=sp_size, stage=2, precision="fp32", master_weights=False
)
booster = Booster(plugin=plugin)
- model = DiT_models["DiT-B/8"](text_dropout_prob=0.0).to(get_current_device())
+ model = DiT_models["DiT-B/8"](text_dropout_prob=0.0, use_cross_attn=cross_attn).to(
+ get_current_device()
+ )
parallel_model = DiT_models["DiT-B/8"](
- text_dropout_prob=0.0, seq_parallel_group=plugin.sp_group
+ text_dropout_prob=0.0,
+ use_cross_attn=cross_attn,
+ seq_parallel_group=plugin.sp_group,
+ seq_parallel_mode=sp_mode,
+ seq_parallel_overlap=sp_overlap,
).to(get_current_device())
parallel_model.load_state_dict(model.state_dict())
opt = HybridAdam(parallel_model.parameters(), lr=1e-3)
diff --git a/tests/test_sp_attn.py b/tests/test_sp_attn.py
index 58d3b7d..d62a4ab 100644
--- a/tests/test_sp_attn.py
+++ b/tests/test_sp_attn.py
@@ -3,15 +3,27 @@ import pytest
import torch
import torch.distributed as dist
from colossalai.logging import disable_existing_loggers
-from colossalai.testing import rerun_if_address_is_in_use, spawn
+from colossalai.testing import parameterize, rerun_if_address_is_in_use, spawn
from colossalai.utils import get_current_device
from torch.testing import assert_close
-from open_sora.modeling.dit import CrossAttention, SeqParallelCrossAttention
+from open_sora.modeling.dit import (
+ CrossAttention,
+ FastSeqParallelCrossAttention,
+ SeqParallelCrossAttention,
+)
from open_sora.utils.comm import gather_seq, split_seq
-def check_sp_attn():
+@parameterize("layer_cls", [SeqParallelCrossAttention, FastSeqParallelCrossAttention])
+@parameterize("overlap", [True, False])
+@parameterize("cross_attn", [True, False])
+def check_sp_attn(layer_cls, overlap, cross_attn):
+ if overlap and layer_cls == SeqParallelCrossAttention:
+ return
+ model_kwargs = {}
+ if layer_cls == FastSeqParallelCrossAttention:
+ model_kwargs["overlap"] = overlap
sp_size = dist.get_world_size()
sp_rank = dist.get_rank()
q_dim, context_dim = 8, 4
@@ -20,19 +32,33 @@ def check_sp_attn():
bs = 2
sq = 8
skv = 4
+ if not cross_attn:
+ context_dim = None
+ skv = sq
attn = CrossAttention(q_dim, context_dim, num_heads, head_dim).to(
get_current_device()
)
- parallel_attn = SeqParallelCrossAttention(
- q_dim, context_dim, num_heads, head_dim, seq_parallel_group=dist.group.WORLD
+ parallel_attn = layer_cls(
+ q_dim,
+ context_dim,
+ num_heads,
+ head_dim,
+ seq_parallel_group=dist.group.WORLD,
+ **model_kwargs,
).to(get_current_device())
parallel_attn.load_state_dict(attn.state_dict())
hidden_states = torch.rand(bs, sq, q_dim, device=get_current_device())
- context = torch.rand(bs, skv, context_dim, device=get_current_device())
+ if cross_attn:
+ context = torch.rand(bs, skv, context_dim, device=get_current_device())
+ else:
+ context = None
mask = torch.zeros(bs, 1, sq, skv, device=get_current_device())
target = attn(hidden_states, context, mask)
hidden_states_parallel = split_seq(hidden_states, sp_size, sp_rank)
- context_parallel = split_seq(context, sp_size, sp_rank)
+ if cross_attn:
+ context_parallel = split_seq(context, sp_size, sp_rank)
+ else:
+ context_parallel = None
output_parallel = parallel_attn(
hidden_states_parallel,
context_parallel,
@@ -49,8 +75,8 @@ def check_sp_attn():
p.grad.data.div_(sp_size)
dist.all_reduce(p.grad.data)
- for p1, p2 in zip(attn.parameters(), parallel_attn.parameters()):
- assert_close(p1.grad, p2.grad)
+ for (n, p1), p2 in zip(attn.named_parameters(), parallel_attn.parameters()):
+ assert_close(p1.grad, p2.grad, msg=lambda m: f"Check {n}\n{m}")
def run_dist(rank, world_size, port):
diff --git a/train.py b/train.py
index 1f4da90..7ddd738 100644
--- a/train.py
+++ b/train.py
@@ -27,6 +27,7 @@ from tqdm import tqdm
from open_sora.diffusion import create_diffusion
from open_sora.modeling import DiT_models
+from open_sora.modeling.dit import SUPPORTED_SEQ_PARALLEL_MODES
from open_sora.utils.data import (
create_video_compressor,
load_datasets,
@@ -111,6 +112,8 @@ def main(args):
model_kwargs = {
"in_channels": video_compressor.out_channels,
"seq_parallel_group": plugin.sp_group,
+ "seq_parallel_mode": args.sp_mode,
+ "seq_parallel_overlap": args.sp_overlap,
}
# Step 4: Create DiT and EMA
@@ -247,6 +250,10 @@ if __name__ == "__main__":
parser.add_argument("-g", "--grad_checkpoint", action="store_true", default=False)
parser.add_argument("-a", "--accumulation_steps", default=1, type=int)
parser.add_argument("--sp_size", type=int, default=1)
+ parser.add_argument(
+ "--sp_mode", type=str, default="ulysses", choices=SUPPORTED_SEQ_PARALLEL_MODES
+ )
+ parser.add_argument("--sp_overlap", action="store_true", default=False)
parser.add_argument("--lr", type=float, default=1e-4)
parser.add_argument("--save_interval", type=int, default=20)
parser.add_argument("--checkpoint_dir", type=str, default="checkpoints")