Open-Sora: Democratizing Efficient Video Production for All
Python
[feature] impl ulysses-style seq parallel (#20)
Hongxin Liu97c089daec48
diff --git a/benchmark.py b/benchmark.pyindex 3244707..dbe9855 100644--- a/benchmark.py+++ b/benchmark.py@@ -14,7 +14,7 @@ import torchfrom colossalai import launch_from_torchfrom colossalai.accelerator import get_acceleratorfrom colossalai.booster import Booster-from colossalai.booster.plugin import LowLevelZeroPlugin, TorchDDPPlugin+from colossalai.booster.plugin import TorchDDPPluginfrom colossalai.cluster import DistCoordinatorfrom colossalai.logging import get_dist_loggerfrom colossalai.nn.optimizer import HybridAdam@@ -24,6 +24,7 @@ from tqdm import tqdmfrom open_sora.diffusion import create_diffusionfrom open_sora.modeling import DiT_modelsfrom open_sora.utils.data import create_video_compressor, preprocess_batch+from open_sora.utils.plugin import ZeroSeqParallelPlugin################################################################################## Training Loop #@@ -44,14 +45,17 @@ def main(args):plugin = TorchDDPPlugin()elif args.plugin == "zero2":# use bf16 to avoid skipping the first few iterations due to NaNs- plugin = LowLevelZeroPlugin(stage=2, precision="bf16")+ plugin = ZeroSeqParallelPlugin(sp_size=args.sp_size, stage=2, precision="bf16")else:raise ValueError(f"Unknown plugin {args.plugin}")booster = Booster(plugin=plugin)# Create video compressorvideo_compressor = create_video_compressor(args.compressor)- model_kwargs = {"in_channels": video_compressor.out_channels}+ model_kwargs = {+ "in_channels": video_compressor.out_channels,+ "seq_parallel_group": plugin.sp_group,+ }# Create DiT and EMAmodel = DiT_models[args.model](**model_kwargs).to(get_current_device())@@ -75,6 +79,7 @@ def main(args):torch.randn(args.num_frames, args.height, args.width, 3)for _ in range(args.batch_size)]+ assert args.num_tokens % args.sp_size == 0input_ids = torch.randn(args.batch_size, args.num_tokens, args.text_embed_dim)text_mask = torch.ones(input_ids.shape[:2], dtype=torch.int)batch = {@@ -82,9 +87,15 @@ def main(args):"text_latent_states": input_ids,"text_padding_mask": text_mask,}- batch = preprocess_batch(batch, patch_size, video_compressor)+ batch = preprocess_batch(+ batch, patch_size, video_compressor, pad_to_multiple=args.sp_size+ )video_inputs = batch.pop("video_latent_states")mask = batch.pop("video_padding_mask")+ logger.info(+ f"Num patches: {video_inputs.shape[1]}, num_tokens: {batch['text_latent_states'].shape[1]}",+ ranks=[0],+ )# setup boostermodel, opt, *_ = booster.boost(model, opt)@@ -125,7 +136,7 @@ def main(args):ranks=[0],)logger.info(- f"Throughput: {throughput:.2f} samples/s",+ f"Throughput per device: {throughput:.2f} samples/s",ranks=[0],)@@ -139,6 +150,7 @@ if __name__ == "__main__":parser.add_argument("-p", "--plugin", type=str, default="zero2", choices=["ddp", "zero2"])+ parser.add_argument("--sp_size", type=int, default=1)parser.add_argument("-w", "--warmup_steps", type=int, default=2)parser.add_argument("-s", "--steps", type=int, default=3)parser.add_argument("-b", "--batch_size", type=int, default=4)diff --git a/open_sora/modeling/dit.py b/open_sora/modeling/dit.pyindex 5710abd..ac9ce8a 100644--- a/open_sora/modeling/dit.py+++ b/open_sora/modeling/dit.py@@ -14,10 +14,13 @@ from typing import Callable, Optionalimport numpy as npimport torch+import torch.distributed as distimport torch.nn as nnimport torch.nn.functional as Ffrom timm.models.vision_transformer import Mlp+from open_sora.utils.comm import all_to_all, gather_seq, split_seq+class CrossAttention(nn.Module):r"""@@ -101,6 +104,120 @@ class CrossAttention(nn.Module):return attn_output+class SeqParallelCrossAttention(nn.Module):+ r"""+ A cross attention layer.++ Parameters:+ query_dim (`int`): The number of channels in the query.+ cross_attention_dim (`int`, *optional*):+ The number of channels in the context. If not given, defaults to `query_dim`.+ num_heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.+ head_dim (`int`, *optional*, defaults to 64): The number of channels in each head.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ bias (`bool`, *optional*, defaults to False):+ Set to `True` for the query, key, and value linear layers to contain a bias parameter.+ """++ def __init__(+ self,+ query_dim: int,+ cross_attention_dim: Optional[int] = None,+ num_heads: int = 8,+ head_dim: int = 64,+ dropout: float = 0.0,+ bias=False,+ sdpa=True,+ seq_parallel_group=None,+ ):+ super().__init__()+ self.hidden_size = head_dim * num_heads+ cross_attention_dim = (+ cross_attention_dim if cross_attention_dim is not None else query_dim+ )++ self.scale = head_dim**-0.5+ self.num_heads = num_heads+ self.head_dim = head_dim+ self.sdpa = sdpa+ self.seq_parallel_group = seq_parallel_group+ self.seq_parallel_size = (+ dist.get_world_size(self.seq_parallel_group)+ if seq_parallel_group is not None+ else 1+ )+ assert self.num_heads % self.seq_parallel_size == 0++ self.to_q = nn.Linear(query_dim, self.hidden_size, bias=bias)+ self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)+ self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)++ self.to_out = nn.Sequential(+ nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)+ )++ def forward(self, hidden_states, context=None, mask=None):+ bsz, q_len, _ = hidden_states.shape++ query = self.to_q(hidden_states)+ context = context if context is not None else hidden_states+ kv_seq_len = context.shape[1]+ key = self.to_k(context)+ value = self.to_v(context)++ # [B, S/P, H] -> [B, S, H/P]+ num_heads_parallel = self.num_heads // self.seq_parallel_size+ hidden_size_parallel = self.hidden_size // self.seq_parallel_size+ if self.seq_parallel_group is not None and self.seq_parallel_size > 1:+ query = all_to_all(+ query, self.seq_parallel_group, scatter_dim=2, gather_dim=1+ )+ key = all_to_all(key, self.seq_parallel_group, scatter_dim=2, gather_dim=1)+ value = all_to_all(+ value, self.seq_parallel_group, scatter_dim=2, gather_dim=1+ )++ q_len *= self.seq_parallel_size+ kv_seq_len *= self.seq_parallel_size++ # [B, S, H/P] -> [B, S, N/P, D] -> [B, N/P, S, D]+ query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(+ 1, 2+ )+ key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(+ 1, 2+ )+ value = value.view(+ bsz, kv_seq_len, num_heads_parallel, self.head_dim+ ).transpose(1, 2)++ if mask is not None:+ assert mask.shape == (bsz, 1, q_len, kv_seq_len)+ if self.sdpa:+ attn_output = F.scaled_dot_product_attention(+ query, key, value, attn_mask=mask, scale=self.scale+ )+ else:+ attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale+ assert attn_weights.shape == (bsz, num_heads_parallel, q_len, kv_seq_len)+ if mask is not None:+ attn_weights = attn_weights + mask+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(+ query.dtype+ )+ attn_output = torch.matmul(attn_weights, value)+ assert attn_output.shape == (bsz, num_heads_parallel, q_len, self.head_dim)+ attn_output = attn_output.transpose(1, 2).contiguous()+ attn_output = attn_output.reshape(bsz, q_len, hidden_size_parallel)+ # [B, S, H/P] -> [B, S/P, H]+ if self.seq_parallel_group is not None and self.seq_parallel_size > 1:+ attn_output = all_to_all(+ attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2+ )+ attn_output = self.to_out(attn_output)+ return attn_output++def modulate(x, shift, scale):return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)@@ -290,16 +407,18 @@ class DiTBlock(nn.Module):num_heads,cross_attention_dim=None,mlp_ratio=4.0,+ seq_parallel_group=None,):super().__init__()self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)- self.attn = CrossAttention(+ self.attn = SeqParallelCrossAttention(query_dim=hidden_size,cross_attention_dim=cross_attention_dim,num_heads=num_heads,head_dim=hidden_size // num_heads,bias=True,sdpa=True,+ seq_parallel_group=seq_parallel_group,)self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)@@ -380,6 +499,7 @@ class DiT(nn.Module):text_dropout_prob=0.1,learn_sigma=True,use_cross_attn=True,+ seq_parallel_group=None,):super().__init__()self.grad_checkpointing = False@@ -388,6 +508,17 @@ class DiT(nn.Module):self.out_channels = in_channels * 2 if learn_sigma else in_channelsself.patch_size = patch_sizeself.num_heads = num_heads+ self.seq_parallel_group = seq_parallel_group+ self.seq_parallel_size = (+ dist.get_world_size(self.seq_parallel_group)+ if seq_parallel_group is not None+ else 1+ )+ self.seq_parallel_rank = (+ dist.get_rank(self.seq_parallel_group)+ if seq_parallel_group is not None+ else 0+ )self.video_embedder = PatchEmbedder(patch_size, in_channels, hidden_size, bias=True@@ -409,7 +540,13 @@ class DiT(nn.Module):self.blocks = nn.ModuleList([- DiTBlock(hidden_size, num_heads, cross_attn_dim, mlp_ratio=mlp_ratio)+ DiTBlock(+ hidden_size,+ num_heads,+ cross_attn_dim,+ mlp_ratio=mlp_ratio,+ seq_parallel_group=seq_parallel_group,+ )for _ in range(depth)])@@ -483,6 +620,18 @@ class DiT(nn.Module):video_latent_states = video_latent_states + pos_embedt = self.t_embedder(t) # (N, D)attention_mask = self._prepare_mask(attention_mask, video_latent_states.dtype)++ if self.seq_parallel_group is not None and self.seq_parallel_size > 1:+ assert video_latent_states.shape[1] % self.seq_parallel_size == 0+ video_latent_states = split_seq(+ video_latent_states, self.seq_parallel_size, self.seq_parallel_rank+ )+ if text_latent_states is not None:+ assert text_latent_states.shape[1] % self.seq_parallel_size == 0+ text_latent_states = split_seq(+ text_latent_states, self.seq_parallel_size, self.seq_parallel_rank+ )+for block in self.blocks:if self.grad_checkpointing and self.training:video_latent_states = torch.utils.checkpoint.checkpoint(@@ -496,6 +645,15 @@ class DiT(nn.Module):video_latent_states = block(video_latent_states, attention_mask, t, text_latent_states)++ if self.seq_parallel_group is not None and self.seq_parallel_size > 1:+ video_latent_states = gather_seq(+ video_latent_states,+ self.seq_parallel_size,+ self.seq_parallel_rank,+ self.seq_parallel_group,+ )+if not self.use_cross_attn:video_latent_states = video_latent_states[:, text_len:]video_latent_states = self.final_layer(video_latent_states, t)diff --git a/open_sora/utils/comm.py b/open_sora/utils/comm.pynew file mode 100644index 0000000..00703d3--- /dev/null+++ b/open_sora/utils/comm.py@@ -0,0 +1,96 @@+import torch+import torch.distributed as dist+from colossalai.moe._operation import MoeInGradScaler, MoeOutGradScaler+from colossalai.shardformer.layer._operation import gather_forward_split_backward+++def _all_to_all(+ input_: torch.Tensor,+ world_size: int,+ group: dist.ProcessGroup,+ scatter_dim: int,+ gather_dim: int,+):+ input_list = [+ t.contiguous() for t in torch.tensor_split(input_, world_size, scatter_dim)+ ]+ output_list = [torch.empty_like(input_list[0]) for _ in range(world_size)]+ dist.all_to_all(output_list, input_list, group=group)+ return torch.cat(output_list, dim=gather_dim).contiguous()+++class _AllToAll(torch.autograd.Function):+ """All-to-all communication.++ Args:+ input_: input matrix+ process_group: communication group+ scatter_dim: scatter dimension+ gather_dim: gather dimension+ """++ @staticmethod+ def forward(ctx, input_, process_group, scatter_dim, gather_dim):+ ctx.process_group = process_group+ ctx.scatter_dim = scatter_dim+ ctx.gather_dim = gather_dim+ ctx.world_size = dist.get_world_size(process_group)+ return _all_to_all(+ input_, ctx.world_size, process_group, scatter_dim, gather_dim+ )++ @staticmethod+ def backward(ctx, grad_output):+ return (+ _all_to_all(+ grad_output,+ ctx.world_size,+ ctx.process_group,+ ctx.gather_dim,+ ctx.scatter_dim,+ ),+ None,+ None,+ None,+ )+++def all_to_all(+ input_: torch.Tensor,+ process_group: dist.ProcessGroup,+ scatter_dim: int = 2,+ gather_dim: int = 1,+):+ return _AllToAll.apply(input_, process_group, scatter_dim, gather_dim)+++def split_seq(input_: torch.Tensor, sp_size: int, sp_rank: int, dim: int = 1):+ """Split a tensor along sequence dimension. It will split input and divide grad by sp_size.++ Args:+ input_ (torch.Tensor): The common shape is (bs, seq, *).+ sp_size (int): Sequence parallel size.+ sp_rank (int): Sequence parallel rank.+ dim (int, optional): Sequence dimension. Defaults to 1.+ """+ input_ = input_.chunk(sp_size, dim=dim)[sp_rank].clone()+ return MoeOutGradScaler.apply(input_, sp_size)+++def gather_seq(+ input_: torch.Tensor,+ sp_size: int,+ sp_rank: int,+ sp_group: dist.ProcessGroup,+ dim: int = 1,+):+ """Gather a tensor along sequence dimension. It will gather input and multiply grad by sp_size.++ Args:+ input_ (torch.Tensor): The common shape is (bs, seq, *).+ sp_size (int): Sequence parallel size.+ sp_rank (int): Sequence parallel rank.+ dim (int, optional): Sequence dimension. Defaults to 1.+ """+ input_ = gather_forward_split_backward(input_, dim, sp_group)+ return MoeInGradScaler.apply(input_, sp_size)diff --git a/open_sora/utils/data.py b/open_sora/utils/data.pyindex d9078ec..99ad63c 100644--- a/open_sora/utils/data.py+++ b/open_sora/utils/data.py@@ -16,6 +16,13 @@ DatasetType = Union[Dataset, ConcatDataset, dataset_dict.Dataset]PathType = Union[str, os.PathLike]+def ceil_to_multiple(x: int, multiple: int) -> int:+ m = x % multiple+ if m == 0:+ return x+ return x + multiple - m++def video2col(video_4d: torch.Tensor, patch_size: int) -> torch.Tensor:"""Convert a 4D video tensor to a 2D tensor where each row is a patch of the video.@@ -67,7 +74,9 @@ def col2video(return video-def pad_sequences(sequences: List[torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor]:+def pad_sequences(+ sequences: List[torch.Tensor], pad_to_multiple: Optional[int] = None+) -> Tuple[torch.Tensor, torch.Tensor]:"""Pad a list of sequences.Args:@@ -77,6 +86,8 @@ def pad_sequences(sequences: List[torch.Tensor]) -> Tuple[torch.Tensor, torch.TeTuple[torch.Tensor, torch.Tensor]: Padded batch of sequences ([B, T, ...]) and padding mask ([B, T])."""max_len = max([sequence.shape[0] for sequence in sequences])+ if pad_to_multiple is not None:+ max_len = ceil_to_multiple(max_len, pad_to_multiple)padded_sequences = [F.pad(sequence, [0] * (sequence.ndim - 1) * 2 + [0, max_len - sequence.shape[0]]@@ -96,7 +107,7 @@ def pad_sequences(sequences: List[torch.Tensor]) -> Tuple[torch.Tensor, torch.Tedef patchify_batch(- videos: List[torch.Tensor], patch_size: int+ videos: List[torch.Tensor], patch_size: int, pad_to_multiple: Optional[int] = None) -> Tuple[torch.Tensor, torch.Tensor]:"""Patchify a batch of videos.@@ -108,7 +119,7 @@ def patchify_batch(Tuple[torch.Tensor, torch.Tensor]: Padded batch of patches ([B, S, C, P, P]) and padding mask ([B, S])."""video_patches = [video2col(video, patch_size) for video in videos]- return pad_sequences(video_patches)+ return pad_sequences(video_patches, pad_to_multiple=pad_to_multiple)def expand_mask_4d(q_mask: torch.Tensor, kv_mask: torch.Tensor) -> torch.Tensor:@@ -127,7 +138,9 @@ def expand_mask_4d(q_mask: torch.Tensor, kv_mask: torch.Tensor) -> torch.Tensor:return mask.unsqueeze(1)-def make_batch(samples: List[dict], video_dir: str) -> dict:+def make_batch(+ samples: List[dict], video_dir: str, pad_to_multiple: Optional[int] = None+) -> dict:"""Make a batch of samples.Args:@@ -141,7 +154,7 @@ def make_batch(samples: List[dict], video_dir: str) -> dict:for sample in samples]texts = [sample["text_latent_states"] for sample in samples]- texts, text_padding_mask = pad_sequences(texts)+ texts, text_padding_mask = pad_sequences(texts, pad_to_multiple=pad_to_multiple)return {"videos": videos,"text_latent_states": texts,@@ -269,6 +282,7 @@ def preprocess_batch(video_compressor: VideoCompressor,device=None,use_cross_attn=True,+ pad_to_multiple: Optional[int] = None,) -> dict:if device is None:device = get_current_device()@@ -278,7 +292,9 @@ def preprocess_batch(video = normalize_video(video)video = video_compressor.encode(video)videos.append(video)- video_latent_states, video_padding_mask = patchify_batch(videos, patch_size)+ video_latent_states, video_padding_mask = patchify_batch(+ videos, patch_size, pad_to_multiple+ )batch["video_latent_states"] = video_latent_statesbatch["video_padding_mask"] = video_padding_masktext_padding_mask = batch.pop("text_padding_mask").to(device)diff --git a/open_sora/utils/plugin.py b/open_sora/utils/plugin.pynew file mode 100644index 0000000..f687b41--- /dev/null+++ b/open_sora/utils/plugin.py@@ -0,0 +1,102 @@+import random+from typing import Optional++import numpy as np+import torch+from colossalai.booster.plugin import LowLevelZeroPlugin+from colossalai.cluster import ProcessGroupMesh+from torch.utils.data import DataLoader+from torch.utils.data.distributed import DistributedSampler++DP_AXIS, SP_AXIS = 0, 1+++class ZeroSeqParallelPlugin(LowLevelZeroPlugin):+ def __init__(+ self,+ sp_size: int = 1,+ stage: int = 2,+ precision: str = "fp16",+ initial_scale: float = 2**32,+ min_scale: float = 1,+ growth_factor: float = 2,+ backoff_factor: float = 0.5,+ growth_interval: int = 1000,+ hysteresis: int = 2,+ max_scale: float = 2**32,+ max_norm: float = 0.0,+ norm_type: float = 2.0,+ reduce_bucket_size_in_m: int = 12,+ communication_dtype: Optional[torch.dtype] = None,+ overlap_communication: bool = True,+ cpu_offload: bool = False,+ master_weights: bool = True,+ verbose: bool = False,+ ) -> None:+ super().__init__(+ stage=stage,+ precision=precision,+ initial_scale=initial_scale,+ min_scale=min_scale,+ growth_factor=growth_factor,+ backoff_factor=backoff_factor,+ growth_interval=growth_interval,+ hysteresis=hysteresis,+ max_scale=max_scale,+ max_norm=max_norm,+ norm_type=norm_type,+ reduce_bucket_size_in_m=reduce_bucket_size_in_m,+ communication_dtype=communication_dtype,+ overlap_communication=overlap_communication,+ cpu_offload=cpu_offload,+ master_weights=master_weights,+ verbose=verbose,+ )+ self.sp_size = sp_size+ assert self.world_size % sp_size == 0, "world_size must be divisible by sp_size"+ self.dp_size = self.world_size // sp_size+ self.pg_mesh = ProcessGroupMesh(self.dp_size, self.sp_size)+ self.dp_group = self.pg_mesh.get_group_along_axis(DP_AXIS)+ self.sp_group = self.pg_mesh.get_group_along_axis(SP_AXIS)+ self.dp_rank = self.pg_mesh.coordinate(DP_AXIS)+ self.sp_rank = self.pg_mesh.coordinate(SP_AXIS)++ def __del__(self):+ """Destroy the prcess groups in ProcessGroupMesh"""+ self.pg_mesh.destroy_mesh_process_groups()++ def prepare_dataloader(+ self,+ dataset,+ batch_size,+ shuffle=False,+ seed=1024,+ drop_last=False,+ pin_memory=False,+ num_workers=0,+ distributed_sampler_cls=None,+ **kwargs+ ):+ _kwargs = kwargs.copy()+ distributed_sampler_cls = distributed_sampler_cls or DistributedSampler+ sampler = distributed_sampler_cls(+ dataset, num_replicas=self.dp_size, rank=self.dp_rank, shuffle=shuffle+ )++ # Deterministic dataloader+ def seed_worker(worker_id):+ worker_seed = seed+ np.random.seed(worker_seed)+ torch.manual_seed(worker_seed)+ random.seed(worker_seed)++ return DataLoader(+ dataset,+ batch_size=batch_size,+ sampler=sampler,+ worker_init_fn=seed_worker,+ drop_last=drop_last,+ pin_memory=pin_memory,+ num_workers=num_workers,+ **_kwargs,+ )diff --git a/tests/test_model.py b/tests/test_model.pynew file mode 100644index 0000000..62b3172--- /dev/null+++ b/tests/test_model.py@@ -0,0 +1,93 @@+import colossalai+import pytest+import torch+import torch.distributed as dist+import torch.nn.functional as F+from colossalai.booster import Booster+from colossalai.logging import disable_existing_loggers+from colossalai.nn.optimizer import HybridAdam+from colossalai.testing import parameterize, rerun_if_address_is_in_use, spawn+from colossalai.utils import get_current_device+from torch.testing import assert_close++from open_sora.modeling import DiT_models+from open_sora.utils.plugin import ZeroSeqParallelPlugin+++@parameterize("sp_size", [2, 4])+def check_dit_model_fwd_bwd(+ sp_size: int, video_latent_states, text_latent_states, t, mask+):+ plugin = ZeroSeqParallelPlugin(+ sp_size=sp_size, stage=2, precision="fp32", master_weights=False+ )+ booster = Booster(plugin=plugin)+ model = DiT_models["DiT-B/8"](text_dropout_prob=0.0).to(get_current_device())+ parallel_model = DiT_models["DiT-B/8"](+ text_dropout_prob=0.0, seq_parallel_group=plugin.sp_group+ ).to(get_current_device())+ parallel_model.load_state_dict(model.state_dict())+ opt = HybridAdam(parallel_model.parameters(), lr=1e-3)+ parallel_model, opt, *_ = booster.boost(parallel_model, opt)++ target = model(video_latent_states, t, text_latent_states, mask)+ noise = torch.randn_like(target)+ target_loss = F.mse_loss(target, noise)+ target_loss.backward()++ dp_video_latent_states = video_latent_states.chunk(plugin.dp_size)[plugin.dp_rank]+ dp_text_latent_states = text_latent_states.chunk(plugin.dp_size)[plugin.dp_rank]+ dp_t = t.chunk(plugin.dp_size)[plugin.dp_rank]+ dp_mask = mask.chunk(plugin.dp_size)[plugin.dp_rank]+ dp_noise = noise.chunk(plugin.dp_size)[plugin.dp_rank]++ output = parallel_model(+ dp_video_latent_states, dp_t, dp_text_latent_states, dp_mask+ )+ loss = F.mse_loss(output, dp_noise)+ booster.backward(loss, opt)++ if plugin.dp_size == 1:+ assert_close(target, output)++ for p1, p2 in zip(model.parameters(), opt._master_param_groups_of_current_rank[0]):+ working_p = opt._param_store.master_to_working_param[id(p2)]+ grads = opt._grad_store.get_partitioned_gradients_by_param_id(0, id(working_p))+ grad_index = 0 if opt._partition_grads else opt._local_rank+ grad = grads[grad_index]+ sharded_grad = p1.grad.view(-1).chunk(dist.get_world_size())[dist.get_rank()]+ assert_close(sharded_grad, grad[: sharded_grad.shape[0]])+++def run_dist(rank, world_size, port):+ disable_existing_loggers()+ colossalai.launch(+ config={},+ rank=rank,+ world_size=world_size,+ port=port,+ host="localhost",+ backend="nccl",+ )+ b, s, c, p = 4, 20, 3, 8+ dim_text, s_text = 512, 12+ video_latent_states = torch.rand(b, s, c, p, p, device=get_current_device())+ text_latent_states = torch.rand(b, s_text, dim_text, device=get_current_device())+ t = torch.randint(0, 1000, (b,), device=get_current_device())+ mask = torch.ones(b, 1, s, s_text, device=get_current_device(), dtype=torch.int)+ check_dit_model_fwd_bwd(+ video_latent_states=video_latent_states,+ text_latent_states=text_latent_states,+ t=t,+ mask=mask,+ )+++@rerun_if_address_is_in_use()+def test_dit_model():+ spawn(run_dist, 4)+++if __name__ == "__main__":+ test_dit_model()diff --git a/tests/test_sp_attn.py b/tests/test_sp_attn.pynew file mode 100644index 0000000..58d3b7d--- /dev/null+++ b/tests/test_sp_attn.py@@ -0,0 +1,76 @@+import colossalai+import pytest+import torch+import torch.distributed as dist+from colossalai.logging import disable_existing_loggers+from colossalai.testing import rerun_if_address_is_in_use, spawn+from colossalai.utils import get_current_device+from torch.testing import assert_close++from open_sora.modeling.dit import CrossAttention, SeqParallelCrossAttention+from open_sora.utils.comm import gather_seq, split_seq+++def check_sp_attn():+ sp_size = dist.get_world_size()+ sp_rank = dist.get_rank()+ q_dim, context_dim = 8, 4+ num_heads = 4+ head_dim = 16+ bs = 2+ sq = 8+ skv = 4+ attn = CrossAttention(q_dim, context_dim, num_heads, head_dim).to(+ get_current_device()+ )+ parallel_attn = SeqParallelCrossAttention(+ q_dim, context_dim, num_heads, head_dim, seq_parallel_group=dist.group.WORLD+ ).to(get_current_device())+ parallel_attn.load_state_dict(attn.state_dict())+ hidden_states = torch.rand(bs, sq, q_dim, device=get_current_device())+ context = torch.rand(bs, skv, context_dim, device=get_current_device())+ mask = torch.zeros(bs, 1, sq, skv, device=get_current_device())+ target = attn(hidden_states, context, mask)+ hidden_states_parallel = split_seq(hidden_states, sp_size, sp_rank)+ context_parallel = split_seq(context, sp_size, sp_rank)+ output_parallel = parallel_attn(+ hidden_states_parallel,+ context_parallel,+ mask,+ )+ assert torch.equal(target.chunk(sp_size, dim=1)[sp_rank], output_parallel)+ output = gather_seq(output_parallel, sp_size, sp_rank, dist.group.WORLD)+ assert torch.equal(target, output)+ target.mean().backward()+ output.mean().backward()++ # all-reduce mean of grads+ for p in parallel_attn.parameters():+ p.grad.data.div_(sp_size)+ dist.all_reduce(p.grad.data)++ for p1, p2 in zip(attn.parameters(), parallel_attn.parameters()):+ assert_close(p1.grad, p2.grad)+++def run_dist(rank, world_size, port):+ disable_existing_loggers()+ colossalai.launch(+ config={},+ rank=rank,+ world_size=world_size,+ port=port,+ host="localhost",+ backend="nccl",+ )+ check_sp_attn()+++@rerun_if_address_is_in_use()+def test_seq_parallel_attn():+ spawn(run_dist, 2)+++if __name__ == "__main__":+ test_seq_parallel_attn()diff --git a/train.py b/train.pyindex 7e8093d..1f4da90 100644--- a/train.py+++ b/train.py@@ -17,7 +17,6 @@ import torch.distributed as distfrom colossalai import launch_from_torchfrom colossalai.accelerator import get_acceleratorfrom colossalai.booster import Booster-from colossalai.booster.plugin import LowLevelZeroPluginfrom colossalai.cluster import DistCoordinatorfrom colossalai.logging import get_dist_loggerfrom colossalai.nn.lr_scheduler import CosineAnnealingLR@@ -28,7 +27,13 @@ from tqdm import tqdmfrom open_sora.diffusion import create_diffusionfrom open_sora.modeling import DiT_models-from open_sora.utils.data import create_video_compressor, load_datasets, make_batch, preprocess_batch+from open_sora.utils.data import (+ create_video_compressor,+ load_datasets,+ make_batch,+ preprocess_batch,+)+from open_sora.utils.plugin import ZeroSeqParallelPlugin################################################################################## Training Helper Functions #@@ -93,7 +98,7 @@ def main(args):configure_backends()# Step 2: set up acceleration plugins- plugin = LowLevelZeroPlugin(stage=2, precision="fp16")+ plugin = ZeroSeqParallelPlugin(sp_size=args.sp_size, stage=2, precision="fp16")booster = Booster(plugin=plugin)if coordinator.is_master():@@ -103,7 +108,10 @@ def main(args):# Step 3: Create video compressorvideo_compressor = create_video_compressor(args.compressor)- model_kwargs = {"in_channels": video_compressor.out_channels}+ model_kwargs = {+ "in_channels": video_compressor.out_channels,+ "seq_parallel_group": plugin.sp_group,+ }# Step 4: Create DiT and EMAmodel = DiT_models[args.model](**model_kwargs).to(get_current_device())@@ -119,7 +127,9 @@ def main(args):model.enable_gradient_checkpointing()# Step 5: create diffusion pipeline- diffusion = create_diffusion(timestep_respacing="") # default: 1000 steps, linear noise schedule+ diffusion = create_diffusion(+ timestep_respacing=""+ ) # default: 1000 steps, linear noise schedule# Step 6: setup optimizer (we used default Adam betas=(0.9, 0.999) and a constant learning rate of 1e-4 in our paper):opt = HybridAdam(model.parameters(), lr=args.lr, weight_decay=0)@@ -129,15 +139,21 @@ def main(args):dataloader = plugin.prepare_dataloader(dataset,batch_size=args.batch_size,- collate_fn=partial(make_batch, video_dir=args.video_dir),+ collate_fn=partial(+ make_batch, video_dir=args.video_dir, pad_to_multiple=args.sp_size+ ),shuffle=True,drop_last=True,)- lr_scheduler = CosineAnnealingLR(opt, args.epochs * len(dataloader) // args.accumulation_steps)+ lr_scheduler = CosineAnnealingLR(+ opt, args.epochs * len(dataloader) // args.accumulation_steps+ )logger.info(f"Dataset contains {len(dataset)} samples", ranks=[0])# Step 8: setup booster- model, opt, _, dataloader, _ = booster.boost(model, opt, dataloader=dataloader)+ model, opt, _, dataloader, lr_scheduler = booster.boost(+ model, opt, dataloader=dataloader, lr_scheduler=lr_scheduler+ )if args.load_model is not None:booster.load_model(model, args.load_model)if args.load_optimizer is not None:@@ -159,7 +175,9 @@ def main(args):) as pbar:total_loss = torch.tensor(0.0, device=get_current_device())for step, batch in enumerate(dataloader):- batch = preprocess_batch(batch, patch_size, video_compressor)+ batch = preprocess_batch(+ batch, patch_size, video_compressor, pad_to_multiple=args.sp_size+ )video_inputs = batch.pop("video_latent_states")mask = batch.pop("video_padding_mask")t = torch.randint(@@ -168,7 +186,9 @@ def main(args):(video_inputs.shape[0],),device=video_inputs.device,)- loss_dict = diffusion.training_losses(model, video_inputs, t, batch, mask=mask)+ loss_dict = diffusion.training_losses(+ model, video_inputs, t, batch, mask=mask+ )loss = loss_dict["loss"].mean() / args.accumulation_stepstotal_loss.add_(loss.data)booster.backward(loss, opt)@@ -182,7 +202,9 @@ def main(args):all_reduce_mean(total_loss)pbar.set_postfix({"Loss": f"{total_loss.item():.4f}"})if coordinator.is_master():- global_step = (epoch * num_steps_per_epoch) + (step + 1) // args.accumulation_steps+ global_step = (epoch * num_steps_per_epoch) + (+ step + 1+ ) // args.accumulation_stepswriter.add_scalar(tag="Loss",scalar_value=total_loss.item(),@@ -193,9 +215,12 @@ def main(args):# Save DiT checkpoint:if args.save_interval > 0 and (- (step + 1) % (args.save_interval * args.accumulation_steps) == 0 or (step + 1) == len(dataloader)+ (step + 1) % (args.save_interval * args.accumulation_steps) == 0+ or (step + 1) == len(dataloader)):- save_path = os.path.join(args.checkpoint_dir, f"epoch-{epoch}-step-{step}")+ save_path = os.path.join(+ args.checkpoint_dir, f"epoch-{epoch}-step-{step}"+ )save_checkpoints(booster, model, opt, ema, save_path, coordinator)logger.info(f"Saved checkpoint to {save_path}", ranks=[0])@@ -212,18 +237,23 @@ def main(args):if __name__ == "__main__":# Default args here will train DiT-XL/2 with the hyperparameters we used in our paper (except training iters).parser = argparse.ArgumentParser()- parser.add_argument("-m", "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8")+ parser.add_argument(+ "-m", "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8"+ )parser.add_argument("-d", "--dataset", nargs="+", default=[])parser.add_argument("-v", "--video_dir", type=str, required=True)parser.add_argument("-e", "--epochs", type=int, default=10)parser.add_argument("-b", "--batch_size", type=int, default=4)parser.add_argument("-g", "--grad_checkpoint", action="store_true", default=False)parser.add_argument("-a", "--accumulation_steps", default=1, type=int)+ parser.add_argument("--sp_size", type=int, default=1)parser.add_argument("--lr", type=float, default=1e-4)parser.add_argument("--save_interval", type=int, default=20)parser.add_argument("--checkpoint_dir", type=str, default="checkpoints")parser.add_argument("--tensorboard_dir", type=str, default="runs")- parser.add_argument("-c", "--compressor", choices=["raw", "vqvae", "vae"], default="raw")+ parser.add_argument(+ "-c", "--compressor", choices=["raw", "vqvae", "vae"], default="raw"+ )parser.add_argument("--load_model", default=None)parser.add_argument("--load_optimizer", default=None)args = parser.parse_args()