Open-Sora: Democratizing Efficient Video Production for All
Python
change flashattn to flash_attn (#387)
Zheng Zangwei (Alex Zheng)a6a0325b470e
diff --git a/README.md b/README.mdindex 1a16f45..fa97f09 100644--- a/README.md+++ b/README.md@@ -161,7 +161,7 @@ conda activate opensorapip install torch torchvision# install flash attention (optional)-# set enable_flashattn=False in config to avoid using flash attention+# set enable_flash_attn=False in config to avoid using flash attentionpip install packaging ninjapip install flash-attn --no-build-isolationdiff --git a/configs/dit/train/16x256x256.py b/configs/dit/train/16x256x256.pyindex 42845de..c36e06b 100644--- a/configs/dit/train/16x256x256.py+++ b/configs/dit/train/16x256x256.py@@ -18,7 +18,7 @@ sp_size = 1model = dict(type="DiT-XL/2",from_pretrained="DiT-XL-2-256x256.pt",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/dit/train/1x256x256.py b/configs/dit/train/1x256x256.pyindex c423b24..aa5d478 100644--- a/configs/dit/train/1x256x256.py+++ b/configs/dit/train/1x256x256.py@@ -19,7 +19,7 @@ sp_size = 1model = dict(type="DiT-XL/2",no_temporal_pos_emb=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/latte/train/16x256x256.py b/configs/latte/train/16x256x256.pyindex 41573a0..e087f8a 100644--- a/configs/latte/train/16x256x256.py+++ b/configs/latte/train/16x256x256.py@@ -17,7 +17,7 @@ sp_size = 1# Define modelmodel = dict(type="Latte-XL/2",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.pyindex 5310b43..dff6ef1 100644--- a/configs/opensora-v1-1/train/benchmark.py+++ b/configs/opensora-v1-1/train/benchmark.py@@ -65,7 +65,7 @@ model = dict(from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/image.py b/configs/opensora-v1-1/train/image.pyindex 45748b7..ed25319 100644--- a/configs/opensora-v1-1/train/image.py+++ b/configs/opensora-v1-1/train/image.py@@ -29,7 +29,7 @@ model = dict(from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/stage1.py b/configs/opensora-v1-1/train/stage1.pyindex 944b565..d44429e 100644--- a/configs/opensora-v1-1/train/stage1.py+++ b/configs/opensora-v1-1/train/stage1.py@@ -41,7 +41,7 @@ model = dict(from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/stage2.py b/configs/opensora-v1-1/train/stage2.pyindex fb7e6d5..687a6c7 100644--- a/configs/opensora-v1-1/train/stage2.py+++ b/configs/opensora-v1-1/train/stage2.py@@ -43,7 +43,7 @@ model = dict(from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/stage3.py b/configs/opensora-v1-1/train/stage3.pyindex 8485762..624ea9e 100644--- a/configs/opensora-v1-1/train/stage3.py+++ b/configs/opensora-v1-1/train/stage3.py@@ -43,7 +43,7 @@ model = dict(from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/video.py b/configs/opensora-v1-1/train/video.pyindex ef574f2..1471664 100644--- a/configs/opensora-v1-1/train/video.py+++ b/configs/opensora-v1-1/train/video.py@@ -31,7 +31,7 @@ model = dict(from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora/inference/16x256x256.py b/configs/opensora/inference/16x256x256.pyindex 50ead83..4053e12 100644--- a/configs/opensora/inference/16x256x256.py+++ b/configs/opensora/inference/16x256x256.py@@ -7,7 +7,7 @@ model = dict(type="STDiT-XL/2",space_scale=0.5,time_scale=1.0,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,from_pretrained="PRETRAINED_MODEL",)diff --git a/configs/opensora/inference/16x512x512.py b/configs/opensora/inference/16x512x512.pyindex 58d8243..478cb5b 100644--- a/configs/opensora/inference/16x512x512.py+++ b/configs/opensora/inference/16x512x512.py@@ -7,7 +7,7 @@ model = dict(type="STDiT-XL/2",space_scale=1.0,time_scale=1.0,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,from_pretrained="PRETRAINED_MODEL",)diff --git a/configs/opensora/inference/64x512x512.py b/configs/opensora/inference/64x512x512.pyindex dbbe240..03cce23 100644--- a/configs/opensora/inference/64x512x512.py+++ b/configs/opensora/inference/64x512x512.py@@ -7,7 +7,7 @@ model = dict(type="STDiT-XL/2",space_scale=1.0,time_scale=2 / 3,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,from_pretrained="PRETRAINED_MODEL",)diff --git a/configs/opensora/train/16x256x256-mask.py b/configs/opensora/train/16x256x256-mask.pyindex 0e478e5..12b4b65 100644--- a/configs/opensora/train/16x256x256-mask.py+++ b/configs/opensora/train/16x256x256-mask.py@@ -20,7 +20,7 @@ model = dict(space_scale=0.5,time_scale=1.0,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)mask_ratios = {diff --git a/configs/opensora/train/16x256x256-spee.py b/configs/opensora/train/16x256x256-spee.pyindex b46c749..3af468b 100644--- a/configs/opensora/train/16x256x256-spee.py+++ b/configs/opensora/train/16x256x256-spee.py@@ -20,7 +20,7 @@ model = dict(space_scale=0.5,time_scale=1.0,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)mask_ratios = {diff --git a/configs/opensora/train/16x256x256.py b/configs/opensora/train/16x256x256.pyindex 144adee..21ebb26 100644--- a/configs/opensora/train/16x256x256.py+++ b/configs/opensora/train/16x256x256.py@@ -20,7 +20,7 @@ model = dict(space_scale=0.5,time_scale=1.0,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora/train/16x512x512.py b/configs/opensora/train/16x512x512.pyindex a2a87b1..c566fd1 100644--- a/configs/opensora/train/16x512x512.py+++ b/configs/opensora/train/16x512x512.py@@ -20,7 +20,7 @@ model = dict(space_scale=1.0,time_scale=1.0,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora/train/360x512x512.py b/configs/opensora/train/360x512x512.pyindex 7cb4f56..62bfd14 100644--- a/configs/opensora/train/360x512x512.py+++ b/configs/opensora/train/360x512x512.py@@ -26,7 +26,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,enable_sequence_parallelism=True, # enable sq here)diff --git a/configs/opensora/train/64x512x512-sp.py b/configs/opensora/train/64x512x512-sp.pyindex 880adb2..fd34a2a 100644--- a/configs/opensora/train/64x512x512-sp.py+++ b/configs/opensora/train/64x512x512-sp.py@@ -20,7 +20,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,enable_sequence_parallelism=True, # enable sq here)diff --git a/configs/opensora/train/64x512x512.py b/configs/opensora/train/64x512x512.pyindex b65a419..e07f8c1 100644--- a/configs/opensora/train/64x512x512.py+++ b/configs/opensora/train/64x512x512.py@@ -20,7 +20,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/pixart/train/16x256x256.py b/configs/pixart/train/16x256x256.pyindex 701b9da..66285bf 100644--- a/configs/pixart/train/16x256x256.py+++ b/configs/pixart/train/16x256x256.py@@ -20,7 +20,7 @@ model = dict(space_scale=0.5,time_scale=1.0,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/pixart/train/1x512x512.py b/configs/pixart/train/1x512x512.pyindex 8b2b5d4..0e7a1f7 100644--- a/configs/pixart/train/1x512x512.py+++ b/configs/pixart/train/1x512x512.py@@ -21,7 +21,7 @@ model = dict(time_scale=1.0,no_temporal_pos_emb=True,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/pixart/train/64x512x512.py b/configs/pixart/train/64x512x512.pyindex 3448ce7..dd59d11 100644--- a/configs/pixart/train/64x512x512.py+++ b/configs/pixart/train/64x512x512.py@@ -21,7 +21,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/docs/config.md b/docs/config.mdindex 5cecc73..3ecf726 100644--- a/docs/config.md+++ b/docs/config.md@@ -26,8 +26,8 @@ model = dict(from_pretrained="PRETRAINED_MODEL", # (Optional) Load from pretrained modelinput_sq_size=512, # Base spatial position embedding sizeqk_norm=True, # Normalize query and key in attention- enable_flashattn=True, # (Optional) Speed up training and inference with flash attention- # Turn enable_flashattn to False if you skip flashattn installation+ enable_flash_attn=True, # (Optional) Speed up training and inference with flash attention+ # Turn enable_flash_attn to False if you skip flashattn installationenable_layernorm_kernel=True, # (Optional) Speed up training and inference with fused kernel# Turn enable_layernorm_kernel to False if you skip apex installation)@@ -177,7 +177,7 @@ model = dict(from_pretrained=None, # Load from pretrained modelinput_sq_size=512, # Base spatial position embedding sizeqk_norm=True, # Normalize query and key in attention- enable_flashattn=True, # (Optional) Speed up training and inference with flash attention+ enable_flash_attn=True, # (Optional) Speed up training and inference with flash attentionenable_layernorm_kernel=True, # (Optional) Speed up training and inference with fused kernel)vae = dict(diff --git a/docs/zh_CN/structure.md b/docs/zh_CN/structure.mdindex 6e25d84..1e3460e 100644--- a/docs/zh_CN/structure.md+++ b/docs/zh_CN/structure.md@@ -89,7 +89,7 @@ model = dict(type="STDiT-XL/2", # Select model type (STDiT-XL/2, DiT-XL/2, etc.)space_scale=1.0, # (Optional) Space positional encoding scale (new height / old height)time_scale=2 / 3, # (Optional) Time positional encoding scale (new frame_interval / old frame_interval)- enable_flashattn=True, # (Optional) Speed up training and inference with flash attention+ enable_flash_attn=True, # (Optional) Speed up training and inference with flash attentionenable_layernorm_kernel=True, # (Optional) Speed up training and inference with fused kernelfrom_pretrained="PRETRAINED_MODEL", # (Optional) Load from pretrained modelno_temporal_pos_emb=True, # (Optional) Disable temporal positional encoding (for image)@@ -144,7 +144,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained="YOUR_PRETRAINED_MODEL",- enable_flashattn=True, # Enable flash attention+ enable_flash_attn=True, # Enable flash attentionenable_layernorm_kernel=True, # Enable layernorm kernel)vae = dict(diff --git a/opensora/models/dit/dit.py b/opensora/models/dit/dit.pyindex f264f8e..68a3eb7 100644--- a/opensora/models/dit/dit.py+++ b/opensora/models/dit/dit.py@@ -44,13 +44,13 @@ class DiTBlock(nn.Module):hidden_size,num_heads,mlp_ratio=4.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,):super().__init__()self.hidden_size = hidden_sizeself.num_heads = num_heads- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnmlp_hidden_dim = int(hidden_size * mlp_ratio)self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)@@ -58,7 +58,7 @@ class DiTBlock(nn.Module):hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,)self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)self.mlp = Mlp(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0)@@ -93,7 +93,7 @@ class DiT(nn.Module):caption_channels=512,model_max_length=77,dtype=torch.float32,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):@@ -111,7 +111,7 @@ class DiT(nn.Module):self.num_heads = num_headsself.dtype = dtypeself.use_text_encoder = not condition.startswith("label")- if enable_flashattn:+ if enable_flash_attn:assert dtype in [torch.float16,torch.bfloat16,@@ -143,7 +143,7 @@ class DiT(nn.Module):hidden_size,num_heads,mlp_ratio=mlp_ratio,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,enable_layernorm_kernel=enable_layernorm_kernel,)for _ in range(depth)@@ -152,7 +152,7 @@ class DiT(nn.Module):self.final_layer = FinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)self.initialize_weights()- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself.enable_layernorm_kernel = enable_layernorm_kerneldef get_spatial_pos_embed(self):diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.pyindex 421f836..376bd5b 100644--- a/opensora/models/pixart/pixart.py+++ b/opensora/models/pixart/pixart.py@@ -61,13 +61,13 @@ class PixArtBlock(nn.Module):num_heads,mlp_ratio=4.0,drop_path=0.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):super().__init__()self.hidden_size = hidden_size- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself._enable_sequence_parallelism = enable_sequence_parallelismif enable_sequence_parallelism:@@ -82,7 +82,7 @@ class PixArtBlock(nn.Module):hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,)self.cross_attn = self.mha_cls(hidden_size, num_heads)self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)@@ -130,7 +130,7 @@ class PixArt(nn.Module):freeze=None,space_scale=1.0,time_scale=1.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):@@ -152,7 +152,7 @@ class PixArt(nn.Module):self.no_temporal_pos_emb = no_temporal_pos_embself.depth = depthself.mlp_ratio = mlp_ratio- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself.enable_layernorm_kernel = enable_layernorm_kernelself.space_scale = space_scaleself.time_scale = time_scale@@ -179,7 +179,7 @@ class PixArt(nn.Module):num_heads,mlp_ratio=mlp_ratio,drop_path=drop_path[i],- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,enable_layernorm_kernel=enable_layernorm_kernel,)for i in range(depth)diff --git a/opensora/models/stdit/stdit.py b/opensora/models/stdit/stdit.pyindex 6e16058..6882572 100644--- a/opensora/models/stdit/stdit.py+++ b/opensora/models/stdit/stdit.py@@ -37,13 +37,13 @@ class STDiTBlock(nn.Module):d_t=None,mlp_ratio=4.0,drop_path=0.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):super().__init__()self.hidden_size = hidden_size- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself._enable_sequence_parallelism = enable_sequence_parallelismif enable_sequence_parallelism:@@ -58,7 +58,7 @@ class STDiTBlock(nn.Module):hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,)self.cross_attn = self.mha_cls(hidden_size, num_heads)self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)@@ -82,7 +82,7 @@ class STDiTBlock(nn.Module):hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=self.enable_flashattn,+ enable_flash_attn=self.enable_flash_attn,)def t_mask_select(self, x, masked_x, x_mask):@@ -174,7 +174,7 @@ class STDiT(nn.Module):space_scale=1.0,time_scale=1.0,freeze=None,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):@@ -194,7 +194,7 @@ class STDiT(nn.Module):self.no_temporal_pos_emb = no_temporal_pos_embself.depth = depthself.mlp_ratio = mlp_ratio- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself.enable_layernorm_kernel = enable_layernorm_kernelself.space_scale = space_scaleself.time_scale = time_scale@@ -221,7 +221,7 @@ class STDiT(nn.Module):self.num_heads,mlp_ratio=self.mlp_ratio,drop_path=drop_path[i],- enable_flashattn=self.enable_flashattn,+ enable_flash_attn=self.enable_flash_attn,enable_layernorm_kernel=self.enable_layernorm_kernel,enable_sequence_parallelism=enable_sequence_parallelism,d_t=self.num_temporal,diff --git a/tests/test_attn.py b/tests/test_attn.pyindex 48f8c43..4cf5844 100644--- a/tests/test_attn.py+++ b/tests/test_attn.py@@ -11,7 +11,7 @@ B, S, H = 128, 32, 1152N, D = 16, 72-def run_attn(enable_flashattn: bool):+def run_attn(enable_flash_attn: bool):get_accelerator().reset_peak_memory_stats()rope = RotaryEmbedding(D).to(device=get_current_device(), dtype=torch.bfloat16)attn = Attention(@@ -19,7 +19,7 @@ def run_attn(enable_flashattn: bool):N,qkv_bias=True,rope=rope.rotate_queries_or_keys,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,).to(device=get_current_device(), dtype=torch.bfloat16)x = torch.randn(B, S, H, device=get_current_device(), dtype=torch.bfloat16).requires_grad_()y = attn(x)diff --git a/tests/test_seq_parallel_attention.py b/tests/test_seq_parallel_attention.pyindex 9cce2b2..841115a 100644--- a/tests/test_seq_parallel_attention.py+++ b/tests/test_seq_parallel_attention.py@@ -18,14 +18,14 @@ def run_attention(rank, world_size):torch.manual_seed(1024)set_sequence_parallel_group(dist.group.WORLD)- seq_parallel_attention = SeqParallelAttention(dim=256, num_heads=4, qkv_bias=True, enable_flashattn=False).cuda()+ seq_parallel_attention = SeqParallelAttention(dim=256, num_heads=4, qkv_bias=True, enable_flash_attn=False).cuda()torch.manual_seed(1024)attention = Attention(dim=256,num_heads=4,qkv_bias=True,- enable_flashattn=False,+ enable_flash_attn=False,).cuda()# create inputs