Open-Sora: Democratizing Efficient Video Production for All
Python
[wip]
Zangwei Zheng917643d5bdcc
diff --git a/README.md b/README.mdindex 6116f8b..633cb8a 100644--- a/README.md+++ b/README.md@@ -161,7 +161,7 @@ conda activate opensorapip install torch torchvision# install flash attention (optional)-# set enable_flashattn=False in config to avoid using flash attention+# set enable_flash_attn=False in config to avoid using flash attentionpip install packaging ninjapip install flash-attn --no-build-isolationdiff --git a/configs/dit/train/16x256x256.py b/configs/dit/train/16x256x256.pyindex 42845de..c36e06b 100644--- a/configs/dit/train/16x256x256.py+++ b/configs/dit/train/16x256x256.py@@ -18,7 +18,7 @@ sp_size = 1model = dict(type="DiT-XL/2",from_pretrained="DiT-XL-2-256x256.pt",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/dit/train/1x256x256.py b/configs/dit/train/1x256x256.pyindex c423b24..aa5d478 100644--- a/configs/dit/train/1x256x256.py+++ b/configs/dit/train/1x256x256.py@@ -19,7 +19,7 @@ sp_size = 1model = dict(type="DiT-XL/2",no_temporal_pos_emb=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/latte/train/16x256x256.py b/configs/latte/train/16x256x256.pyindex 41573a0..e087f8a 100644--- a/configs/latte/train/16x256x256.py+++ b/configs/latte/train/16x256x256.py@@ -17,7 +17,7 @@ sp_size = 1# Define modelmodel = dict(type="Latte-XL/2",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.pyindex 0a6a01b..da6cc06 100644--- a/configs/opensora-v1-1/train/benchmark.py+++ b/configs/opensora-v1-1/train/benchmark.py@@ -66,7 +66,7 @@ model = dict(input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,qk_norm_legacy=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/image.py b/configs/opensora-v1-1/train/image.pyindex de104d9..ee43c2e 100644--- a/configs/opensora-v1-1/train/image.py+++ b/configs/opensora-v1-1/train/image.py@@ -30,7 +30,7 @@ model = dict(input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,qk_norm_legacy=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/image_rflow.py b/configs/opensora-v1-1/train/image_rflow.pyindex 66c7b56..08d52ef 100644--- a/configs/opensora-v1-1/train/image_rflow.py+++ b/configs/opensora-v1-1/train/image_rflow.py@@ -35,7 +35,7 @@ sp_size = 1# type="DiT-XL/2",# from_pretrained="/home/zhaowangbo/wangbo/PixArt-alpha/pretrained_models/PixArt-XL-2-512x512.pth",# # input_sq_size=512, # pretrained model is trained on 512x512-# enable_flashattn=True,+# enable_flash_attn=True,# enable_layernorm_kernel=True,# )model = dict(@@ -44,7 +44,7 @@ model = dict(time_scale=1.0,no_temporal_pos_emb=True,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)# model = dict(@@ -54,7 +54,7 @@ model = dict(# no_temporal_pos_emb=True,# # from_pretrained="PixArt-XL-2-512x512.pth",# from_pretrained="/home/zhaowangbo/wangbo/PixArt-alpha/pretrained_models/PixArt-XL-2-512x512.pth",-# enable_flashattn=True,+# enable_flash_attn=True,# enable_layernorm_kernel=True,# )vae = dict(diff --git a/configs/opensora-v1-1/train/stage1.py b/configs/opensora-v1-1/train/stage1.pyindex 8bd9b35..03905fe 100644--- a/configs/opensora-v1-1/train/stage1.py+++ b/configs/opensora-v1-1/train/stage1.py@@ -42,7 +42,7 @@ model = dict(input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,qk_norm_legacy=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/stage2.py b/configs/opensora-v1-1/train/stage2.pyindex 66572d5..ad66ed1 100644--- a/configs/opensora-v1-1/train/stage2.py+++ b/configs/opensora-v1-1/train/stage2.py@@ -44,7 +44,7 @@ model = dict(input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,qk_norm_legacy=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/stage3.py b/configs/opensora-v1-1/train/stage3.pyindex db4d42b..a774af3 100644--- a/configs/opensora-v1-1/train/stage3.py+++ b/configs/opensora-v1-1/train/stage3.py@@ -44,7 +44,7 @@ model = dict(input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,qk_norm_legacy=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-1/train/video.py b/configs/opensora-v1-1/train/video.pyindex 2912a40..0a068a5 100644--- a/configs/opensora-v1-1/train/video.py+++ b/configs/opensora-v1-1/train/video.py@@ -32,7 +32,7 @@ model = dict(input_sq_size=512, # pretrained model is trained on 512x512qk_norm=True,qk_norm_legacy=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-2/inference/sample-ref.py b/configs/opensora-v1-2/inference/sample-ref.pyindex e57e2ec..4d9dd4d 100644--- a/configs/opensora-v1-2/inference/sample-ref.py+++ b/configs/opensora-v1-2/inference/sample-ref.py@@ -22,7 +22,7 @@ model = dict(type="STDiT3-XL/2",from_pretrained=None,qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-2/inference/sample.py b/configs/opensora-v1-2/inference/sample.pyindex ae0b7ca..58306ce 100644--- a/configs/opensora-v1-2/inference/sample.py+++ b/configs/opensora-v1-2/inference/sample.py@@ -14,7 +14,7 @@ model = dict(type="STDiT3-XL/2",from_pretrained=None,qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-2/train/stage1-gc.py b/configs/opensora-v1-2/train/stage1-gc.pyindex 3b98f78..7e58a36 100644--- a/configs/opensora-v1-2/train/stage1-gc.py+++ b/configs/opensora-v1-2/train/stage1-gc.py@@ -36,7 +36,7 @@ model = dict(type="STDiT3-XL/2",from_pretrained=None,qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora-v1-2/train/stage1.py b/configs/opensora-v1-2/train/stage1.pyindex 2a05262..3bb43b9 100644--- a/configs/opensora-v1-2/train/stage1.py+++ b/configs/opensora-v1-2/train/stage1.py@@ -34,12 +34,12 @@ model = dict(type="STDiT3-XL/2",from_pretrained=None,qk_norm=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(type="VideoAutoencoderPipeline",- from_pretrained="pretrained_models/vae-v3",+ from_pretrained="pretrained_models/vae-v2",micro_frame_size=17,vae_2d=dict(type="VideoAutoencoderKL",diff --git a/configs/opensora/inference/16x256x256.py b/configs/opensora/inference/16x256x256.pyindex 50ead83..4053e12 100644--- a/configs/opensora/inference/16x256x256.py+++ b/configs/opensora/inference/16x256x256.py@@ -7,7 +7,7 @@ model = dict(type="STDiT-XL/2",space_scale=0.5,time_scale=1.0,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,from_pretrained="PRETRAINED_MODEL",)diff --git a/configs/opensora/inference/16x512x512-rflow.py b/configs/opensora/inference/16x512x512-rflow.pyindex 35804d5..cf23810 100644--- a/configs/opensora/inference/16x512x512-rflow.py+++ b/configs/opensora/inference/16x512x512-rflow.py@@ -7,7 +7,7 @@ model = dict(type="STDiT-XL/2",space_scale=1.0,time_scale=1.0,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,from_pretrained="PRETRAINED_MODEL",)diff --git a/configs/opensora/inference/16x512x512.py b/configs/opensora/inference/16x512x512.pyindex 58d8243..478cb5b 100644--- a/configs/opensora/inference/16x512x512.py+++ b/configs/opensora/inference/16x512x512.py@@ -7,7 +7,7 @@ model = dict(type="STDiT-XL/2",space_scale=1.0,time_scale=1.0,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,from_pretrained="PRETRAINED_MODEL",)diff --git a/configs/opensora/inference/64x512x512.py b/configs/opensora/inference/64x512x512.pyindex dbbe240..03cce23 100644--- a/configs/opensora/inference/64x512x512.py+++ b/configs/opensora/inference/64x512x512.py@@ -7,7 +7,7 @@ model = dict(type="STDiT-XL/2",space_scale=1.0,time_scale=2 / 3,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,from_pretrained="PRETRAINED_MODEL",)diff --git a/configs/opensora/train/16x256x256-mask.py b/configs/opensora/train/16x256x256-mask.pyindex 0e478e5..12b4b65 100644--- a/configs/opensora/train/16x256x256-mask.py+++ b/configs/opensora/train/16x256x256-mask.py@@ -20,7 +20,7 @@ model = dict(space_scale=0.5,time_scale=1.0,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)mask_ratios = {diff --git a/configs/opensora/train/16x256x256-spee-rflow.py b/configs/opensora/train/16x256x256-spee-rflow.pyindex cbb929c..809803b 100644--- a/configs/opensora/train/16x256x256-spee-rflow.py+++ b/configs/opensora/train/16x256x256-spee-rflow.py@@ -22,8 +22,8 @@ model = dict(# from_pretrained="PixArt-XL-2-512x512.pth",# from_pretrained = "/home/zhaowangbo/wangbo/PixArt-alpha/pretrained_models/OpenSora-v1-HQ-16x512x512.pth",# from_pretrained = "OpenSora-v1-HQ-16x512x512.pth",- from_pretrained = "PRETRAINED_MODEL",- enable_flashattn=True,+ from_pretrained="PRETRAINED_MODEL",+ enable_flash_attn=True,enable_layernorm_kernel=True,)# mask_ratios = [0.5, 0.29, 0.07, 0.07, 0.07]diff --git a/configs/opensora/train/16x256x256-spee.py b/configs/opensora/train/16x256x256-spee.pyindex b46c749..3af468b 100644--- a/configs/opensora/train/16x256x256-spee.py+++ b/configs/opensora/train/16x256x256-spee.py@@ -20,7 +20,7 @@ model = dict(space_scale=0.5,time_scale=1.0,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)mask_ratios = {diff --git a/configs/opensora/train/16x256x256.py b/configs/opensora/train/16x256x256.pyindex 144adee..21ebb26 100644--- a/configs/opensora/train/16x256x256.py+++ b/configs/opensora/train/16x256x256.py@@ -20,7 +20,7 @@ model = dict(space_scale=0.5,time_scale=1.0,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora/train/16x512x512.py b/configs/opensora/train/16x512x512.pyindex a2a87b1..c566fd1 100644--- a/configs/opensora/train/16x512x512.py+++ b/configs/opensora/train/16x512x512.py@@ -20,7 +20,7 @@ model = dict(space_scale=1.0,time_scale=1.0,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/opensora/train/360x512x512.py b/configs/opensora/train/360x512x512.pyindex 7cb4f56..62bfd14 100644--- a/configs/opensora/train/360x512x512.py+++ b/configs/opensora/train/360x512x512.py@@ -26,7 +26,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,enable_sequence_parallelism=True, # enable sq here)diff --git a/configs/opensora/train/64x512x512-sp.py b/configs/opensora/train/64x512x512-sp.pyindex 880adb2..fd34a2a 100644--- a/configs/opensora/train/64x512x512-sp.py+++ b/configs/opensora/train/64x512x512-sp.py@@ -20,7 +20,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,enable_sequence_parallelism=True, # enable sq here)diff --git a/configs/opensora/train/64x512x512.py b/configs/opensora/train/64x512x512.pyindex b65a419..e07f8c1 100644--- a/configs/opensora/train/64x512x512.py+++ b/configs/opensora/train/64x512x512.py@@ -20,7 +20,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/pixart/inference/1x20481B.py b/configs/pixart/inference/1x20481B.pyindex f3bfe8c..3d5eba9 100644--- a/configs/pixart/inference/1x20481B.py+++ b/configs/pixart/inference/1x20481B.py@@ -8,7 +8,7 @@ model = dict(from_pretrained="PixArt-1B-2.pth",space_scale=4,no_temporal_pos_emb=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,base_size=2048 // 8,)diff --git a/configs/pixart/inference/1x2048MS.py b/configs/pixart/inference/1x2048MS.pyindex 42a7fa3..a0daca4 100644--- a/configs/pixart/inference/1x2048MS.py+++ b/configs/pixart/inference/1x2048MS.py@@ -8,7 +8,7 @@ model = dict(from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",space_scale=4,no_temporal_pos_emb=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,base_size=2048 // 8,)diff --git a/configs/pixart/train/16x256x256.py b/configs/pixart/train/16x256x256.pyindex 701b9da..66285bf 100644--- a/configs/pixart/train/16x256x256.py+++ b/configs/pixart/train/16x256x256.py@@ -20,7 +20,7 @@ model = dict(space_scale=0.5,time_scale=1.0,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/pixart/train/1x2048x2048.py b/configs/pixart/train/1x2048x2048.pyindex fdd4f1d..31f1784 100644--- a/configs/pixart/train/1x2048x2048.py+++ b/configs/pixart/train/1x2048x2048.py@@ -20,7 +20,7 @@ model = dict(space_scale=4.0,no_temporal_pos_emb=True,from_pretrained="PixArt-1B-2.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)diff --git a/configs/pixart/train/1x512x512-rflow.py b/configs/pixart/train/1x512x512-rflow.pyindex 8925cdb..9551ce8 100644--- a/configs/pixart/train/1x512x512-rflow.py+++ b/configs/pixart/train/1x512x512-rflow.py@@ -21,8 +21,8 @@ model = dict(time_scale=1.0,no_temporal_pos_emb=True,# from_pretrained="PixArt-XL-2-512x512.pth",- from_pretrained = "PRETRAINED_MODEL",- enable_flashattn=True,+ from_pretrained="PRETRAINED_MODEL",+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/pixart/train/1x512x512.py b/configs/pixart/train/1x512x512.pyindex 8b2b5d4..0e7a1f7 100644--- a/configs/pixart/train/1x512x512.py+++ b/configs/pixart/train/1x512x512.py@@ -21,7 +21,7 @@ model = dict(time_scale=1.0,no_temporal_pos_emb=True,from_pretrained="PixArt-XL-2-512x512.pth",- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/configs/pixart/train/64x512x512.py b/configs/pixart/train/64x512x512.pyindex 3448ce7..dd59d11 100644--- a/configs/pixart/train/64x512x512.py+++ b/configs/pixart/train/64x512x512.py@@ -21,7 +21,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained=None,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --git a/docs/config.md b/docs/config.mdindex 5cecc73..3ecf726 100644--- a/docs/config.md+++ b/docs/config.md@@ -26,8 +26,8 @@ model = dict(from_pretrained="PRETRAINED_MODEL", # (Optional) Load from pretrained modelinput_sq_size=512, # Base spatial position embedding sizeqk_norm=True, # Normalize query and key in attention- enable_flashattn=True, # (Optional) Speed up training and inference with flash attention- # Turn enable_flashattn to False if you skip flashattn installation+ enable_flash_attn=True, # (Optional) Speed up training and inference with flash attention+ # Turn enable_flash_attn to False if you skip flashattn installationenable_layernorm_kernel=True, # (Optional) Speed up training and inference with fused kernel# Turn enable_layernorm_kernel to False if you skip apex installation)@@ -177,7 +177,7 @@ model = dict(from_pretrained=None, # Load from pretrained modelinput_sq_size=512, # Base spatial position embedding sizeqk_norm=True, # Normalize query and key in attention- enable_flashattn=True, # (Optional) Speed up training and inference with flash attention+ enable_flash_attn=True, # (Optional) Speed up training and inference with flash attentionenable_layernorm_kernel=True, # (Optional) Speed up training and inference with fused kernel)vae = dict(diff --git a/docs/zh_CN/structure.md b/docs/zh_CN/structure.mdindex 6e25d84..1e3460e 100644--- a/docs/zh_CN/structure.md+++ b/docs/zh_CN/structure.md@@ -89,7 +89,7 @@ model = dict(type="STDiT-XL/2", # Select model type (STDiT-XL/2, DiT-XL/2, etc.)space_scale=1.0, # (Optional) Space positional encoding scale (new height / old height)time_scale=2 / 3, # (Optional) Time positional encoding scale (new frame_interval / old frame_interval)- enable_flashattn=True, # (Optional) Speed up training and inference with flash attention+ enable_flash_attn=True, # (Optional) Speed up training and inference with flash attentionenable_layernorm_kernel=True, # (Optional) Speed up training and inference with fused kernelfrom_pretrained="PRETRAINED_MODEL", # (Optional) Load from pretrained modelno_temporal_pos_emb=True, # (Optional) Disable temporal positional encoding (for image)@@ -144,7 +144,7 @@ model = dict(space_scale=1.0,time_scale=2 / 3,from_pretrained="YOUR_PRETRAINED_MODEL",- enable_flashattn=True, # Enable flash attention+ enable_flash_attn=True, # Enable flash attentionenable_layernorm_kernel=True, # Enable layernorm kernel)vae = dict(diff --git a/opensora/models/dit/dit.py b/opensora/models/dit/dit.pyindex f264f8e..68a3eb7 100644--- a/opensora/models/dit/dit.py+++ b/opensora/models/dit/dit.py@@ -44,13 +44,13 @@ class DiTBlock(nn.Module):hidden_size,num_heads,mlp_ratio=4.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,):super().__init__()self.hidden_size = hidden_sizeself.num_heads = num_heads- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnmlp_hidden_dim = int(hidden_size * mlp_ratio)self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)@@ -58,7 +58,7 @@ class DiTBlock(nn.Module):hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,)self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)self.mlp = Mlp(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0)@@ -93,7 +93,7 @@ class DiT(nn.Module):caption_channels=512,model_max_length=77,dtype=torch.float32,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):@@ -111,7 +111,7 @@ class DiT(nn.Module):self.num_heads = num_headsself.dtype = dtypeself.use_text_encoder = not condition.startswith("label")- if enable_flashattn:+ if enable_flash_attn:assert dtype in [torch.float16,torch.bfloat16,@@ -143,7 +143,7 @@ class DiT(nn.Module):hidden_size,num_heads,mlp_ratio=mlp_ratio,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,enable_layernorm_kernel=enable_layernorm_kernel,)for _ in range(depth)@@ -152,7 +152,7 @@ class DiT(nn.Module):self.final_layer = FinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)self.initialize_weights()- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself.enable_layernorm_kernel = enable_layernorm_kerneldef get_spatial_pos_embed(self):diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.pyindex eb0b5b5..61c10f7 100644--- a/opensora/models/layers/blocks.py+++ b/opensora/models/layers/blocks.py@@ -228,7 +228,7 @@ class KVCompressAttention(nn.Module):attn_drop: float = 0.0,proj_drop: float = 0.0,norm_layer: nn.Module = LlamaRMSNorm,- enable_flashattn: bool = False,+ enable_flash_attn: bool = False,sampling="conv",sr_ratio=1,mem_eff_attention=False,@@ -240,7 +240,7 @@ class KVCompressAttention(nn.Module):self.num_heads = num_headsself.head_dim = dim // num_headsself.scale = self.head_dim**-0.5- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)@@ -291,7 +291,7 @@ class KVCompressAttention(nn.Module):new_N = NH, W = HW# flash attn is not memory efficient for small sequences, this is empirical- enable_flashattn = self.enable_flashattn and (N > B)+ enable_flash_attn = self.enable_flash_attn and (N > B)qkv = self.qkv(x).reshape(B, N, 3, C)q, k, v = qkv.unbind(2)@@ -307,7 +307,7 @@ class KVCompressAttention(nn.Module):q, k = self.q_norm(q), self.k_norm(k)- if enable_flashattn:+ if enable_flash_attn:from flash_attn import flash_attn_funcx = flash_attn_func(@@ -340,7 +340,7 @@ class KVCompressAttention(nn.Module):x = attn @ vx_output_shape = (B, N, C)- if not enable_flashattn:+ if not enable_flash_attn:x = x.transpose(1, 2)x = x.reshape(x_output_shape)x = self.proj(x)diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.pyindex 2ae9643..02f8b67 100644--- a/opensora/models/pixart/pixart.py+++ b/opensora/models/pixart/pixart.py@@ -61,13 +61,13 @@ class PixArtBlock(nn.Module):num_heads,mlp_ratio=4.0,drop_path=0.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):super().__init__()self.hidden_size = hidden_size- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself._enable_sequence_parallelism = enable_sequence_parallelismif enable_sequence_parallelism:@@ -82,7 +82,7 @@ class PixArtBlock(nn.Module):hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,)self.cross_attn = self.mha_cls(hidden_size, num_heads)self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)@@ -130,7 +130,7 @@ class PixArt(nn.Module):freeze=None,space_scale=1.0,time_scale=1.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,base_size=None,@@ -156,7 +156,7 @@ class PixArt(nn.Module):self.no_temporal_pos_emb = no_temporal_pos_embself.depth = depthself.mlp_ratio = mlp_ratio- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself.enable_layernorm_kernel = enable_layernorm_kernelself.space_scale = space_scaleself.time_scale = time_scale@@ -183,7 +183,7 @@ class PixArt(nn.Module):num_heads,mlp_ratio=mlp_ratio,drop_path=drop_path[i],- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,enable_layernorm_kernel=enable_layernorm_kernel,)for i in range(depth)diff --git a/opensora/models/pixart/pixart_sigma.py b/opensora/models/pixart/pixart_sigma.pyindex e815aa9..835831c 100644--- a/opensora/models/pixart/pixart_sigma.py+++ b/opensora/models/pixart/pixart_sigma.py@@ -58,7 +58,7 @@ class PixArtBlock(nn.Module):num_heads,mlp_ratio=4.0,drop_path=0.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,qk_norm=False,@@ -67,7 +67,7 @@ class PixArtBlock(nn.Module):):super().__init__()self.hidden_size = hidden_size- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself._enable_sequence_parallelism = enable_sequence_parallelismassert not enable_sequence_parallelism, "Sequence parallelism is not supported in this version."@@ -79,7 +79,7 @@ class PixArtBlock(nn.Module):hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,qk_norm=qk_norm,sr_ratio=sr_ratio,sampling=sampling,@@ -136,7 +136,7 @@ class PixArt_Sigma(nn.Module):qk_norm=False,space_scale=1.0,time_scale=1.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,kv_compress_config=None,@@ -159,7 +159,7 @@ class PixArt_Sigma(nn.Module):self.no_temporal_pos_emb = no_temporal_pos_embself.depth = depthself.mlp_ratio = mlp_ratio- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself.enable_layernorm_kernel = enable_layernorm_kernelself.space_scale = space_scaleself.time_scale = time_scale@@ -195,7 +195,7 @@ class PixArt_Sigma(nn.Module):num_heads,mlp_ratio=mlp_ratio,drop_path=drop_path[i],- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,enable_layernorm_kernel=enable_layernorm_kernel,qk_norm=qk_norm,sr_ratio=(diff --git a/opensora/models/stdit/stdit.py b/opensora/models/stdit/stdit.pyindex 6e16058..6882572 100644--- a/opensora/models/stdit/stdit.py+++ b/opensora/models/stdit/stdit.py@@ -37,13 +37,13 @@ class STDiTBlock(nn.Module):d_t=None,mlp_ratio=4.0,drop_path=0.0,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):super().__init__()self.hidden_size = hidden_size- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself._enable_sequence_parallelism = enable_sequence_parallelismif enable_sequence_parallelism:@@ -58,7 +58,7 @@ class STDiTBlock(nn.Module):hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,)self.cross_attn = self.mha_cls(hidden_size, num_heads)self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)@@ -82,7 +82,7 @@ class STDiTBlock(nn.Module):hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=self.enable_flashattn,+ enable_flash_attn=self.enable_flash_attn,)def t_mask_select(self, x, masked_x, x_mask):@@ -174,7 +174,7 @@ class STDiT(nn.Module):space_scale=1.0,time_scale=1.0,freeze=None,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):@@ -194,7 +194,7 @@ class STDiT(nn.Module):self.no_temporal_pos_emb = no_temporal_pos_embself.depth = depthself.mlp_ratio = mlp_ratio- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself.enable_layernorm_kernel = enable_layernorm_kernelself.space_scale = space_scaleself.time_scale = time_scale@@ -221,7 +221,7 @@ class STDiT(nn.Module):self.num_heads,mlp_ratio=self.mlp_ratio,drop_path=drop_path[i],- enable_flashattn=self.enable_flashattn,+ enable_flash_attn=self.enable_flash_attn,enable_layernorm_kernel=self.enable_layernorm_kernel,enable_sequence_parallelism=enable_sequence_parallelism,d_t=self.num_temporal,diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.pyindex efafc37..f7fb2a4 100644--- a/opensora/models/stdit/stdit3.py+++ b/opensora/models/stdit/stdit3.py@@ -34,14 +34,14 @@ class STDiT3Block(nn.Module):rope=None,qk_norm=False,temporal=False,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,):super().__init__()self.temporal = temporalself.hidden_size = hidden_size- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself._enable_sequence_parallelism = enable_sequence_parallelismassert not enable_sequence_parallelism, "Sequence parallelism is not supported in STDiT3Block"@@ -55,7 +55,7 @@ class STDiT3Block(nn.Module):qkv_bias=True,qk_norm=qk_norm,rope=rope,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,)self.cross_attn = mha_cls(hidden_size, num_heads)self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)@@ -163,7 +163,7 @@ class STDiT3(nn.Module):model_max_length=300,dtype=torch.float32,qk_norm=False,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,only_train_temporal=False,@@ -182,7 +182,7 @@ class STDiT3(nn.Module):# computation relatedself.drop_path = drop_pathself.dtype = dtype- self.enable_flashattn = enable_flashattn+ self.enable_flash_attn = enable_flash_attnself.enable_layernorm_kernel = enable_layernorm_kernel# input size related@@ -217,7 +217,7 @@ class STDiT3(nn.Module):mlp_ratio=mlp_ratio,drop_path=drop_path[i],qk_norm=qk_norm,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,enable_layernorm_kernel=enable_layernorm_kernel,enable_sequence_parallelism=enable_sequence_parallelism,)@@ -235,7 +235,7 @@ class STDiT3(nn.Module):mlp_ratio=mlp_ratio,drop_path=drop_path[i],qk_norm=qk_norm,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,enable_layernorm_kernel=enable_layernorm_kernel,enable_sequence_parallelism=enable_sequence_parallelism,# temporal@@ -297,6 +297,7 @@ class STDiT3(nn.Module):x = x.to(self.dtype)timestep = timestep.to(self.dtype)y = y.to(self.dtype)+ breakpoint()# === get pos embed ===_, _, Tx, Hx, Wx = x.size()diff --git a/scripts/train.py b/scripts/train.pyindex 5adce31..184008c 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -133,7 +133,7 @@ def main():input_size=latent_size,in_channels=vae.out_channels,caption_channels=text_encoder.output_dim,- model_max_length=text_encoder.model_max_length+ model_max_length=text_encoder.model_max_length,)model_numel, model_numel_trainable = get_model_numel(model)logger.info(@@ -238,6 +238,7 @@ def main():x = vae.encode(x) # [B, C, T, H/P, W/P]# Prepare text inputsmodel_args = text_encoder.encode(y)+ breakpoint()# Maskif cfg.mask_ratios is not None:diff --git a/tests/test_attn.py b/tests/test_attn.pyindex 48f8c43..4cf5844 100644--- a/tests/test_attn.py+++ b/tests/test_attn.py@@ -11,7 +11,7 @@ B, S, H = 128, 32, 1152N, D = 16, 72-def run_attn(enable_flashattn: bool):+def run_attn(enable_flash_attn: bool):get_accelerator().reset_peak_memory_stats()rope = RotaryEmbedding(D).to(device=get_current_device(), dtype=torch.bfloat16)attn = Attention(@@ -19,7 +19,7 @@ def run_attn(enable_flashattn: bool):N,qkv_bias=True,rope=rope.rotate_queries_or_keys,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,).to(device=get_current_device(), dtype=torch.bfloat16)x = torch.randn(B, S, H, device=get_current_device(), dtype=torch.bfloat16).requires_grad_()y = attn(x)diff --git a/tests/test_seq_parallel_attention.py b/tests/test_seq_parallel_attention.pyindex 9cce2b2..841115a 100644--- a/tests/test_seq_parallel_attention.py+++ b/tests/test_seq_parallel_attention.py@@ -18,14 +18,14 @@ def run_attention(rank, world_size):torch.manual_seed(1024)set_sequence_parallel_group(dist.group.WORLD)- seq_parallel_attention = SeqParallelAttention(dim=256, num_heads=4, qkv_bias=True, enable_flashattn=False).cuda()+ seq_parallel_attention = SeqParallelAttention(dim=256, num_heads=4, qkv_bias=True, enable_flash_attn=False).cuda()torch.manual_seed(1024)attention = Attention(dim=256,num_heads=4,qkv_bias=True,- enable_flashattn=False,+ enable_flash_attn=False,).cuda()# create inputsdiff --git a/tools/architecture/net2net.py b/tools/architecture/net2net.pyindex 80d346a..d5d7eb3 100644--- a/tools/architecture/net2net.py+++ b/tools/architecture/net2net.py@@ -43,7 +43,7 @@ if __name__ == "__main__":from opensora.models.pixart import PixArt_1B_2- model = PixArt_1B_2(no_temporal_pos_emb=True, space_scale=4, enable_flashattn=True, enable_layernorm_kernel=True)+ model = PixArt_1B_2(no_temporal_pos_emb=True, space_scale=4, enable_flash_attn=True, enable_layernorm_kernel=True)print("load model done")ckpt = torch.load("/home/zhouyukun/projs/opensora/pretrained_models/PixArt-Sigma-XL-2-2K-MS.pth")