Open-Sora: Democratizing Efficient Video Production for All
Python
resolved conflicts
FrankLeeeeeec88d2d54ea7
diff --cc .gitignoreindex eb382c6,917be8b..258ec66--- a/.gitignore+++ b/.gitignore@@@ -183,5 -182,7 +183,12 @@@ hostfilgradio_cached_examples/wandb/++<<<<<<< HEAD+# vae weights+eval/vae/flolpips/weights/++=======+ # npm+ node_modules/+ package-lock.json+ package.json++>>>>>>> upstream/maindiff --cc configs/opensora-v1-1/inference/sample-ref.pyindex c214dc8,c896cae..c4c46a2--- a/configs/opensora-v1-1/inference/sample-ref.py+++ b/configs/opensora-v1-1/inference/sample-ref.py@@@ -30,11 -38,10 +38,11 @@@ reference_path =# Define modelmodel = dict(type="STDiT2-XL/2",- from_pretrained=None,+ from_pretrained="hpcai-tech/OpenSora-STDiT-v2-stage3",input_sq_size=512,qk_norm=True,+ qk_norm_legacy=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --cc configs/opensora-v1-1/inference/sample.pyindex 5b12cd2,29e63ca..c280046--- a/configs/opensora-v1-1/inference/sample.py+++ b/configs/opensora-v1-1/inference/sample.py@@@ -7,11 -7,10 +7,11 @@@ multi_resolution = "STDiT2# Define modelmodel = dict(type="STDiT2-XL/2",- from_pretrained=None,+ from_pretrained="hpcai-tech/OpenSora-STDiT-v2-stage3",input_sq_size=512,qk_norm=True,+ qk_norm_legacy=True,- enable_flashattn=True,+ enable_flash_attn=True,enable_layernorm_kernel=True,)vae = dict(diff --cc opensora/models/layers/blocks.pyindex 00fcb7c,c4dd5c3..eb0b5b5--- a/opensora/models/layers/blocks.py+++ b/opensora/models/layers/blocks.py@@@ -139,9 -139,8 +139,9 @@@ class Attention(nn.Module)attn_drop: float = 0.0,proj_drop: float = 0.0,norm_layer: nn.Module = LlamaRMSNorm,- enable_flashattn: bool = False,+ enable_flash_attn: bool = False,rope=None,+ qk_norm_legacy: bool = False,) -> None:super().__init__()assert dim % num_heads == 0, "dim should be divisible by num_heads"@@@ -173,19 -171,13 +173,19 @@@qkv = qkv.view(qkv_shape).permute(2, 0, 3, 1, 4)q, k, v = qkv.unbind(0)- # WARNING: this may be a bug- if self.rope:- q = self.rotary_emb(q)- k = self.rotary_emb(k)- q, k = self.q_norm(q), self.k_norm(k)+ if self.qk_norm_legacy:+ # WARNING: this may be a bug+ if self.rope:+ q = self.rotary_emb(q)+ k = self.rotary_emb(k)+ q, k = self.q_norm(q), self.k_norm(k)+ else:+ q, k = self.q_norm(q), self.k_norm(k)+ if self.rope:+ q = self.rotary_emb(q)+ k = self.rotary_emb(k)- if enable_flashattn:+ if enable_flash_attn:from flash_attn import flash_attn_func# (B, #heads, N, #dim) -> (B, N, #heads, #dim)@@@ -358,9 -220,8 +358,9 @@@ class SeqParallelAttention(Attention)attn_drop: float = 0.0,proj_drop: float = 0.0,norm_layer: nn.Module = LlamaRMSNorm,- enable_flashattn: bool = False,+ enable_flash_attn: bool = False,rope=None,+ qk_norm_legacy: bool = False,) -> None:assert rope is None, "Rope is not supported in SeqParallelAttention"super().__init__(diff --cc opensora/models/stdit/stdit2.pyindex 7eafec9,5de1769..56fd361--- a/opensora/models/stdit/stdit2.py+++ b/opensora/models/stdit/stdit2.py@@@ -63,9 -51,8 +52,9 @@@ class STDiT2Block(nn.Module)hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=enable_flashattn,+ enable_flash_attn=enable_flash_attn,qk_norm=qk_norm,+ qk_norm_legacy=qk_norm_legacy,)self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)@@@ -85,10 -72,9 +74,10 @@@hidden_size,num_heads=num_heads,qkv_bias=True,- enable_flashattn=self.enable_flashattn,+ enable_flash_attn=self.enable_flash_attn,rope=rope,qk_norm=qk_norm,+ qk_norm_legacy=qk_norm_legacy,)self.scale_shift_table_temporal = nn.Parameter(torch.randn(3, hidden_size) / hidden_size**0.5) # new@@@ -195,46 -183,73 +186,75 @@@ class STDiT2Config(PretrainedConfig)no_temporal_pos_emb=False,caption_channels=4096,model_max_length=120,- dtype=torch.float32,freeze=None,qk_norm=False,+ qk_norm_legacy=False,- enable_flashattn=False,+ enable_flash_attn=False,enable_layernorm_kernel=False,- enable_sequence_parallelism=False,+ **kwargs,):- super().__init__()- self.pred_sigma = pred_sigma+ self.input_size = input_size+ self.input_sq_size = input_sq_sizeself.in_channels = in_channels- self.out_channels = in_channels * 2 if pred_sigma else in_channels+ self.patch_size = patch_sizeself.hidden_size = hidden_size- self.num_heads = num_heads- self.dtype = dtype- self.no_temporal_pos_emb = no_temporal_pos_embself.depth = depth+ self.num_heads = num_headsself.mlp_ratio = mlp_ratio- self.enable_flashattn = enable_flashattn+ self.class_dropout_prob = class_dropout_prob+ self.pred_sigma = pred_sigma+ self.drop_path = drop_path+ self.no_temporal_pos_emb = no_temporal_pos_emb+ self.caption_channels = caption_channels+ self.model_max_length = model_max_length+ self.freeze = freeze+ self.qk_norm = qk_norm++ self.qk_norm_legacy = qk_norm_legacy+ self.enable_flash_attn = enable_flash_attnself.enable_layernorm_kernel = enable_layernorm_kernel+ super().__init__(**kwargs)- # support dynamic input- self.patch_size = patch_size- self.input_size = input_size- self.input_sq_size = input_sq_size- self.pos_embed = PositionEmbedding2D(hidden_size)- self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)- self.t_embedder = TimestepEmbedder(hidden_size)- self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))- self.t_block_temp = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 3 * hidden_size, bias=True)) # new+ @MODELS.register_module()+ class STDiT2(PreTrainedModel):++ config_class = STDiT2Config++ def __init__(+ self,+ config+ ):+ super().__init__(config)+ self.pred_sigma = config.pred_sigma+ self.in_channels = config.in_channels+ self.out_channels = config.in_channels * 2 if config.pred_sigma else config.in_channels+ self.hidden_size = config.hidden_size+ self.num_heads = config.num_heads+ self.no_temporal_pos_emb = config.no_temporal_pos_emb+ self.depth = config.depth+ self.mlp_ratio = config.mlp_ratio+ self.enable_flash_attn = config.enable_flash_attn+ self.enable_layernorm_kernel = config.enable_layernorm_kernel++ # support dynamic input+ self.patch_size = config.patch_size+ self.input_size = config.input_size+ self.input_sq_size = config.input_sq_size+ self.pos_embed = PositionEmbedding2D(config.hidden_size)++ self.x_embedder = PatchEmbed3D(config.patch_size, config.in_channels, config.hidden_size)+ self.t_embedder = TimestepEmbedder(config.hidden_size)+ self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(config.hidden_size, 6 * config.hidden_size, bias=True))+ self.t_block_temp = nn.Sequential(nn.SiLU(), nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=True)) # newself.y_embedder = CaptionEmbedder(- in_channels=caption_channels,- hidden_size=hidden_size,- uncond_prob=class_dropout_prob,+ in_channels=config.caption_channels,+ hidden_size=config.hidden_size,+ uncond_prob=config.class_dropout_prob,act_layer=approx_gelu,- token_num=model_max_length,+ token_num=config.model_max_length,)- drop_path = [x.item() for x in torch.linspace(0, drop_path, depth)]+ drop_path = [x.item() for x in torch.linspace(0, config.drop_path, config.depth)]self.rope = RotaryEmbedding(dim=self.hidden_size // self.num_heads) # newself.blocks = nn.ModuleList([@@@ -243,12 -258,10 +263,11 @@@self.num_heads,mlp_ratio=self.mlp_ratio,drop_path=drop_path[i],- enable_flashattn=self.enable_flashattn,+ enable_flash_attn=self.enable_flash_attn,enable_layernorm_kernel=self.enable_layernorm_kernel,- enable_sequence_parallelism=enable_sequence_parallelism,rope=self.rope.rotate_queries_or_keys,- qk_norm=qk_norm,- qk_norm_legacy=qk_norm_legacy,+ qk_norm=config.qk_norm,++ qk_norm_legacy=config.qk_norm_legacy,)for i in range(self.depth)]