Open-Sora: Democratizing Efficient Video Production for All
Python
[feat] complete dynamic pos
Zangwei Zheng8b778213d171
diff --git a/configs/opensora-v1-1/train/Vx360p.py b/configs/opensora-v1-1/train/Vx360p.pyindex 4c7ea62..8d831a6 100644--- a/configs/opensora-v1-1/train/Vx360p.py+++ b/configs/opensora-v1-1/train/Vx360p.py@@ -4,7 +4,7 @@ dataset = dict(data_path=None,num_frames=None,frame_interval=3,- image_size=(360, 480), # base size+ image_size=(512, 512), # pretrained model is trained on 512x512transform_name="resize_crop",)bucket_config = {@@ -24,7 +24,6 @@ sp_size = 1# Define modelmodel = dict(type="STDiT2-XL/2",- space_scale=0.5,from_pretrained="PixArt-XL-2-1024-MS.pth",enable_flashattn=True,enable_layernorm_kernel=True,diff --git a/opensora/models/stdit/stdit2.py b/opensora/models/stdit/stdit2.pyindex bb08f24..25973fd 100644--- a/opensora/models/stdit/stdit2.py+++ b/opensora/models/stdit/stdit2.py@@ -189,7 +189,6 @@ class STDiT2(nn.Module):caption_channels=4096,model_max_length=120,dtype=torch.float32,- space_scale=1.0,freeze=None,enable_flashattn=False,enable_layernorm_kernel=False,@@ -207,12 +206,11 @@ class STDiT2(nn.Module):self.mlp_ratio = mlp_ratioself.enable_flashattn = enable_flashattnself.enable_layernorm_kernel = enable_layernorm_kernel- self.space_scale = space_scale# support dynamic inputself.patch_size = patch_sizeself.input_size = input_size- self.base_size = (input_size[1] / patch_size[1] * input_size[2] / patch_size[2]) ** 0.5+ self.input_sq_size = int((input_size[1] * input_size[2]) ** 0.5)self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)self.t_embedder = TimestepEmbedder(hidden_size)@@ -324,7 +322,9 @@ class STDiT2(nn.Module):_, _, Tx, Hx, Wx = x.size()T, H, W = self.get_dynamic_size(x)S = H * W- pos_emb = self.get_spatial_pos_embed(H, W, self.base_size).to(x.device, x.dtype)+ rs = (height[0] * width[0]).sqrt().item()+ scale = rs / self.input_sq_size+ pos_emb = self.get_spatial_pos_embed(H, W, scale=scale, base_size=round(S**0.5)).to(x.device, x.dtype)# embeddingx = self.x_embedder(x) # [B, N, C]@@ -432,11 +432,11 @@ class STDiT2(nn.Module):imgs = x.reshape(shape=(x.shape[0], c, t * pt, h * ph, w * pw))return imgs- def get_spatial_pos_embed(self, H, W, base_size=None):+ def get_spatial_pos_embed(self, H, W, scale=1.0, base_size=None):pos_embed = get_2d_sincos_pos_embed(self.hidden_size,(H, W),- scale=self.space_scale,+ scale=scale,base_size=base_size,)pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)