Open-Sora: Democratizing Efficient Video Production for All
Python
[exp] update config
Zangwei Zhengc4f17b41fcd5
diff --git a/configs/opensora-v1-2/inference/xl-rf.py b/configs/opensora-v1-2/inference/xl-rf.pyindex 7871f09..943a64b 100644--- a/configs/opensora-v1-2/inference/xl-rf.py+++ b/configs/opensora-v1-2/inference/xl-rf.py@@ -11,12 +11,27 @@ model = dict(enable_flashattn=True,enable_layernorm_kernel=True,)+# vae = dict(+# type="VideoAutoencoderKL",+# micro_batch_size=4,+# from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+# subfolder="vae",+# local_files_only=True,+# )vae = dict(- type="VideoAutoencoderKL",- micro_batch_size=4,- from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",- subfolder="vae",- local_files_only=True,+ type="VideoAutoencoderPipeline",+ from_pretrained="pretrained_models/vae-v1",+ vae_2d=dict(+ type="VideoAutoencoderKL",+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+ micro_batch_size=4,+ local_files_only=True,+ ),+ vae_temporal=dict(+ type="VAE_Temporal_SD",+ from_pretrained=None,+ ),)text_encoder = dict(type="t5",@@ -28,7 +43,7 @@ scheduler = dict(type="rflow",use_discrete_timesteps=False,use_timestep_transform=True,- num_sampling_steps=8,+ num_sampling_steps=30,cfg_scale=4.5,)dtype = "bf16"diff --git a/configs/opensora-v1-2/train/adapt-vae.py b/configs/opensora-v1-2/train/adapt-vae.pynew file mode 100644index 0000000..eb99249--- /dev/null+++ b/configs/opensora-v1-2/train/adapt-vae.py@@ -0,0 +1,92 @@+# Dataset settings+dataset = dict(+ type="VariableVideoTextDataset",+ transform_name="resize_crop",+)+bucket_config = {+ "144p": {1: (1.0, 64)},+ # ---+ "256": {1: (0.5, 48)},+ "240p": {1: (0.5, 48)},+ # ---+ "360p": {1: (0.5, 18)},+ "512": {1: (0.5, 18)},+ # ---+ "480p": {1: (0.5, 8)},+ # ---+ "720p": {1: (0.2, 4)},+ "1024": {1: (0.2, 4)},+ # ---+ "1080p": {1: (0.3, 2)},+ # ---+ "2048": {1: (0.3, 1)},+}+grad_checkpoint = False++# Acceleration settings+num_workers = 4+num_bucket_build_workers = 16+dtype = "bf16"+plugin = "zero2"+sp_size = 1++# Model settings+model = dict(+ type="STDiT3-XL/2",+ from_pretrained=None,+ qk_norm=True,+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+# vae = dict(+# type="VideoAutoencoderKL",+# micro_batch_size=4,+# from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+# subfolder="vae",+# local_files_only=True,+# )+vae = dict(+ type="VideoAutoencoderPipeline",+ from_pretrained="pretrained_models/vae-v1",+ training=False,+ vae_2d=dict(+ type="VideoAutoencoderKL",+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+ micro_batch_size=4,+ local_files_only=True,+ ),+ vae_temporal=dict(+ type="VAE_Temporal_SD",+ from_pretrained=None,+ ),+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+ shardformer=True,+ local_files_only=True,+)+scheduler = dict(+ type="rflow",+ use_discrete_timesteps=False,+ use_timestep_transform=True,+ sample_method="logit-normal",+)++# Log settings+seed = 42+outputs = "outputs"+wandb = False+epochs = 1000+log_every = 10+ckpt_every = 500++# optimization settings+load = None+batch_size = None+grad_clip = 1.0+lr = 1e-4+ema_decay = 0.99+adam_eps = 1e-15diff --git a/opensora/datasets/utils.py b/opensora/datasets/utils.pyindex 18ec6e3..c5b712a 100644--- a/opensora/datasets/utils.py+++ b/opensora/datasets/utils.py@@ -145,7 +145,7 @@ def read_from_path(path, image_size, transform_name="center"):return read_image_from_path(path, image_size=image_size, transform_name=transform_name)-def save_sample(x, fps=8, save_path=None, normalize=True, value_range=(-1, 1), force_video=False, verbose=True):+def save_sample(x, save_path=None, fps=8, normalize=True, value_range=(-1, 1), force_video=False, verbose=True):"""Args:x (Tensor): shape [C, T, H, W]diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.pyindex f28f136..44e0a5c 100644--- a/opensora/models/stdit/stdit3.py+++ b/opensora/models/stdit/stdit3.py@@ -251,10 +251,10 @@ class STDiT3(nn.Module):def initialize_weights(self):# Initialize fps_embedder- nn.init.normal_(self.fps.mlp[0].weight, std=0.02)- nn.init.constant_(self.fps.mlp[0].bias, 0)- nn.init.constant_(self.fps.mlp[2].weight, 0)- nn.init.constant_(self.fps.mlp[2].bias, 0)+ nn.init.normal_(self.fps_embedder.mlp[0].weight, std=0.02)+ nn.init.constant_(self.fps_embedder.mlp[0].bias, 0)+ nn.init.constant_(self.fps_embedder.mlp[2].weight, 0)+ nn.init.constant_(self.fps_embedder.mlp[2].bias, 0)def get_dynamic_size(self, x):_, _, T, H, W = x.size()diff --git a/opensora/models/vae/vae.py b/opensora/models/vae/vae.pyindex ec2643d..2dcc25f 100644--- a/opensora/models/vae/vae.py+++ b/opensora/models/vae/vae.py@@ -42,7 +42,7 @@ class VideoAutoencoderKL(nn.Module):x = rearrange(x, "(B T) C H W -> B C T H W", B=B)return x- def decode(self, x):+ def decode(self, x, **kwargs):# x: (B, C, T, H, W)B = x.shape[0]x = rearrange(x, "B C T H W -> (B T) C H W")@@ -91,7 +91,7 @@ class VideoAutoencoderKLTemporalDecoder(nn.Module):def encode(self, x):raise NotImplementedError- def decode(self, x):+ def decode(self, x, **kwargs):B, _, T = x.shape[:3]x = rearrange(x, "B C T H W -> (B T) C H W")x = self.module.decode(x / 0.18215, num_frames=T).sample@@ -118,10 +118,11 @@ class VideoAutoencoderKLTemporalDecoder(nn.Module):@MODELS.register_module()class VideoAutoencoderPipeline(nn.Module):- def __init__(self, vae_2d=None, vae_temporal=None, freeze_vae_2d=False, from_pretrained=None):+ def __init__(self, vae_2d=None, vae_temporal=None, freeze_vae_2d=False, from_pretrained=None, training=True):super().__init__()self.spatial_vae = build_module(vae_2d, MODELS)self.temporal_vae = build_module(vae_temporal, MODELS)+ self.training = trainingif from_pretrained is not None:load_checkpoint(self, from_pretrained)@@ -129,24 +130,29 @@ class VideoAutoencoderPipeline(nn.Module):for param in self.spatial_vae.parameters():param.requires_grad = False- def encode(self, x, training=True):+ self.out_channels = self.temporal_vae.out_channels+ self.scale = 2.5 # make std = 1.0++ def encode(self, x):x_z = self.spatial_vae.encode(x)posterior = self.temporal_vae.encode(x_z)z = posterior.sample()- if training:+ if self.training:return z, posterior, x_z- return z+ return z / self.scale- def decode(self, z, num_frames=None, training=True):+ def decode(self, z, num_frames=None):+ if not self.training:+ z = z * self.scalex_z = self.temporal_vae.decode(z, num_frames=num_frames)x = self.spatial_vae.decode(x_z)- if training:+ if self.training:return x, x_zreturn xdef forward(self, x):- z, posterior, x_z = self.encode(x, training=True)- x_rec, x_z_rec = self.decode(z, num_frames=x_z.shape[2], training=True)+ z, posterior, x_z = self.encode(x)+ x_rec, x_z_rec = self.decode(z, num_frames=x_z.shape[2])return x_rec, x_z_rec, z, posterior, x_zdef get_latent_size(self, input_size):@@ -154,7 +160,7 @@ class VideoAutoencoderPipeline(nn.Module):def get_temporal_last_layer(self):return self.temporal_vae.decoder.conv_out.conv.weight-+@propertydef device(self):return next(self.parameters()).devicediff --git a/opensora/models/vae/vae_temporal.py b/opensora/models/vae/vae_temporal.pyindex b6d96be..473d057 100644--- a/opensora/models/vae/vae_temporal.py+++ b/opensora/models/vae/vae_temporal.py@@ -341,6 +341,7 @@ class VAE_Temporal(nn.Module):self.time_downsample_factor = 2 ** sum(temporal_downsample)# self.time_padding = self.time_downsample_factor - 1self.patch_size = (self.time_downsample_factor, 1, 1)+ self.out_channels = in_out_channels# NOTE: following MAGVIT, conv in bias=False in encoder first convself.encoder = Encoder(@@ -368,10 +369,17 @@ class VAE_Temporal(nn.Module):)def get_latent_size(self, input_size):- for i in range(len(input_size)):- assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"- input_size = [input_size[i] // self.patch_size[i] for i in range(3)]- return input_size+ latent_size = []+ for i in range(3):+ if input_size[i] is None:+ lsize = None+ elif i == 0:+ time_padding = self.time_downsample_factor - input_size[i] % self.time_downsample_factor+ lsize = (input_size[i] + time_padding) // self.patch_size[i]+ else:+ lsize = input_size[i] // self.patch_size[i]+ latent_size.append(lsize)+ return latent_sizedef encode(self, x):time_padding = self.time_downsample_factor - x.shape[2] % self.time_downsample_factor@@ -396,7 +404,6 @@ class VAE_Temporal(nn.Module):z = posterior.mode()recon_video = self.decode(z, num_frames=x.shape[2])return recon_video, posterior, z-@MODELS.register_module("VAE_Temporal_SD")diff --git a/scripts/inference.py b/scripts/inference.pyindex 5eb114c..993fa05 100644--- a/scripts/inference.py+++ b/scripts/inference.py@@ -157,7 +157,7 @@ def main():additional_args=model_args,progress=verbose >= 2,)- samples = vae.decode(samples.to(dtype))+ samples = vae.decode(samples.to(dtype), num_frames=cfg.num_frames)# 4.4. save samplesif not use_dist or coordinator.is_master():