Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[exp] update config

Zangwei Zhengc4f17b41fcd5
diff --git a/configs/opensora-v1-2/inference/xl-rf.py b/configs/opensora-v1-2/inference/xl-rf.py
index 7871f09..943a64b 100644
--- a/configs/opensora-v1-2/inference/xl-rf.py
+++ b/configs/opensora-v1-2/inference/xl-rf.py
@@ -11,12 +11,27 @@ model = dict(
enable_flashattn=True,
enable_layernorm_kernel=True,
)
+# vae = dict(
+# type="VideoAutoencoderKL",
+# micro_batch_size=4,
+# from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+# subfolder="vae",
+# local_files_only=True,
+# )
vae = dict(
- type="VideoAutoencoderKL",
- micro_batch_size=4,
- from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
- subfolder="vae",
- local_files_only=True,
+ type="VideoAutoencoderPipeline",
+ from_pretrained="pretrained_models/vae-v1",
+ vae_2d=dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+ subfolder="vae",
+ micro_batch_size=4,
+ local_files_only=True,
+ ),
+ vae_temporal=dict(
+ type="VAE_Temporal_SD",
+ from_pretrained=None,
+ ),
)
text_encoder = dict(
type="t5",
@@ -28,7 +43,7 @@ scheduler = dict(
type="rflow",
use_discrete_timesteps=False,
use_timestep_transform=True,
- num_sampling_steps=8,
+ num_sampling_steps=30,
cfg_scale=4.5,
)
dtype = "bf16"
diff --git a/configs/opensora-v1-2/train/adapt-vae.py b/configs/opensora-v1-2/train/adapt-vae.py
new file mode 100644
index 0000000..eb99249
--- /dev/null
+++ b/configs/opensora-v1-2/train/adapt-vae.py
@@ -0,0 +1,92 @@
+# Dataset settings
+dataset = dict(
+ type="VariableVideoTextDataset",
+ transform_name="resize_crop",
+)
+bucket_config = {
+ "144p": {1: (1.0, 64)},
+ # ---
+ "256": {1: (0.5, 48)},
+ "240p": {1: (0.5, 48)},
+ # ---
+ "360p": {1: (0.5, 18)},
+ "512": {1: (0.5, 18)},
+ # ---
+ "480p": {1: (0.5, 8)},
+ # ---
+ "720p": {1: (0.2, 4)},
+ "1024": {1: (0.2, 4)},
+ # ---
+ "1080p": {1: (0.3, 2)},
+ # ---
+ "2048": {1: (0.3, 1)},
+}
+grad_checkpoint = False
+
+# Acceleration settings
+num_workers = 4
+num_bucket_build_workers = 16
+dtype = "bf16"
+plugin = "zero2"
+sp_size = 1
+
+# Model settings
+model = dict(
+ type="STDiT3-XL/2",
+ from_pretrained=None,
+ qk_norm=True,
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+)
+# vae = dict(
+# type="VideoAutoencoderKL",
+# micro_batch_size=4,
+# from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+# subfolder="vae",
+# local_files_only=True,
+# )
+vae = dict(
+ type="VideoAutoencoderPipeline",
+ from_pretrained="pretrained_models/vae-v1",
+ training=False,
+ vae_2d=dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+ subfolder="vae",
+ micro_batch_size=4,
+ local_files_only=True,
+ ),
+ vae_temporal=dict(
+ type="VAE_Temporal_SD",
+ from_pretrained=None,
+ ),
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+ shardformer=True,
+ local_files_only=True,
+)
+scheduler = dict(
+ type="rflow",
+ use_discrete_timesteps=False,
+ use_timestep_transform=True,
+ sample_method="logit-normal",
+)
+
+# Log settings
+seed = 42
+outputs = "outputs"
+wandb = False
+epochs = 1000
+log_every = 10
+ckpt_every = 500
+
+# optimization settings
+load = None
+batch_size = None
+grad_clip = 1.0
+lr = 1e-4
+ema_decay = 0.99
+adam_eps = 1e-15
diff --git a/opensora/datasets/utils.py b/opensora/datasets/utils.py
index 18ec6e3..c5b712a 100644
--- a/opensora/datasets/utils.py
+++ b/opensora/datasets/utils.py
@@ -145,7 +145,7 @@ def read_from_path(path, image_size, transform_name="center"):
return read_image_from_path(path, image_size=image_size, transform_name=transform_name)
-def save_sample(x, fps=8, save_path=None, normalize=True, value_range=(-1, 1), force_video=False, verbose=True):
+def save_sample(x, save_path=None, fps=8, normalize=True, value_range=(-1, 1), force_video=False, verbose=True):
"""
Args:
x (Tensor): shape [C, T, H, W]
diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.py
index f28f136..44e0a5c 100644
--- a/opensora/models/stdit/stdit3.py
+++ b/opensora/models/stdit/stdit3.py
@@ -251,10 +251,10 @@ class STDiT3(nn.Module):
def initialize_weights(self):
# Initialize fps_embedder
- nn.init.normal_(self.fps.mlp[0].weight, std=0.02)
- nn.init.constant_(self.fps.mlp[0].bias, 0)
- nn.init.constant_(self.fps.mlp[2].weight, 0)
- nn.init.constant_(self.fps.mlp[2].bias, 0)
+ nn.init.normal_(self.fps_embedder.mlp[0].weight, std=0.02)
+ nn.init.constant_(self.fps_embedder.mlp[0].bias, 0)
+ nn.init.constant_(self.fps_embedder.mlp[2].weight, 0)
+ nn.init.constant_(self.fps_embedder.mlp[2].bias, 0)
def get_dynamic_size(self, x):
_, _, T, H, W = x.size()
diff --git a/opensora/models/vae/vae.py b/opensora/models/vae/vae.py
index ec2643d..2dcc25f 100644
--- a/opensora/models/vae/vae.py
+++ b/opensora/models/vae/vae.py
@@ -42,7 +42,7 @@ class VideoAutoencoderKL(nn.Module):
x = rearrange(x, "(B T) C H W -> B C T H W", B=B)
return x
- def decode(self, x):
+ def decode(self, x, **kwargs):
# x: (B, C, T, H, W)
B = x.shape[0]
x = rearrange(x, "B C T H W -> (B T) C H W")
@@ -91,7 +91,7 @@ class VideoAutoencoderKLTemporalDecoder(nn.Module):
def encode(self, x):
raise NotImplementedError
- def decode(self, x):
+ def decode(self, x, **kwargs):
B, _, T = x.shape[:3]
x = rearrange(x, "B C T H W -> (B T) C H W")
x = self.module.decode(x / 0.18215, num_frames=T).sample
@@ -118,10 +118,11 @@ class VideoAutoencoderKLTemporalDecoder(nn.Module):
@MODELS.register_module()
class VideoAutoencoderPipeline(nn.Module):
- def __init__(self, vae_2d=None, vae_temporal=None, freeze_vae_2d=False, from_pretrained=None):
+ def __init__(self, vae_2d=None, vae_temporal=None, freeze_vae_2d=False, from_pretrained=None, training=True):
super().__init__()
self.spatial_vae = build_module(vae_2d, MODELS)
self.temporal_vae = build_module(vae_temporal, MODELS)
+ self.training = training
if from_pretrained is not None:
load_checkpoint(self, from_pretrained)
@@ -129,24 +130,29 @@ class VideoAutoencoderPipeline(nn.Module):
for param in self.spatial_vae.parameters():
param.requires_grad = False
- def encode(self, x, training=True):
+ self.out_channels = self.temporal_vae.out_channels
+ self.scale = 2.5 # make std = 1.0
+
+ def encode(self, x):
x_z = self.spatial_vae.encode(x)
posterior = self.temporal_vae.encode(x_z)
z = posterior.sample()
- if training:
+ if self.training:
return z, posterior, x_z
- return z
+ return z / self.scale
- def decode(self, z, num_frames=None, training=True):
+ def decode(self, z, num_frames=None):
+ if not self.training:
+ z = z * self.scale
x_z = self.temporal_vae.decode(z, num_frames=num_frames)
x = self.spatial_vae.decode(x_z)
- if training:
+ if self.training:
return x, x_z
return x
def forward(self, x):
- z, posterior, x_z = self.encode(x, training=True)
- x_rec, x_z_rec = self.decode(z, num_frames=x_z.shape[2], training=True)
+ z, posterior, x_z = self.encode(x)
+ x_rec, x_z_rec = self.decode(z, num_frames=x_z.shape[2])
return x_rec, x_z_rec, z, posterior, x_z
def get_latent_size(self, input_size):
@@ -154,7 +160,7 @@ class VideoAutoencoderPipeline(nn.Module):
def get_temporal_last_layer(self):
return self.temporal_vae.decoder.conv_out.conv.weight
-
+
@property
def device(self):
return next(self.parameters()).device
diff --git a/opensora/models/vae/vae_temporal.py b/opensora/models/vae/vae_temporal.py
index b6d96be..473d057 100644
--- a/opensora/models/vae/vae_temporal.py
+++ b/opensora/models/vae/vae_temporal.py
@@ -341,6 +341,7 @@ class VAE_Temporal(nn.Module):
self.time_downsample_factor = 2 ** sum(temporal_downsample)
# self.time_padding = self.time_downsample_factor - 1
self.patch_size = (self.time_downsample_factor, 1, 1)
+ self.out_channels = in_out_channels
# NOTE: following MAGVIT, conv in bias=False in encoder first conv
self.encoder = Encoder(
@@ -368,10 +369,17 @@ class VAE_Temporal(nn.Module):
)
def get_latent_size(self, input_size):
- for i in range(len(input_size)):
- assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"
- input_size = [input_size[i] // self.patch_size[i] for i in range(3)]
- return input_size
+ latent_size = []
+ for i in range(3):
+ if input_size[i] is None:
+ lsize = None
+ elif i == 0:
+ time_padding = self.time_downsample_factor - input_size[i] % self.time_downsample_factor
+ lsize = (input_size[i] + time_padding) // self.patch_size[i]
+ else:
+ lsize = input_size[i] // self.patch_size[i]
+ latent_size.append(lsize)
+ return latent_size
def encode(self, x):
time_padding = self.time_downsample_factor - x.shape[2] % self.time_downsample_factor
@@ -396,7 +404,6 @@ class VAE_Temporal(nn.Module):
z = posterior.mode()
recon_video = self.decode(z, num_frames=x.shape[2])
return recon_video, posterior, z
-
@MODELS.register_module("VAE_Temporal_SD")
diff --git a/scripts/inference.py b/scripts/inference.py
index 5eb114c..993fa05 100644
--- a/scripts/inference.py
+++ b/scripts/inference.py
@@ -157,7 +157,7 @@ def main():
additional_args=model_args,
progress=verbose >= 2,
)
- samples = vae.decode(samples.to(dtype))
+ samples = vae.decode(samples.to(dtype), num_frames=cfg.num_frames)
# 4.4. save samples
if not use_dist or coordinator.is_master():