Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

add wrapper for pretrained loading

Shen-Chenhui44b8142b4f0e
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.py
index b1cac14..8ae73e2 100644
--- a/configs/vae_3d/inference/16x256x256.py
+++ b/configs/vae_3d/inference/16x256x256.py
@@ -23,7 +23,7 @@ perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGV
# Define model
model = dict(
- type="VAE_3D",
+ type="VAE_3D_B",
from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained model
in_out_channels = 3,
latent_embed_dim = 256,
diff --git a/configs/vae_3d/train/16x256x256.py b/configs/vae_3d/train/16x256x256.py
index c3fe27f..5d5927a 100644
--- a/configs/vae_3d/train/16x256x256.py
+++ b/configs/vae_3d/train/16x256x256.py
@@ -21,7 +21,7 @@ perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGV
# Define model
model = dict(
- type="VAE_3D",
+ type="VAE_3D_B",
in_out_channels = 3,
latent_embed_dim = 256,
filters = 64,
diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.py
index 16dc4ab..e2aebbf 100644
--- a/opensora/models/vae/vae_3d.py
+++ b/opensora/models/vae/vae_3d.py
@@ -7,6 +7,7 @@ import numpy as np
from numpy import typing as nptyping
from opensora.models.vae import model_utils
from opensora.registry import MODELS
+from opensora.utils.ckpt_utils import load_checkpoint
"""Encoder and Decoder stuctures with 3D CNNs."""
@@ -370,7 +371,6 @@ class VAE_3D(nn.Module):
kl_embed_dim = 64,
device="cpu",
dtype="bf16",
- from_pretrained=None,
# precision: Any = jax.lax.Precision.DEFAULT
):
super().__init__()
@@ -385,43 +385,39 @@ class VAE_3D(nn.Module):
# Model Initialization
- if from_pretrained is None:
- self.module.encoder = Encoder(
- filters=filters,
- num_res_blocks=num_res_blocks,
- channel_multipliers=channel_multipliers,
- temporal_downsample=temporal_downsample,
- num_groups = num_groups, # for nn.GroupNorm
- in_out_channels = in_out_channels,
- latent_embed_dim = latent_embed_dim,
- conv_downsample = conv_downsample,
- custom_conv_padding = custom_conv_padding,
- activation_fn = activation_fn,
- device=device,
- dtype=dtype,
- )
- self.module.decoder = Decoder(
- latent_embed_dim = latent_embed_dim,
- filters = filters,
- in_out_channels = in_out_channels,
- num_res_blocks = num_res_blocks,
- channel_multipliers = channel_multipliers,
- temporal_downsample = temporal_downsample,
- num_groups = num_groups, # for nn.GroupNorm
- upsample = upsample, # options: "deconv", "nearest+conv"
- custom_conv_padding = custom_conv_padding,
- activation_fn = activation_fn,
- device=device,
- dtype=dtype,
- )
-
- self.module.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1)
- self.module.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1)
- else:
- # TODO: add appropriate function and renaming
- self.module = VAE_3D.from_pretrained(from_pretrained)
+ self.encoder = Encoder(
+ filters=filters,
+ num_res_blocks=num_res_blocks,
+ channel_multipliers=channel_multipliers,
+ temporal_downsample=temporal_downsample,
+ num_groups = num_groups, # for nn.GroupNorm
+ in_out_channels = in_out_channels,
+ latent_embed_dim = latent_embed_dim,
+ conv_downsample = conv_downsample,
+ custom_conv_padding = custom_conv_padding,
+ activation_fn = activation_fn,
+ device=device,
+ dtype=dtype,
+ )
+ self.decoder = Decoder(
+ latent_embed_dim = latent_embed_dim,
+ filters = filters,
+ in_out_channels = in_out_channels,
+ num_res_blocks = num_res_blocks,
+ channel_multipliers = channel_multipliers,
+ temporal_downsample = temporal_downsample,
+ num_groups = num_groups, # for nn.GroupNorm
+ upsample = upsample, # options: "deconv", "nearest+conv"
+ custom_conv_padding = custom_conv_padding,
+ activation_fn = activation_fn,
+ device=device,
+ dtype=dtype,
+ )
+ self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1)
+ self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1)
+
image_down = 2 ** len(temporal_downsample)
t_down = 2 ** len([x for x in temporal_downsample if x == True])
self.patch_size = (t_down, image_down, image_down)
@@ -436,8 +432,8 @@ class VAE_3D(nn.Module):
self,
x,
):
- encoded_feature = self.module.encoder(x)
- moments = self.module.quant_conv(encoded_feature).to(x.dtype)
+ encoded_feature = self.encoder(x)
+ moments = self.quant_conv(encoded_feature).to(x.dtype)
posterior = model_utils.DiagonalGaussianDistribution(moments)
return posterior
@@ -446,8 +442,8 @@ class VAE_3D(nn.Module):
z,
):
dtype = z.dtype
- z = self.module.post_quant_conv(z).to(dtype)
- dec = self.module.decoder(z)
+ z = self.post_quant_conv(z).to(dtype)
+ dec = self.decoder(z)
return dec
def forward(
@@ -461,4 +457,12 @@ class VAE_3D(nn.Module):
else:
z = posterior.mode()
dec = self.decode(z)
- return dec, posterior
\ No newline at end of file
+ return dec, posterior
+
+
[email protected]_module("VAE_3D_B")
+def VAE_3D_B(from_pretrained=None, **kwargs):
+ model = VAE_3D(**kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
\ No newline at end of file