Open-Sora: Democratizing Efficient Video Production for All
Python
add wrapper for pretrained loading
Shen-Chenhui44b8142b4f0e
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.pyindex b1cac14..8ae73e2 100644--- a/configs/vae_3d/inference/16x256x256.py+++ b/configs/vae_3d/inference/16x256x256.py@@ -23,7 +23,7 @@ perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGV# Define modelmodel = dict(- type="VAE_3D",+ type="VAE_3D_B",from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained modelin_out_channels = 3,latent_embed_dim = 256,diff --git a/configs/vae_3d/train/16x256x256.py b/configs/vae_3d/train/16x256x256.pyindex c3fe27f..5d5927a 100644--- a/configs/vae_3d/train/16x256x256.py+++ b/configs/vae_3d/train/16x256x256.py@@ -21,7 +21,7 @@ perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGV# Define modelmodel = dict(- type="VAE_3D",+ type="VAE_3D_B",in_out_channels = 3,latent_embed_dim = 256,filters = 64,diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.pyindex 16dc4ab..e2aebbf 100644--- a/opensora/models/vae/vae_3d.py+++ b/opensora/models/vae/vae_3d.py@@ -7,6 +7,7 @@ import numpy as npfrom numpy import typing as nptypingfrom opensora.models.vae import model_utilsfrom opensora.registry import MODELS+from opensora.utils.ckpt_utils import load_checkpoint"""Encoder and Decoder stuctures with 3D CNNs."""@@ -370,7 +371,6 @@ class VAE_3D(nn.Module):kl_embed_dim = 64,device="cpu",dtype="bf16",- from_pretrained=None,# precision: Any = jax.lax.Precision.DEFAULT):super().__init__()@@ -385,43 +385,39 @@ class VAE_3D(nn.Module):# Model Initialization- if from_pretrained is None:- self.module.encoder = Encoder(- filters=filters,- num_res_blocks=num_res_blocks,- channel_multipliers=channel_multipliers,- temporal_downsample=temporal_downsample,- num_groups = num_groups, # for nn.GroupNorm- in_out_channels = in_out_channels,- latent_embed_dim = latent_embed_dim,- conv_downsample = conv_downsample,- custom_conv_padding = custom_conv_padding,- activation_fn = activation_fn,- device=device,- dtype=dtype,- )- self.module.decoder = Decoder(- latent_embed_dim = latent_embed_dim,- filters = filters,- in_out_channels = in_out_channels,- num_res_blocks = num_res_blocks,- channel_multipliers = channel_multipliers,- temporal_downsample = temporal_downsample,- num_groups = num_groups, # for nn.GroupNorm- upsample = upsample, # options: "deconv", "nearest+conv"- custom_conv_padding = custom_conv_padding,- activation_fn = activation_fn,- device=device,- dtype=dtype,- )-- self.module.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1)- self.module.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1)- else:- # TODO: add appropriate function and renaming- self.module = VAE_3D.from_pretrained(from_pretrained)+ self.encoder = Encoder(+ filters=filters,+ num_res_blocks=num_res_blocks,+ channel_multipliers=channel_multipliers,+ temporal_downsample=temporal_downsample,+ num_groups = num_groups, # for nn.GroupNorm+ in_out_channels = in_out_channels,+ latent_embed_dim = latent_embed_dim,+ conv_downsample = conv_downsample,+ custom_conv_padding = custom_conv_padding,+ activation_fn = activation_fn,+ device=device,+ dtype=dtype,+ )+ self.decoder = Decoder(+ latent_embed_dim = latent_embed_dim,+ filters = filters,+ in_out_channels = in_out_channels,+ num_res_blocks = num_res_blocks,+ channel_multipliers = channel_multipliers,+ temporal_downsample = temporal_downsample,+ num_groups = num_groups, # for nn.GroupNorm+ upsample = upsample, # options: "deconv", "nearest+conv"+ custom_conv_padding = custom_conv_padding,+ activation_fn = activation_fn,+ device=device,+ dtype=dtype,+ )+ self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1)+ self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1)+image_down = 2 ** len(temporal_downsample)t_down = 2 ** len([x for x in temporal_downsample if x == True])self.patch_size = (t_down, image_down, image_down)@@ -436,8 +432,8 @@ class VAE_3D(nn.Module):self,x,):- encoded_feature = self.module.encoder(x)- moments = self.module.quant_conv(encoded_feature).to(x.dtype)+ encoded_feature = self.encoder(x)+ moments = self.quant_conv(encoded_feature).to(x.dtype)posterior = model_utils.DiagonalGaussianDistribution(moments)return posterior@@ -446,8 +442,8 @@ class VAE_3D(nn.Module):z,):dtype = z.dtype- z = self.module.post_quant_conv(z).to(dtype)- dec = self.module.decoder(z)+ z = self.post_quant_conv(z).to(dtype)+ dec = self.decoder(z)return decdef forward(@@ -461,4 +457,12 @@ class VAE_3D(nn.Module):else:z = posterior.mode()dec = self.decode(z)- return dec, posterior\ No newline at end of file+ return dec, posterior++[email protected]_module("VAE_3D_B")+def VAE_3D_B(from_pretrained=None, **kwargs):+ model = VAE_3D(**kwargs)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return model\ No newline at end of file