Open-Sora: Democratizing Efficient Video Production for All
Python
enable pipeline vae training
Shen-Chenhui246eddecddc8
diff --git a/configs/vae_magvit_v2/train/pipeline_16x128x128.py b/configs/vae_magvit_v2/train/pipeline_16x128x128.pyindex a26f8a8..2751951 100644--- a/configs/vae_magvit_v2/train/pipeline_16x128x128.py+++ b/configs/vae_magvit_v2/train/pipeline_16x128x128.py@@ -25,7 +25,7 @@ vae_2d = dict(model = dict(type="VAE_MAGVIT_V2",- in_out_channels = 3,+ in_out_channels = 4,latent_embed_dim = 256,filters = 128,num_res_blocks = 4,@@ -44,7 +44,7 @@ discriminator = dict(type="DISCRIMINATOR_3D",image_size = image_size,num_frames = num_frames,- in_channels = 3,+ in_channels = 4,filters = 128,channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution)diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex 05ee2a8..bf81b89 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -932,6 +932,8 @@ class VAE_3D_V2(nn.Module): # , ModelMixinvideo_contains_first_frame = video_contains_first_frame)+ breakpoint()+return recon_video, posterior@@ -991,13 +993,18 @@ class VEALoss(nn.Module):if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:# handle channelschannels = video.shape[1]+ assert channels in {1,3,4}if channels == 1:- input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)- recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)+ input_vgg_input = repeat(video, 'b 1 h w -> b c h w', c = 3)+ recon_vgg_input = repeat(recon_video, 'b 1 h w -> b c h w', c = 3)elif channels == 4: # SCH: take the first 3 for perceptual loss calc- input_vgg_input = input_vgg_input[:, :3]- recon_vgg_input = recon_vgg_input[:, :3]- perceptual_loss = self.perceptual_loss_fn(video, recon_video)+ input_vgg_input = video[:, :3]+ recon_vgg_input = recon_video[:, :3]+ else:+ input_vgg_input = video+ recon_vgg_input = recon_video++ perceptual_loss = self.perceptual_loss_fn(input_vgg_input, recon_vgg_input)recon_loss = recon_loss + self.perceptual_loss_weight * perceptual_lossnll_loss = recon_loss / torch.exp(self.logvar) + self.logvar@@ -1223,7 +1230,6 @@ def load_checkpoint_with_inflation(model, ckpt_path):"""if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):state_dict = find_model(ckpt_path)- breakpoint() # NOTE: need to manually check before first usewith torch.no_grad():for key in state_dict:if key in model:diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pyindex ed63341..3d10db0 100644--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@ -314,13 +314,11 @@ def main():video_contains_first_frame = Trueelse:video = x-+# ===== Spatial VAE =====if cfg.get("use_pipeline") == True:with torch.no_grad():- video = vae.encode(video)-- breakpoint()+ video = vae_2d.encode(video)# ====== VAE ======recon_video, posterior = vae(