Open-Sora: Democratizing Efficient Video Production for All
Python
add perceptual loss
Shen-Chenhuica1d5863c3d4
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.pyindex 845aff5..a5e16a5 100644--- a/configs/vae_magvit_v2/train/17x128x128.py+++ b/configs/vae_magvit_v2/train/17x128x128.py@@ -16,7 +16,6 @@ sp_size = 1# Define Losskl_weight = 0.000001-perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper# Define model@@ -34,6 +33,7 @@ model = dict(activation_fn = 'swish',image_size = image_size,separate_first_frame_encoding = False,+ perceptual_loss_weight = 0.1, # use vgg is not None and more than 0)# Othersdiff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex 4eba9fd..6204618 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -8,8 +8,11 @@ from numpy import typing as nptypingfrom opensora.models.vae import model_utilsfrom opensora.registry import MODELSfrom opensora.utils.ckpt_utils import load_checkpoint-from einops import rearrange, pack, unpack+from einops import rearrange, repeat, pack, unpackimport torch.nn.functional as F+import torchvision+from torchvision.models import VGG16_Weights+from collections import namedtuple"""Encoder and Decoder stuctures with 3D CNNs."""@@ -38,6 +41,27 @@ def pad_at_dim(t, pad, dim = -1, value = 0.):zeros = ((0, 0) * dims_from_right)return F.pad(t, (*zeros, *pad), value = value)+def pick_video_frame(video, frame_indices):+ """get frame_indices from the video of [B, C, T, H, W] and return images of [B, C, H, W]"""+ batch, device = video.shape[0], video.device+ video = rearrange(video, 'b c f ... -> b f c ...')+ batch_indices = torch.arange(batch, device = device)+ batch_indices = rearrange(batch_indices, 'b -> b 1')+ images = video[batch_indices, frame_indices]+ images = rearrange(images, 'b 1 c ... -> b c ...')+ return images++def exists(v):+ return v is not None++def Sequential(*modules):+ modules = [*filter(exists, modules)]++ if len(modules) == 0:+ return nn.Identity()++ return nn.Sequential(*modules)+def SameConv2d(dim_in, dim_out, kernel_size):kernel_size = cast_tuple(kernel_size, 2)padding = [k // 2 for k in kernel_size]@@ -367,7 +391,20 @@ class Decoder(nn.Module):# NOTE: moved to VAE for separate first frame processing# x = self.conv2(x)return x-+++LossBreakdown = namedtuple('LossBreakdown', [+ 'recon_loss',+ 'perceptual_loss',+ # 'adversarial_gen_loss',+ # 'adaptive_adversarial_weight',+])++DiscrLossBreakdown = namedtuple('DiscrLossBreakdown', [+ 'discr_loss',+ 'multiscale_discr_losses',+ 'gradient_penalty'+])@MODELS.register_module()class VAE_3D_V2(nn.Module):@@ -379,6 +416,9 @@ class VAE_3D_V2(nn.Module):num_res_blocks = 2,image_size = (128, 128),separate_first_frame_encoding = False,+ perceptual_loss_weight = 0.1,+ vgg = None,+ vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,channel_multipliers = (1, 2, 2, 4),temporal_downsample = (True, True, False),num_groups = 32, # for nn.GroupNorm@@ -409,6 +449,10 @@ class VAE_3D_V2(nn.Module):self.time_padding = self.time_downsample_factor - 1self.separate_first_frame_encoding = separate_first_frame_encoding+ image_down = 2 ** len(temporal_downsample)+ t_down = 2 ** len([x for x in temporal_downsample if x == True])+ self.patch_size = (t_down, image_down, image_down)+# ==== Model Initialization ====# encoder & decoder first and last conv layer@@ -421,7 +465,6 @@ class VAE_3D_V2(nn.Module):self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3,3))self.conv_out = CausalConv3d(filters, in_out_channels, 3, dtype=dtype, device=device)-self.encoder = Encoder(filters=filters,num_res_blocks=num_res_blocks,@@ -454,9 +497,15 @@ class VAE_3D_V2(nn.Module):self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1, device=device, dtype=dtype)self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)- image_down = 2 ** len(temporal_downsample)- t_down = 2 ** len([x for x in temporal_downsample if x == True])- self.patch_size = (t_down, image_down, image_down)+ # Perceptual Loss+ if perceptual_loss_weight is not None and perceptual_loss_weight > 0:+ if not exists(vgg):+ vgg = torchvision.models.vgg16(+ weights = vgg_weights+ )+ vgg.classifier = Sequential(*vgg.classifier[:-2])+ self.vgg = vgg+ self.perceptual_loss_weight = perceptual_loss_weightdef get_latent_size(self, input_size):for i in range(len(input_size)):@@ -554,11 +603,45 @@ class VAE_3D_V2(nn.Module):recon_loss = F.mse_loss(video, recon_video)+ # TODO: add adversarial loss+# TODO: add perceptual loss- # TODO: add adversarial loss+ # perceptual loss++ if self.use_vgg:++ frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices++ input_vgg_input = pick_video_frame(video, frame_indices)+ recon_vgg_input = pick_video_frame(recon_video, frame_indices)++ if channels == 1:+ input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)+ recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)++ elif channels == 4: # SCH: take the first 3 for perceptual loss calc+ input_vgg_input = input_vgg_input[:, :3]+ recon_vgg_input = recon_vgg_input[:, :3]++ input_vgg_feats = self.vgg(input_vgg_input)+ recon_vgg_feats = self.vgg(recon_vgg_input)++ perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)+ else:+ perceptual_loss = self.zero+++ total_loss = recon_loss \+ + perceptual_loss * self.perceptual_loss_weight++ # loss breakdown+ loss_breakdown = LossBreakdown(+ recon_loss,+ perceptual_loss,+ )- return recon_loss, recon_video+ return total_loss, recon_video@MODELS.register_module("VAE_MAGVIT_V2")