Open-Sora: Democratizing Efficient Video Production for All
Python
add simple discriminator
Shen-Chenhui63f3737eb067
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.pyindex a5e16a5..3760233 100644--- a/configs/vae_magvit_v2/train/17x128x128.py+++ b/configs/vae_magvit_v2/train/17x128x128.py@@ -1,4 +1,3 @@-num_frames = 17frame_interval = 3image_size = (128, 128)@@ -34,6 +33,7 @@ model = dict(image_size = image_size,separate_first_frame_encoding = False,perceptual_loss_weight = 0.1, # use vgg is not None and more than 0+ num_frames = 17,)# Othersdiff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.mdindex 0a6094c..8184257 100644--- a/opensora/models/vae/README.md+++ b/opensora/models/vae/README.md@@ -41,3 +41,4 @@ CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node='x-transformers'```+diff --git a/opensora/models/vae/model_utils.py b/opensora/models/vae/model_utils.pyindex f5c487e..421dfea 100644--- a/opensora/models/vae/model_utils.py+++ b/opensora/models/vae/model_utils.py@@ -79,7 +79,7 @@ class VEA3DLoss(nn.Module):logvar_init=0.0,kl_weight=1.0,pixelloss_weight=1.0,- perceptual_weight=1.0,+ perceptual_weight=0.1,disc_loss="hinge",):diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex 649fce4..9d11610 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -9,10 +9,15 @@ from opensora.models.vae import model_utilsfrom opensora.registry import MODELSfrom opensora.utils.ckpt_utils import load_checkpointfrom einops import rearrange, repeat, pack, unpack+from einops.layers.torch import Rearrangeimport torch.nn.functional as Fimport torchvisionfrom torchvision.models import VGG16_Weightsfrom collections import namedtuple+from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers+from torch import nn, einsum, Tensor+from kornia.filters import filter3d+import math"""Encoder and Decoder stuctures with 3D CNNs."""@@ -54,6 +59,18 @@ def pick_video_frame(video, frame_indices):def exists(v):return v is not None+def hinge_discr_loss(fake, real):+ return (F.relu(1 + fake) + F.relu(1 - real)).mean()++def hinge_gen_loss(fake):+ return -fake.mean()++def xavier_uniform_weight_init(m):+ if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):+ nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain('relu'))+ nn.init.zeros_(m.bias)+ print("initialized module to xavier_uniform:", m)+def Sequential(*modules):modules = [*filter(exists, modules)]@@ -154,7 +171,161 @@ class ResBlock(nn.Module):residual = self.conv3(residual)return x + residual+# discriminator with anti-aliased downsampling (blurpool Zhang et al.)+class Blur(nn.Module):+ def __init__(self):+ super().__init__()+ f = torch.Tensor([1, 2, 1])+ self.register_buffer('f', f)++ def forward(+ self,+ x,+ space_only = False,+ time_only = False+ ):+ assert not (space_only and time_only)++ f = self.f++ if space_only:+ f = einsum('i, j -> i j', f, f)+ f = rearrange(f, '... -> 1 1 ...')+ elif time_only:+ f = rearrange(f, 'f -> 1 f 1 1')+ else:+ f = einsum('i, j, k -> i j k', f, f, f)+ f = rearrange(f, '... -> 1 ...')++ is_images = x.ndim == 4++ if is_images:+ x = rearrange(x, 'b c h w -> b c 1 h w')++ out = filter3d(x, f, normalized = True)++ if is_images:+ out = rearrange(out, 'b c 1 h w -> b c h w')++ return out++class ResBlockDown(nn.Module):+ """3D StyleGAN ResBlock for D."""++ def __init__(+ self,+ in_channels,+ filters,+ activation_fn,+ num_groups=32,+ device="cpu",+ dtype=torch.bfloat16,+ ):+ super().__init__()++ self.filters = filters+ self.activation_fn = activation_fn++ # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code+ self.conv1 = nn.Conv3d(in_channels, in_channels, (3,3,3)) # NOTE: init to xavier_uniform+ self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)++ # SCH: NOTE: use blur pooling instead, pooling bias is False following enc dec conv pool+ self.blur = Blur()+ self.conv_pool_residual = nn.Conv3d(in_channels * 8, in_channels, 3, use_bias=False) # NOTE: init to xavier_uniform+ self.conv_pool_input = nn.Conv3d(in_channels * 8, in_channels, 3, use_bias=False) # NOTE: init to xavier_uniform+ self.conv2 = nn.Conv3d(in_channels, self.filters,(1,1,1), use_bias=False) # NOTE: init to xavier_uniform+ self.conv3 = nn.Conv3d(in_channels, self.filters, (3,3,3)) # NOTE: init to xavier_uniform+ self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)++ def forward(self, x):+ residual = x+ x = self.conv1(x)+ x = self.norm1(x)+ x = self.activation_fn(x)++ residual = self.blur(residual)+ residual = rearrange(residual, 'b c (t pt) (h ph) (w pw) -> b (c pt ph pw) t h w', pt = 2, ph = 2, pw = 2)+ residual = self.conv_pool_residual(residual)+ residual = self.conv2(residual)++ x = self.blur(x)+ x = rearrange(x, 'b c (t pt) (h ph) (w pw) -> b (c pt ph pw) t h w', pt = 2, ph = 2, pw = 2)+ x = self.conv_pool_input(x)+ x = self.conv3(x)+ x = self.norm2(x)+ x = self.activation_fn(x)+ out = (residual + x) / math.sqrt(2)+ return out++class StyleGANDiscriminator(nn.Module):+ """StyleGAN Discriminator."""+ """+ SCH: NOTE:+ this discriminator requries the num_frames to be fixed during training;+ in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!+ """+ def __init__(+ self,+ image_size = (128, 128),+ num_frames = 17,+ discriminator_in_channels = 3,+ discriminator_filters = 128,+ discriminator_channel_multipliers = (2,4,4,4,4),+ num_groups=32,+ dtype = torch.bfloat16,+ device="cpu",+ ):+ self.dtype = dtype+ self.input_size = cast_tuple(image_size, 2)+ self.filters = discriminator_filters+ self.activation_fn = nn.LeakyReLu(negative_slope=0.2)+ self.channel_multipliers = discriminator_channel_multipliers++ self.conv1 = nn.Conv3d(discriminator_in_channels, self.filters, (3, 3, 3)) # NOTE: init to xavier_uniform++ prev_filters = self.filters # record in_channels+ self.num_blocks = len(self.channel_multipliers)+ self.res_block_list = []+ for i in range(self.num_blocks):+ filters = self.filters * self.channel_multipliers[i]+ self.res_block_list.append(ResBlockDown(prev_filters, filters, self.activation_fn))+ prev_filters = filters # update in_channels++ self.conv2 = nn.Conv3d(prev_filters, prev_filters, (3,3,3)) # NOTE: init to xavier_uniform+ torch.nn.init.xavier_uniform(self.conv2.weight)++ self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)++ scale_factor = 2 ** len(self.num_blocks)+ if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage+ time_scaled = num_frames // scale_factor + 1+ else:+ time_scaled = num_frames / scale_factor+ w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor+ in_features = prev_filters * time_scaled * w_scaled * h_scaled # (C*T*W*H)+ self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform+ self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform++ def forward(self, x):++ x = self.conv1(x)+ x = self.activation_fn(x)++ for i in range(self.num_blocks):+ x = self.res_block_list[i](x)++ x = self.conv2(x)+ x = self.norm1(x)+ x = self.activation_fn(x)+ x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?++ x = self.linear1(x)+ x = self.activation_fn(x)+ x = self.linear2(x)+ return x+class Encoder(nn.Module):"""Encoder Blocks."""def __init__(self,@@ -227,7 +398,6 @@ class Encoder(nn.Module):if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x 2 x 2t_stride = 2 if self.temporal_downsample[i] else 1- # TODO: conv_fn usess default stride t x 1 x 1, cannotself.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, 2, 2))) # SCH: should be same in_channel and out_channelprev_filters = filters # update in_channels@@ -265,8 +435,6 @@ class Encoder(nn.Module):x = self.conv2(x)return x--class Decoder(nn.Module):"""Decoder Blocks."""def __init__(self,@@ -392,7 +560,6 @@ class Decoder(nn.Module):# x = self.conv2(x)return x-LossBreakdown = namedtuple('LossBreakdown', ['recon_loss','perceptual_loss',@@ -417,10 +584,15 @@ class VAE_3D_V2(nn.Module):image_size = (128, 128),separate_first_frame_encoding = False,perceptual_loss_weight = 0.1,+ adversarial_loss_weight = None,vgg = None,vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,channel_multipliers = (1, 2, 2, 4),temporal_downsample = (True, True, False),+ num_frames = 17,+ discriminator_in_channels = 3,+ discriminator_filters = 128,+ discriminator_channel_multipliers = (2,4,4,4,4),num_groups = 32, # for nn.GroupNorm# conv_downsample = False,# upsample = "nearest+conv", # options: "deconv", "nearest+conv"@@ -466,7 +638,7 @@ class VAE_3D_V2(nn.Module):self.conv_out = CausalConv3d(filters, in_out_channels, 3, dtype=dtype, device=device)self.encoder = Encoder(- filters=filters,+ filters = filters,num_res_blocks=num_res_blocks,channel_multipliers=channel_multipliers,temporal_downsample=temporal_downsample,@@ -476,8 +648,8 @@ class VAE_3D_V2(nn.Module):# conv_downsample = conv_downsample,custom_conv_padding = custom_conv_padding,activation_fn = activation_fn,- device=device,- dtype=dtype,+ device = device,+ dtype = dtype,)self.decoder = Decoder(latent_embed_dim = latent_embed_dim,@@ -490,22 +662,41 @@ class VAE_3D_V2(nn.Module):# upsample = upsample, # options: "deconv", "nearest+conv"custom_conv_padding = custom_conv_padding,activation_fn = activation_fn,- device=device,- dtype=dtype,+ device = device,+ dtype = dtype,)self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1, device=device, dtype=dtype)self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)# Perceptual Loss+ self.vgg = None+ self.perceptual_loss_weight = perceptual_loss_weightif perceptual_loss_weight is not None and perceptual_loss_weight > 0:+ # self.lpips = LPIPS().eval()if not exists(vgg):vgg = torchvision.models.vgg16(- weights = vgg_weights- )+ weights = vgg_weights+ )vgg.classifier = Sequential(*vgg.classifier[:-2])self.vgg = vgg- self.perceptual_loss_weight = perceptual_loss_weight++ # Adversarial Loss+ self.adversarial_loss_weight = adversarial_loss_weight+ self.disc = None+ if adversarial_loss_weight is not None and adversarial_loss_weight > 0:+ self.disc = StyleGANDiscriminator(+ image_size = image_size,+ num_frames = num_frames,+ discriminator_in_channels = discriminator_in_channels,+ discriminator_filters = discriminator_filters,+ discriminator_channel_multipliers = discriminator_channel_multipliers,+ num_groups = num_groups,+ dtype = dtype,+ device = device,+ )+ self.disc.apply(xavier_uniform_weight_init)+def get_latent_size(self, input_size):for i in range(len(input_size)):@@ -603,12 +794,22 @@ class VAE_3D_V2(nn.Module):recon_loss = F.mse_loss(video, recon_video)- # TODO: add adversarial loss+ # TODO: DOUBLE add more sophisticated discrminator loss+ if self.adversarial_loss_weight is not None and self.adversarial_loss_weight > 0:+ if video_contains_first_frame:+ # video_len = video.shape[2]+ # real_video = pad_at_dim(video, (self.time_padding, 0), value = 0., dim = 2)+ # video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]+ fake_video = pad_at_dim(recon_video, (self.time_padding, 0), value = 0., dim = 2)+ # real_logits = self.discr(real_video)+ fake_logits = self.discr(fake_video.detach())+ # dsicr_loss = hinge_discr_loss(fake_logits, real_logits)+ gen_loss = hinge_gen_loss(fake_logits)- # TODO: add perceptual loss- # perceptual loss+ # perceptual loss+ # SCH: NOTE: if mse can pick single frame, if use sum of errors, need to calc for all frames!if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0:frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices@@ -626,18 +827,20 @@ class VAE_3D_V2(nn.Module):input_vgg_feats = self.vgg(input_vgg_input)recon_vgg_feats = self.vgg(recon_vgg_input)perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)-+ # perceptual_loss = self.lpips(input_vgg_input.contiguous(), recon_vgg_input.contiguous())else:perceptual_loss = self.zerototal_loss = recon_loss \- + perceptual_loss * self.perceptual_loss_weight+ + perceptual_loss * self.perceptual_loss_weight \+ + gen_loss * self.adversarial_loss_weight# loss breakdownloss_breakdown = LossBreakdown(recon_loss,perceptual_loss,+ gen_loss)return total_loss, recon_videodiff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pyindex 18046c9..bb17874 100644--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@ -102,7 +102,7 @@ def main():if not cfg.use_image_transformelse get_transforms_image(cfg.image_size[0])),- num_frames=cfg.num_frames,+ num_frames=cfg.model.num_frames,frame_interval=cfg.frame_interval,root=cfg.root,)