Open-Sora: Democratizing Efficient Video Production for All
Python
debug
Shen-Chenhuiaa733aa1d727
diff --git a/opensora/models/vae/discriminator_3d.py b/opensora/models/vae/discriminator_3d.pynew file mode 100644index 0000000..53fc388--- /dev/null+++ b/opensora/models/vae/discriminator_3d.py@@ -0,0 +1,116 @@+"""3D StyleGAN discriminator."""++import functools+import math+from typing import Any++import torch+import torch.nn as nn+import ml_collections++# TODO: torch.nn.init.xavier_uniform_+# default_kernel_init = nn.initializers.xavier_uniform()++class ResBlock(nn.Module):+ """3D StyleGAN ResBlock for D."""++ def __init__(+ self,+ in_channels,+ filters,+ activation_fn,+ input_dim, # x.shape[-1], TODO+ num_groups=32,+ device="cpu",+ dtype=torch.bfloat16,+ ):+ super().__init__()++ self.filters = filters+ self.activation_fn = activation_fn++ # TODO: figure out the input_dim++ self.conv1 = nn.Conv3d(in_channels, self.filters, (3,3,3)) # need to init to xavier_uniform+ self.norm1 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)+ self.avg_pool_with_t = nn.AvgPool3d((2,2,2))+ self.conv2 = nn.Conv3d(in_channels, self.filters,(1,1,1), use_bias=False) # need to init to xavier_uniform+ self.conv3 = nn.Conv3d(input_dim, self.filters, (3,3,3)) # need to init to xavier_uniform+ self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)++ def forward(self, x):+ residual = x+ x = self.conv1(x)+ x = self.norm1(x)+ x = self.activation_fn(x)++ x = self.avg_pool_with_t(x)+ residual = self.avg_pool_with_t(residual)+ residual = self.conv2(residual)+ x = self.conv3(x)+ x = self.norm2(x)+ x = self.activation_fn(x)+ out = (residual + x) / math.sqrt(2)+ return out++class StyleGANDiscriminator(nn.Module):+ """StyleGAN Discriminator."""+ def __init__(+ self,+ config,+ image_size,+ input_dim, # x.shape[-1]+ discriminator_in_channels = 3,+ discriminator_filters = 64,+ discriminator_channel_multipliers = (2,4,4,4,4),+ num_groups=32,+ dtype = torch.bfloat16,+ device="cpu",+ ):+ self.config = config+ self.dtype = dtype++ self.input_size = image_size+ self.filters = discriminator_filters++ self.activation_fn = nn.LeakyReLu(negative_slope=0.2)++ self.channel_multipliers = discriminator_channel_multipliers++++ self.conv1 = nn.Conv3d(discriminator_in_channels, self.filters, (3, 3, 3)) # need to init to xavier_uniform++ prev_filters = self.filters # record in_channels+ self.num_blocks = len(self.channel_multipliers)+ self.res_block_list = []+ for i in range(self.num_blocks):+ filters = self.filters * self.channel_multipliers[i]+ self.res_block_list.append(ResBlock(prev_filters, filters, self.activation_fn)) # TODO+ prev_filters = filters # update in_channels++ self.conv2 = nn.Conv3d(prev_filters, prev_filters, (3,3,3)) # need to init to xavier_uniform++ self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)++ # TODO: what is the in_features+ self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # need to init to xavier_uniform+ self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # need to init to xavier_uniform++ def forward(self, x):++ x = self.conv1(x)+ x = self.activation_fn(x)++ for i in range(self.num_blocks):+ x = self.res_block_list[i](x)++ x = self.conv2(x)+ x = self.norm1(x)+ x = self.activation_fn(x)+ x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?++ x = self.linear1(x)+ x = self.activation_fn(x)+ x = self.linear2(x)+ return xdiff --git a/opensora/models/vae/lpips.py b/opensora/models/vae/lpips.pynew file mode 100644index 0000000..3a16111--- /dev/null+++ b/opensora/models/vae/lpips.py@@ -0,0 +1,120 @@+import torch+import torch.nn as nn+from torchvision import models+from collections import namedtuple+from taming.util import get_ckpt_path+++class LPIPS(nn.Module):+ # Learned perceptual metric+ def __init__(self, use_dropout=True):+ super().__init__()+ self.scaling_layer = ScalingLayer()+ self.chns = [64, 128, 256, 512, 512] # vg16 features+ self.net = vgg16(pretrained=True, requires_grad=False) # NOTE: TODO: need in_channels = 4 to use+ self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout)+ self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout)+ self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout)+ self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout)+ self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout)+ self.load_from_pretrained()+ for param in self.parameters():+ param.requires_grad = False++ def load_from_pretrained(self, name="vgg_lpips"):+ ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips")+ self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)+ print("loaded pretrained LPIPS loss from {}".format(ckpt))++ @classmethod+ def from_pretrained(cls, name="vgg_lpips"):+ if name != "vgg_lpips":+ raise NotImplementedError+ model = cls()+ ckpt = get_ckpt_path(name)+ model.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)+ return model++ def forward(self, input, target):+ in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target))+ outs0, outs1 = self.net(in0_input), self.net(in1_input)+ feats0, feats1, diffs = {}, {}, {}+ lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4]+ for kk in range(len(self.chns)):+ feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk])+ diffs[kk] = (feats0[kk] - feats1[kk]) ** 2++ res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))]+ val = res[0]+ for l in range(1, len(self.chns)):+ val += res[l]+ return val+++# SCH: TODO: this channel shift & scale may need to be changed+class ScalingLayer(nn.Module):+ def __init__(self):+ super(ScalingLayer, self).__init__()+ self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None])+ self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None])++ def forward(self, inp):+ return (inp - self.shift) / self.scale+++class NetLinLayer(nn.Module):+ """ A single linear layer which does a 1x1 conv """+ def __init__(self, chn_in, chn_out=1, use_dropout=False):+ super(NetLinLayer, self).__init__()+ layers = [nn.Dropout(), ] if (use_dropout) else []+ layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ]+ self.model = nn.Sequential(*layers)+++class vgg16(torch.nn.Module):+ def __init__(self, requires_grad=False, pretrained=True):+ super(vgg16, self).__init__()+ vgg_pretrained_features = models.vgg16(pretrained=pretrained).features+ self.slice1 = torch.nn.Sequential()+ self.slice2 = torch.nn.Sequential()+ self.slice3 = torch.nn.Sequential()+ self.slice4 = torch.nn.Sequential()+ self.slice5 = torch.nn.Sequential()+ self.N_slices = 5+ for x in range(4):+ self.slice1.add_module(str(x), vgg_pretrained_features[x])+ for x in range(4, 9):+ self.slice2.add_module(str(x), vgg_pretrained_features[x])+ for x in range(9, 16):+ self.slice3.add_module(str(x), vgg_pretrained_features[x])+ for x in range(16, 23):+ self.slice4.add_module(str(x), vgg_pretrained_features[x])+ for x in range(23, 30):+ self.slice5.add_module(str(x), vgg_pretrained_features[x])+ if not requires_grad:+ for param in self.parameters():+ param.requires_grad = False++ def forward(self, X):+ h = self.slice1(X)+ h_relu1_2 = h+ h = self.slice2(h)+ h_relu2_2 = h+ h = self.slice3(h)+ h_relu3_3 = h+ h = self.slice4(h)+ h_relu4_3 = h+ h = self.slice5(h)+ h_relu5_3 = h+ vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3'])+ out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3)+ return out+++def normalize_tensor(x,eps=1e-10):+ norm_factor = torch.sqrt(torch.sum(x**2,dim=1,keepdim=True))+ return x/(norm_factor+eps)+++def spatial_average(x, keepdim=True):+ return x.mean([2,3],keepdim=keepdim)\ No newline at end of filediff --git a/opensora/models/vae/model_utils.py b/opensora/models/vae/model_utils.pyindex b624288..8ad0bd9 100644--- a/opensora/models/vae/model_utils.py+++ b/opensora/models/vae/model_utils.py@@ -5,131 +5,12 @@ from typing import Any, Optional, Sequence, Typeimport torch.nn as nnimport numpy as npimport torch-# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers+from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformersfrom taming.modules.discriminator.model import NLayerDiscriminator, weights_init"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models"""-import torch-from torchvision import models-from collections import namedtuple-from taming.util import get_ckpt_path---class LPIPS(nn.Module):- # Learned perceptual metric- def __init__(self, use_dropout=True):- super().__init__()- self.scaling_layer = ScalingLayer()- self.chns = [64, 128, 256, 512, 512] # vg16 features- self.net = vgg16(pretrained=True, requires_grad=False)- self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout)- self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout)- self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout)- self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout)- self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout)- self.load_from_pretrained()- for param in self.parameters():- param.requires_grad = False-- def load_from_pretrained(self, name="vgg_lpips"):- ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips")- self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)- print("loaded pretrained LPIPS loss from {}".format(ckpt))-- @classmethod- def from_pretrained(cls, name="vgg_lpips"):- if name != "vgg_lpips":- raise NotImplementedError- model = cls()- ckpt = get_ckpt_path(name)- model.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)- return model-- def forward(self, input, target):- in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target))- outs0, outs1 = self.net(in0_input), self.net(in1_input)- feats0, feats1, diffs = {}, {}, {}- lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4]- for kk in range(len(self.chns)):- feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk])- diffs[kk] = (feats0[kk] - feats1[kk]) ** 2-- res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))]- val = res[0]- for l in range(1, len(self.chns)):- val += res[l]- return val---# SCH: TODO: this channel shift & scale may need to be changed-class ScalingLayer(nn.Module):- def __init__(self):- super(ScalingLayer, self).__init__()- self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None])- self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None])-- def forward(self, inp):- return (inp - self.shift) / self.scale---class NetLinLayer(nn.Module):- """ A single linear layer which does a 1x1 conv """- def __init__(self, chn_in, chn_out=1, use_dropout=False):- super(NetLinLayer, self).__init__()- layers = [nn.Dropout(), ] if (use_dropout) else []- layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ]- self.model = nn.Sequential(*layers)---class vgg16(torch.nn.Module):- def __init__(self, requires_grad=False, pretrained=True):- super(vgg16, self).__init__()- vgg_pretrained_features = models.vgg16(pretrained=pretrained).features- self.slice1 = torch.nn.Sequential()- self.slice2 = torch.nn.Sequential()- self.slice3 = torch.nn.Sequential()- self.slice4 = torch.nn.Sequential()- self.slice5 = torch.nn.Sequential()- self.N_slices = 5- for x in range(4):- self.slice1.add_module(str(x), vgg_pretrained_features[x])- for x in range(4, 9):- self.slice2.add_module(str(x), vgg_pretrained_features[x])- for x in range(9, 16):- self.slice3.add_module(str(x), vgg_pretrained_features[x])- for x in range(16, 23):- self.slice4.add_module(str(x), vgg_pretrained_features[x])- for x in range(23, 30):- self.slice5.add_module(str(x), vgg_pretrained_features[x])- if not requires_grad:- for param in self.parameters():- param.requires_grad = False-- def forward(self, X):- h = self.slice1(X)- h_relu1_2 = h- h = self.slice2(h)- h_relu2_2 = h- h = self.slice3(h)- h_relu3_3 = h- h = self.slice4(h)- h_relu4_3 = h- h = self.slice5(h)- h_relu5_3 = h- vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3'])- out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3)- return out---def normalize_tensor(x,eps=1e-10):- norm_factor = torch.sqrt(torch.sum(x**2,dim=1,keepdim=True))- return x/(norm_factor+eps)---def spatial_average(x, keepdim=True):- return x.mean([2,3],keepdim=keepdim)## NOTE: not used since we only have 'GN'# def get_norm_layer(norm_type, dtype):@@ -189,48 +70,66 @@ class DiagonalGaussianDistribution(object):def mode(self):return self.mean-# class VEA3DLoss(nn.Module):-# def __init__(-# self,-# # disc_start,-# logvar_init=0.0,-# kl_weight=1.0,-# pixelloss_weight=1.0,-# perceptual_weight=1.0,-# disc_loss="hinge"-# ):-# super().__init__()-# assert disc_loss in ["hinge", "vanilla"]-# self.kl_weight = kl_weight-# self.pixel_weight = pixelloss_weight-# # self.perceptual_loss = LPIPS().eval() # TODO-# self.perceptual_weight = perceptual_weight-# # output log variance-# self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)++class VEA3DLoss(nn.Module):+ def __init__(+ self,+ # disc_start,+ logvar_init=0.0,+ kl_weight=1.0,+ pixelloss_weight=1.0,+ perceptual_weight=1.0,+ disc_loss="hinge",++ ):+ super().__init__()+ assert disc_loss in ["hinge", "vanilla"]+ self.kl_weight = kl_weight+ self.pixel_weight = pixelloss_weight+ self.perceptual_loss = LPIPS().eval()+ self.perceptual_weight = perceptual_weight+ # output log variance+ self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)+-# def forward(-# self,-# inputs,-# reconstructions,-# posteriors,-# weights=None,-# ):-# rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())+ def forward(+ self,+ inputs,+ reconstructions,+ posteriors,+ # optimizer_idx,+ # global_step,+ weights=None,+ ):+ rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())+ if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!+ assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)}"+ # SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frame+ permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]+ permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))+ data_shape = permutated_input.size()+ p_loss = self.perceptual_loss(+ permutated_input.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous(),+ permutated_rec.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous()+ )+ # SCH: shape back p_loss+ permuted_p_loss = torch.permute(p_loss.reshape(data_shape[0], data_shape[1], 1, 1, 1), (0,2,1,3,4))+ rec_loss = rec_loss + self.perceptual_weight * permuted_p_loss-# nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar-# weighted_nll_loss = nll_loss-# if weights is not None:-# weighted_nll_loss = weights*nll_loss-# weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]-# nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]-# kl_loss = posteriors.kl()-# kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]+ nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar+ weighted_nll_loss = nll_loss+ if weights is not None:+ weighted_nll_loss = weights*nll_loss+ weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]+ nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]+ kl_loss = posteriors.kl()+ kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]-# loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later--# return loss+ loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later-class VEA3DLoss(nn.Module):+ return loss++class VEA3DLossWithDiscriminator(nn.Module):def __init__(self,# disc_start,@@ -293,7 +192,8 @@ class VEA3DLoss(nn.Module):weights=None,):rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())- if self.perceptual_weight > 0:+ if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!+ assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)} "# SCH: transform to [(B,T), C, H, W] shape for percetual loss over each framepermutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))@@ -327,4 +227,4 @@ class VEA3DLoss(nn.Module):# # "{}/g_loss".format(split): g_loss.detach().mean(),# }- return loss+ return loss\ No newline at end of filediff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.pyindex 54e000d..b9891a3 100644--- a/opensora/models/vae/vae_3d.py+++ b/opensora/models/vae/vae_3d.py@@ -159,12 +159,13 @@ class Encoder(nn.Module):# conv blocks handlingif self.conv_downsample:t_stride = 2 if self.temporal_downsample[i] else 1- self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(4, 4, 4), strides=(t_stride, 2, 2))) # SCH: should be same in_channel and out_channel+ t_pad = 1 if self.temporal_downsample[i] else 0+ self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(4, 4, 4), strides=(t_stride, 2, 2)), padding=(t_pad,1,1)) # SCH: should be same in_channel and out_channelprev_filters = filters # update in_channels# NOTE: downsample, dimensions T, H, W- self.avg_pool_with_t = nn.AvgPool3d((2,2,2))- self.avg_pool = nn.AvgPool3d((1,2,2))+ self.avg_pool_with_t = nn.AvgPool3d((2,2,2), count_include_pad=False)+ self.avg_pool = nn.AvgPool3d((1,2,2), count_include_pad=False)# last layer res blockself.res_blocks = []@@ -301,14 +302,15 @@ class Decoder(nn.Module):# conv blocks handlingif i > 0:t_stride = 2 if self.temporal_downsample[i - 1] else 1+ t_kernel = 4 if self.temporal_downsample[i - 1] else 3 # SCH: hack to keep dimension sameif self.upsample == "deconv":assert self.custom_conv_padding is None, ('Custom padding not implemented for ConvTranspose')# SCH: append in front- self.conv_blocks.insert(0,- self.conv_t_fn(prev_filters, filters, kernel_size=(4, 4, 4), strides=(t_stride, 2, 2)))+ self.conv_blocks.insert(0,+ self.conv_t_fn(prev_filters, filters, kernel_size=(t_kernel, 4, 4), stride=(t_stride, 2, 2), padding=1))prev_filters = filters # SCH: update in_channelselif self.upsample == 'nearest+conv':- # SCH: append in front+ # SCH: append in frontself.conv_blocks.insert(0, self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3)))prev_filters = filters # SCH: update in_channelselse: