Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

debug

Shen-Chenhuiaa733aa1d727
diff --git a/opensora/models/vae/discriminator_3d.py b/opensora/models/vae/discriminator_3d.py
new file mode 100644
index 0000000..53fc388
--- /dev/null
+++ b/opensora/models/vae/discriminator_3d.py
@@ -0,0 +1,116 @@
+"""3D StyleGAN discriminator."""
+
+import functools
+import math
+from typing import Any
+
+import torch
+import torch.nn as nn
+import ml_collections
+
+# TODO: torch.nn.init.xavier_uniform_
+# default_kernel_init = nn.initializers.xavier_uniform()
+
+class ResBlock(nn.Module):
+ """3D StyleGAN ResBlock for D."""
+
+ def __init__(
+ self,
+ in_channels,
+ filters,
+ activation_fn,
+ input_dim, # x.shape[-1], TODO
+ num_groups=32,
+ device="cpu",
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+
+ self.filters = filters
+ self.activation_fn = activation_fn
+
+ # TODO: figure out the input_dim
+
+ self.conv1 = nn.Conv3d(in_channels, self.filters, (3,3,3)) # need to init to xavier_uniform
+ self.norm1 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
+ self.avg_pool_with_t = nn.AvgPool3d((2,2,2))
+ self.conv2 = nn.Conv3d(in_channels, self.filters,(1,1,1), use_bias=False) # need to init to xavier_uniform
+ self.conv3 = nn.Conv3d(input_dim, self.filters, (3,3,3)) # need to init to xavier_uniform
+ self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
+
+ def forward(self, x):
+ residual = x
+ x = self.conv1(x)
+ x = self.norm1(x)
+ x = self.activation_fn(x)
+
+ x = self.avg_pool_with_t(x)
+ residual = self.avg_pool_with_t(residual)
+ residual = self.conv2(residual)
+ x = self.conv3(x)
+ x = self.norm2(x)
+ x = self.activation_fn(x)
+ out = (residual + x) / math.sqrt(2)
+ return out
+
+class StyleGANDiscriminator(nn.Module):
+ """StyleGAN Discriminator."""
+ def __init__(
+ self,
+ config,
+ image_size,
+ input_dim, # x.shape[-1]
+ discriminator_in_channels = 3,
+ discriminator_filters = 64,
+ discriminator_channel_multipliers = (2,4,4,4,4),
+ num_groups=32,
+ dtype = torch.bfloat16,
+ device="cpu",
+ ):
+ self.config = config
+ self.dtype = dtype
+
+ self.input_size = image_size
+ self.filters = discriminator_filters
+
+ self.activation_fn = nn.LeakyReLu(negative_slope=0.2)
+
+ self.channel_multipliers = discriminator_channel_multipliers
+
+
+
+ self.conv1 = nn.Conv3d(discriminator_in_channels, self.filters, (3, 3, 3)) # need to init to xavier_uniform
+
+ prev_filters = self.filters # record in_channels
+ self.num_blocks = len(self.channel_multipliers)
+ self.res_block_list = []
+ for i in range(self.num_blocks):
+ filters = self.filters * self.channel_multipliers[i]
+ self.res_block_list.append(ResBlock(prev_filters, filters, self.activation_fn)) # TODO
+ prev_filters = filters # update in_channels
+
+ self.conv2 = nn.Conv3d(prev_filters, prev_filters, (3,3,3)) # need to init to xavier_uniform
+
+ self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)
+
+ # TODO: what is the in_features
+ self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # need to init to xavier_uniform
+ self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # need to init to xavier_uniform
+
+ def forward(self, x):
+
+ x = self.conv1(x)
+ x = self.activation_fn(x)
+
+ for i in range(self.num_blocks):
+ x = self.res_block_list[i](x)
+
+ x = self.conv2(x)
+ x = self.norm1(x)
+ x = self.activation_fn(x)
+ x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?
+
+ x = self.linear1(x)
+ x = self.activation_fn(x)
+ x = self.linear2(x)
+ return x
diff --git a/opensora/models/vae/lpips.py b/opensora/models/vae/lpips.py
new file mode 100644
index 0000000..3a16111
--- /dev/null
+++ b/opensora/models/vae/lpips.py
@@ -0,0 +1,120 @@
+import torch
+import torch.nn as nn
+from torchvision import models
+from collections import namedtuple
+from taming.util import get_ckpt_path
+
+
+class LPIPS(nn.Module):
+ # Learned perceptual metric
+ def __init__(self, use_dropout=True):
+ super().__init__()
+ self.scaling_layer = ScalingLayer()
+ self.chns = [64, 128, 256, 512, 512] # vg16 features
+ self.net = vgg16(pretrained=True, requires_grad=False) # NOTE: TODO: need in_channels = 4 to use
+ self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout)
+ self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout)
+ self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout)
+ self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout)
+ self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout)
+ self.load_from_pretrained()
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def load_from_pretrained(self, name="vgg_lpips"):
+ ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips")
+ self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)
+ print("loaded pretrained LPIPS loss from {}".format(ckpt))
+
+ @classmethod
+ def from_pretrained(cls, name="vgg_lpips"):
+ if name != "vgg_lpips":
+ raise NotImplementedError
+ model = cls()
+ ckpt = get_ckpt_path(name)
+ model.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)
+ return model
+
+ def forward(self, input, target):
+ in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target))
+ outs0, outs1 = self.net(in0_input), self.net(in1_input)
+ feats0, feats1, diffs = {}, {}, {}
+ lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4]
+ for kk in range(len(self.chns)):
+ feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk])
+ diffs[kk] = (feats0[kk] - feats1[kk]) ** 2
+
+ res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))]
+ val = res[0]
+ for l in range(1, len(self.chns)):
+ val += res[l]
+ return val
+
+
+# SCH: TODO: this channel shift & scale may need to be changed
+class ScalingLayer(nn.Module):
+ def __init__(self):
+ super(ScalingLayer, self).__init__()
+ self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None])
+ self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None])
+
+ def forward(self, inp):
+ return (inp - self.shift) / self.scale
+
+
+class NetLinLayer(nn.Module):
+ """ A single linear layer which does a 1x1 conv """
+ def __init__(self, chn_in, chn_out=1, use_dropout=False):
+ super(NetLinLayer, self).__init__()
+ layers = [nn.Dropout(), ] if (use_dropout) else []
+ layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ]
+ self.model = nn.Sequential(*layers)
+
+
+class vgg16(torch.nn.Module):
+ def __init__(self, requires_grad=False, pretrained=True):
+ super(vgg16, self).__init__()
+ vgg_pretrained_features = models.vgg16(pretrained=pretrained).features
+ self.slice1 = torch.nn.Sequential()
+ self.slice2 = torch.nn.Sequential()
+ self.slice3 = torch.nn.Sequential()
+ self.slice4 = torch.nn.Sequential()
+ self.slice5 = torch.nn.Sequential()
+ self.N_slices = 5
+ for x in range(4):
+ self.slice1.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(4, 9):
+ self.slice2.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(9, 16):
+ self.slice3.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(16, 23):
+ self.slice4.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(23, 30):
+ self.slice5.add_module(str(x), vgg_pretrained_features[x])
+ if not requires_grad:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, X):
+ h = self.slice1(X)
+ h_relu1_2 = h
+ h = self.slice2(h)
+ h_relu2_2 = h
+ h = self.slice3(h)
+ h_relu3_3 = h
+ h = self.slice4(h)
+ h_relu4_3 = h
+ h = self.slice5(h)
+ h_relu5_3 = h
+ vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3'])
+ out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3)
+ return out
+
+
+def normalize_tensor(x,eps=1e-10):
+ norm_factor = torch.sqrt(torch.sum(x**2,dim=1,keepdim=True))
+ return x/(norm_factor+eps)
+
+
+def spatial_average(x, keepdim=True):
+ return x.mean([2,3],keepdim=keepdim)
\ No newline at end of file
diff --git a/opensora/models/vae/model_utils.py b/opensora/models/vae/model_utils.py
index b624288..8ad0bd9 100644
--- a/opensora/models/vae/model_utils.py
+++ b/opensora/models/vae/model_utils.py
@@ -5,131 +5,12 @@ from typing import Any, Optional, Sequence, Type
import torch.nn as nn
import numpy as np
import torch
-# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers
+from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers
from taming.modules.discriminator.model import NLayerDiscriminator, weights_init
"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models"""
-import torch
-from torchvision import models
-from collections import namedtuple
-from taming.util import get_ckpt_path
-
-
-class LPIPS(nn.Module):
- # Learned perceptual metric
- def __init__(self, use_dropout=True):
- super().__init__()
- self.scaling_layer = ScalingLayer()
- self.chns = [64, 128, 256, 512, 512] # vg16 features
- self.net = vgg16(pretrained=True, requires_grad=False)
- self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout)
- self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout)
- self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout)
- self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout)
- self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout)
- self.load_from_pretrained()
- for param in self.parameters():
- param.requires_grad = False
-
- def load_from_pretrained(self, name="vgg_lpips"):
- ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips")
- self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)
- print("loaded pretrained LPIPS loss from {}".format(ckpt))
-
- @classmethod
- def from_pretrained(cls, name="vgg_lpips"):
- if name != "vgg_lpips":
- raise NotImplementedError
- model = cls()
- ckpt = get_ckpt_path(name)
- model.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)
- return model
-
- def forward(self, input, target):
- in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target))
- outs0, outs1 = self.net(in0_input), self.net(in1_input)
- feats0, feats1, diffs = {}, {}, {}
- lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4]
- for kk in range(len(self.chns)):
- feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk])
- diffs[kk] = (feats0[kk] - feats1[kk]) ** 2
-
- res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))]
- val = res[0]
- for l in range(1, len(self.chns)):
- val += res[l]
- return val
-
-
-# SCH: TODO: this channel shift & scale may need to be changed
-class ScalingLayer(nn.Module):
- def __init__(self):
- super(ScalingLayer, self).__init__()
- self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None])
- self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None])
-
- def forward(self, inp):
- return (inp - self.shift) / self.scale
-
-
-class NetLinLayer(nn.Module):
- """ A single linear layer which does a 1x1 conv """
- def __init__(self, chn_in, chn_out=1, use_dropout=False):
- super(NetLinLayer, self).__init__()
- layers = [nn.Dropout(), ] if (use_dropout) else []
- layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ]
- self.model = nn.Sequential(*layers)
-
-
-class vgg16(torch.nn.Module):
- def __init__(self, requires_grad=False, pretrained=True):
- super(vgg16, self).__init__()
- vgg_pretrained_features = models.vgg16(pretrained=pretrained).features
- self.slice1 = torch.nn.Sequential()
- self.slice2 = torch.nn.Sequential()
- self.slice3 = torch.nn.Sequential()
- self.slice4 = torch.nn.Sequential()
- self.slice5 = torch.nn.Sequential()
- self.N_slices = 5
- for x in range(4):
- self.slice1.add_module(str(x), vgg_pretrained_features[x])
- for x in range(4, 9):
- self.slice2.add_module(str(x), vgg_pretrained_features[x])
- for x in range(9, 16):
- self.slice3.add_module(str(x), vgg_pretrained_features[x])
- for x in range(16, 23):
- self.slice4.add_module(str(x), vgg_pretrained_features[x])
- for x in range(23, 30):
- self.slice5.add_module(str(x), vgg_pretrained_features[x])
- if not requires_grad:
- for param in self.parameters():
- param.requires_grad = False
-
- def forward(self, X):
- h = self.slice1(X)
- h_relu1_2 = h
- h = self.slice2(h)
- h_relu2_2 = h
- h = self.slice3(h)
- h_relu3_3 = h
- h = self.slice4(h)
- h_relu4_3 = h
- h = self.slice5(h)
- h_relu5_3 = h
- vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3'])
- out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3)
- return out
-
-
-def normalize_tensor(x,eps=1e-10):
- norm_factor = torch.sqrt(torch.sum(x**2,dim=1,keepdim=True))
- return x/(norm_factor+eps)
-
-
-def spatial_average(x, keepdim=True):
- return x.mean([2,3],keepdim=keepdim)
## NOTE: not used since we only have 'GN'
# def get_norm_layer(norm_type, dtype):
@@ -189,48 +70,66 @@ class DiagonalGaussianDistribution(object):
def mode(self):
return self.mean
-# class VEA3DLoss(nn.Module):
-# def __init__(
-# self,
-# # disc_start,
-# logvar_init=0.0,
-# kl_weight=1.0,
-# pixelloss_weight=1.0,
-# perceptual_weight=1.0,
-# disc_loss="hinge"
-# ):
-# super().__init__()
-# assert disc_loss in ["hinge", "vanilla"]
-# self.kl_weight = kl_weight
-# self.pixel_weight = pixelloss_weight
-# # self.perceptual_loss = LPIPS().eval() # TODO
-# self.perceptual_weight = perceptual_weight
-# # output log variance
-# self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
+
+class VEA3DLoss(nn.Module):
+ def __init__(
+ self,
+ # disc_start,
+ logvar_init=0.0,
+ kl_weight=1.0,
+ pixelloss_weight=1.0,
+ perceptual_weight=1.0,
+ disc_loss="hinge",
+
+ ):
+ super().__init__()
+ assert disc_loss in ["hinge", "vanilla"]
+ self.kl_weight = kl_weight
+ self.pixel_weight = pixelloss_weight
+ self.perceptual_loss = LPIPS().eval()
+ self.perceptual_weight = perceptual_weight
+ # output log variance
+ self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
+
-# def forward(
-# self,
-# inputs,
-# reconstructions,
-# posteriors,
-# weights=None,
-# ):
-# rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())
+ def forward(
+ self,
+ inputs,
+ reconstructions,
+ posteriors,
+ # optimizer_idx,
+ # global_step,
+ weights=None,
+ ):
+ rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())
+ if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!
+ assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)}"
+ # SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frame
+ permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]
+ permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))
+ data_shape = permutated_input.size()
+ p_loss = self.perceptual_loss(
+ permutated_input.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous(),
+ permutated_rec.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous()
+ )
+ # SCH: shape back p_loss
+ permuted_p_loss = torch.permute(p_loss.reshape(data_shape[0], data_shape[1], 1, 1, 1), (0,2,1,3,4))
+ rec_loss = rec_loss + self.perceptual_weight * permuted_p_loss
-# nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar
-# weighted_nll_loss = nll_loss
-# if weights is not None:
-# weighted_nll_loss = weights*nll_loss
-# weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
-# nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
-# kl_loss = posteriors.kl()
-# kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
+ nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar
+ weighted_nll_loss = nll_loss
+ if weights is not None:
+ weighted_nll_loss = weights*nll_loss
+ weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
+ nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
+ kl_loss = posteriors.kl()
+ kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
-# loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later
-
-# return loss
+ loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later
-class VEA3DLoss(nn.Module):
+ return loss
+
+class VEA3DLossWithDiscriminator(nn.Module):
def __init__(
self,
# disc_start,
@@ -293,7 +192,8 @@ class VEA3DLoss(nn.Module):
weights=None,
):
rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())
- if self.perceptual_weight > 0:
+ if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!
+ assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)} "
# SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frame
permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]
permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))
@@ -327,4 +227,4 @@ class VEA3DLoss(nn.Module):
# # "{}/g_loss".format(split): g_loss.detach().mean(),
# }
- return loss
+ return loss
\ No newline at end of file
diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.py
index 54e000d..b9891a3 100644
--- a/opensora/models/vae/vae_3d.py
+++ b/opensora/models/vae/vae_3d.py
@@ -159,12 +159,13 @@ class Encoder(nn.Module):
# conv blocks handling
if self.conv_downsample:
t_stride = 2 if self.temporal_downsample[i] else 1
- self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(4, 4, 4), strides=(t_stride, 2, 2))) # SCH: should be same in_channel and out_channel
+ t_pad = 1 if self.temporal_downsample[i] else 0
+ self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(4, 4, 4), strides=(t_stride, 2, 2)), padding=(t_pad,1,1)) # SCH: should be same in_channel and out_channel
prev_filters = filters # update in_channels
# NOTE: downsample, dimensions T, H, W
- self.avg_pool_with_t = nn.AvgPool3d((2,2,2))
- self.avg_pool = nn.AvgPool3d((1,2,2))
+ self.avg_pool_with_t = nn.AvgPool3d((2,2,2), count_include_pad=False)
+ self.avg_pool = nn.AvgPool3d((1,2,2), count_include_pad=False)
# last layer res block
self.res_blocks = []
@@ -301,14 +302,15 @@ class Decoder(nn.Module):
# conv blocks handling
if i > 0:
t_stride = 2 if self.temporal_downsample[i - 1] else 1
+ t_kernel = 4 if self.temporal_downsample[i - 1] else 3 # SCH: hack to keep dimension same
if self.upsample == "deconv":
assert self.custom_conv_padding is None, ('Custom padding not implemented for ConvTranspose')
# SCH: append in front
- self.conv_blocks.insert(0,
- self.conv_t_fn(prev_filters, filters, kernel_size=(4, 4, 4), strides=(t_stride, 2, 2)))
+ self.conv_blocks.insert(0,
+ self.conv_t_fn(prev_filters, filters, kernel_size=(t_kernel, 4, 4), stride=(t_stride, 2, 2), padding=1))
prev_filters = filters # SCH: update in_channels
elif self.upsample == 'nearest+conv':
- # SCH: append in front
+ # SCH: append in front
self.conv_blocks.insert(0, self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3)))
prev_filters = filters # SCH: update in_channels
else: