Open-Sora: Democratizing Efficient Video Production for All
Python
[wip] debug vae
zhengzangw478b5850244b
diff --git a/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py b/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.pyindex 86f8711..31af832 100644--- a/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py+++ b/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py@@ -1,21 +1,20 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)-use_pipeline = True--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4-video_contains_first_frame = False+dataset = dict(+ type="VideoTextDataset",+ data_path=None,+ num_frames=16,+ frame_interval=3,+ image_size=(128, 128),+)# Define acceleration+num_workers = 4dtype = "bf16"grad_checkpoint = Trueplugin = "zero2"sp_size = 1+use_pipeline = True+video_contains_first_frame = False# Define modelvae_2d = dict(@@ -23,50 +22,49 @@ vae_2d = dict(from_pretrained="stabilityai/sd-vae-ft-ema",# SDXL)-model = dict(type="VAE_MAGVIT_V2",- in_out_channels = 4,- latent_embed_dim = 4,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 4,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = True,- encoder_double_z = True,- custom_conv_padding = None+ in_out_channels=4,+ latent_embed_dim=4,+ filters=128,+ num_res_blocks=4,+ channel_multipliers=(1, 2, 2, 4),+ temporal_downsample=(False, True, True),+ num_groups=32, # for nn.GroupNorm+ kl_embed_dim=4,+ activation_fn="swish",+ separate_first_frame_encoding=False,+ disable_space=True,+ encoder_double_z=True,+ custom_conv_padding=None,)discriminator = dict(type="DISCRIMINATOR_3D",- image_size = (16, 16), # NOTE: here image size is different- num_frames = num_frames,- in_channels = 4,- filters = 128,- use_pretrained=True, # NOTE: set to False only if we want to disable load+ image_size=(16, 16), # NOTE: here image size is different+ num_frames=16,+ in_channels=4,+ filters=128,+ use_pretrained=True, # NOTE: set to False only if we want to disable load# channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- channel_multipliers= (2,4,4) # since on intermediate layer dimension ofs z+ channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z)-# loss weights-logvar_init=0.0+# loss weights+logvar_init = 0.0kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0+discriminator_factor = 1.0 # for discriminator adversarial loss+generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1+lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001+discriminator_loss_type = "non-saturating"+generator_loss_type = "non-saturating"# discriminator_loss_type="hinge"# generator_loss_type="hinge"-discriminator_start = 100 # 8k data / (8*32) = 31 steps per epoch, use around 3 epochs-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use+discriminator_start = 100 # 8k data / (8*32) = 31 steps per epoch, use around 3 epochs+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't useema_decay = 0.999 # ema decay factor for generator@@ -76,11 +74,11 @@ outputs = "outputs"wandb = False# Training-''' NOTE:+""" NOTE:magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],3-6 epochs for pexel, from pexel observation its correct-'''+"""epochs = 200log_every = 1diff --git a/opensora/models/vae/model_utils.py b/opensora/models/vae/model_utils.pyindex 421dfea..c00f886 100644--- a/opensora/models/vae/model_utils.py+++ b/opensora/models/vae/model_utils.py@@ -1,12 +1,9 @@-import functools-import math-from typing import Any, Optional, Sequence, Type--import torch.nn as nnimport numpy as npimport torch-from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers-from taming.modules.discriminator.model import NLayerDiscriminator, weights_init+import torch.nn as nn++# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers+# from taming.modules.discriminator.model import NLayerDiscriminator, weights_initfrom einops import rearrange"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models"""@@ -17,23 +14,23 @@ from einops import rearrange# if norm_type == 'LN':# # supply a few args with partial function and pass the rest of the args when this norm_fn is called# norm_fn = functools.partial(nn.LayerNorm, dtype=dtype)-# elif norm_type == 'GN': #+# elif norm_type == 'GN': ## norm_fn = functools.partial(nn.GroupNorm, dtype=dtype)# elif norm_type is None:# norm_fn = lambda: (lambda x: x)# else:# raise NotImplementedError(f'norm_type: {norm_type}')# return norm_fn-+class DiagonalGaussianDistribution(object):def __init__(- self,- parameters,+ self,+ parameters,deterministic=False,):self.parameters = parameters- self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) # SCH: channels dim+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) # SCH: channels dimself.logvar = torch.clamp(self.logvar, -30.0, 20.0)self.deterministic = deterministicself.std = torch.exp(0.5 * self.logvar)@@ -48,40 +45,39 @@ class DiagonalGaussianDistribution(object):def kl(self, other=None):if self.deterministic:- return torch.Tensor([0.])+ return torch.Tensor([0.0])else:- if other is None: # SCH: assumes other is a standard normal distribution- return 0.5 * torch.sum(torch.pow(self.mean, 2)- + self.var - 1.0 - self.logvar,- dim=[1, 2, 3, 4])+ if other is None: # SCH: assumes other is a standard normal distribution+ return 0.5 * torch.sum(torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, dim=[1, 2, 3, 4])else:return 0.5 * torch.sum(torch.pow(self.mean - other.mean, 2) / other.var- + self.var / other.var - 1.0 - self.logvar + other.logvar,- dim=[1, 2, 3, 4])-- def nll(self, sample, dims=[1,2,3,4]): # TODO: what does this do?+ + self.var / other.var+ - 1.0+ - self.logvar+ + other.logvar,+ dim=[1, 2, 3, 4],+ )++ def nll(self, sample, dims=[1, 2, 3, 4]): # TODO: what does this do?if self.deterministic:- return torch.Tensor([0.])+ return torch.Tensor([0.0])logtwopi = np.log(2.0 * np.pi)- return 0.5 * torch.sum(- logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,- dim=dims)+ return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, dim=dims)- def mode(self): # SCH: used for vae inference?+ def mode(self): # SCH: used for vae inference?return self.mean-+class VEA3DLoss(nn.Module):def __init__(self,- # disc_start,- logvar_init=0.0,- kl_weight=1.0,+ # disc_start,+ logvar_init=0.0,+ kl_weight=1.0,pixelloss_weight=1.0,- perceptual_weight=0.1,+ perceptual_weight=0.1,disc_loss="hinge",-):super().__init__()assert disc_loss in ["hinge", "vanilla"]@@ -92,28 +88,27 @@ class VEA3DLoss(nn.Module):# output log varianceself.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)-def forward(self,inputs,reconstructions,posteriors,# optimizer_idx,- # global_step,+ # global_step,weights=None,):rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())- if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!+ if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)}"# SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frameB = inputs.shape[0]- inputs = rearrange(inputs,"B C T H W -> (B T) C H W")+ inputs = rearrange(inputs, "B C T H W -> (B T) C H W")reconstructions = rearrange(reconstructions, "B C T H W -> (B T) C H W")# permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]# permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))# data_shape = permutated_input.size()# p_loss = self.perceptual_loss(- # permutated_input.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous(),+ # permutated_input.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous(),# permutated_rec.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous()# )p_loss = self.perceptual_loss(inputs.contiguous(), reconstructions.contiguous())@@ -126,32 +121,32 @@ class VEA3DLoss(nn.Module):nll_loss = rec_loss / torch.exp(self.logvar) + self.logvarweighted_nll_loss = nll_lossif weights is not None:- weighted_nll_loss = weights*nll_loss+ weighted_nll_loss = weights * nll_lossweighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]kl_loss = posteriors.kl()kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]- loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later+ loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss laterreturn loss+class VEA3DLossWithDiscriminator(nn.Module):def __init__(self,- # disc_start,- logvar_init=0.0,- kl_weight=1.0,+ # disc_start,+ logvar_init=0.0,+ kl_weight=1.0,pixelloss_weight=1.0,- disc_num_layers=3,- disc_in_channels=3,- disc_factor=1.0,+ disc_num_layers=3,+ disc_in_channels=3,+ disc_factor=1.0,disc_weight=1.0,- perceptual_weight=1.0,- use_actnorm=False,+ perceptual_weight=1.0,+ use_actnorm=False,disc_conditional=False,disc_loss="hinge",-):super().__init__()assert disc_loss in ["hinge", "vanilla"]@@ -185,53 +180,53 @@ class VEA3DLossWithDiscriminator(nn.Module):# d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()# d_weight = d_weight * self.discriminator_weight# return d_weight-+def forward(self,inputs,reconstructions,posteriors,# optimizer_idx,- # global_step,- last_layer=None,- cond=None,+ # global_step,+ last_layer=None,+ cond=None,split="train",weights=None,):rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())- if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!- assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)} "+ if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!+ assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)} "# SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frame- permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]+ permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))data_shape = permutated_input.size()p_loss = self.perceptual_loss(- permutated_input.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous(),- permutated_rec.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous()+ permutated_input.reshape(-1, data_shape[-3], data_shape[-2], data_shape[-1]).contiguous(),+ permutated_rec.reshape(-1, data_shape[-3], data_shape[-2], data_shape[-1]).contiguous(),)# SCH: shape back p_loss- permuted_p_loss = torch.permute(p_loss.reshape(data_shape[0], data_shape[1], 1, 1, 1), (0,2,1,3,4))+ permuted_p_loss = torch.permute(p_loss.reshape(data_shape[0], data_shape[1], 1, 1, 1), (0, 2, 1, 3, 4))rec_loss = rec_loss + self.perceptual_weight * permuted_p_lossnll_loss = rec_loss / torch.exp(self.logvar) + self.logvarweighted_nll_loss = nll_lossif weights is not None:- weighted_nll_loss = weights*nll_loss+ weighted_nll_loss = weights * nll_lossweighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]kl_loss = posteriors.kl()kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]- loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later+ loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later- # log = {"{}/total_loss".format(split): loss.clone().detach().mean(),+ # log = {"{}/total_loss".format(split): loss.clone().detach().mean(),# "{}/logvar".format(split): self.logvar.detach(),- # "{}/kl_loss".format(split): kl_loss.detach().mean(),+ # "{}/kl_loss".format(split): kl_loss.detach().mean(),# "{}/nll_loss".format(split): nll_loss.detach().mean(),# "{}/rec_loss".format(split): rec_loss.detach().mean(),# # "{}/d_weight".format(split): d_weight.detach(),# # "{}/disc_factor".format(split): torch.tensor(disc_factor),# # "{}/g_loss".format(split): g_loss.detach().mean(),# }-- return loss\ No newline at end of file++ return lossdiff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex 85b5e4a..c220b7b 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -1,21 +1,19 @@import functools-from typing import Any, Dict, Tuple, Type, Union, Sequence, Optional-from absl import logging+import math+from typing import Tuple, Union++import numpy as npimport torch-import torch.nn as nn-import numpy as np-from numpy import typing as nptyping-from opensora.models.vae import model_utils-from opensora.registry import MODELS-from opensora.utils.ckpt_utils import load_checkpoint, find_model-from einops import rearrange, repeat, pack, unpack+import torch.nn as nnimport torch.nn.functional as F-import torchvision-from torchvision.models import VGG16_Weights-from opensora.models.vae.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers+from einops import pack, rearrange, repeat, unpackfrom torch import nn-import math-import os++from opensora.models.vae import model_utils+from opensora.models.vae.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers+from opensora.registry import MODELS+from opensora.utils.ckpt_utils import find_model, load_checkpoint+# from diffusers.models.modeling_utils import ModelMixin@@ -30,58 +28,68 @@ NOTE:!!! opensora read video into [B,C,T,H,W] format output"""-def cast_tuple(t, length = 1):+++def cast_tuple(t, length=1):return t if isinstance(t, tuple) else ((t,) * length)+def divisible_by(num, den):return (num % den) == 0+def is_odd(n):return not divisible_by(n, 2)-def pad_at_dim(t, pad, dim = -1, value = 0.):- dims_from_right = (- dim - 1) if dim < 0 else (t.ndim - dim - 1)- zeros = ((0, 0) * dims_from_right)- return F.pad(t, (*zeros, *pad), value = value)++def pad_at_dim(t, pad, dim=-1, value=0.0):+ dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)+ zeros = (0, 0) * dims_from_right+ return F.pad(t, (*zeros, *pad), value=value)+def pick_video_frame(video, frame_indices):"""get frame_indices from the video of [B, C, T, H, W] and return images of [B, C, H, W]"""batch, device = video.shape[0], video.device- video = rearrange(video, 'b c f ... -> b f c ...')- batch_indices = torch.arange(batch, device = device)- batch_indices = rearrange(batch_indices, 'b -> b 1')+ video = rearrange(video, "b c f ... -> b f c ...")+ batch_indices = torch.arange(batch, device=device)+ batch_indices = rearrange(batch_indices, "b -> b 1")images = video[batch_indices, frame_indices]- images = rearrange(images, 'b 1 c ... -> b c ...')+ images = rearrange(images, "b 1 c ... -> b c ...")return images+def exists(v):return v is not None+# ============== Generator Adversarial Loss Functions ==============def lecam_reg(real_pred, fake_pred, ema_real_pred, ema_fake_pred):- assert real_pred.ndim == 0 and ema_fake_pred.ndim == 0- lecam_loss = torch.mean(torch.pow(nn.ReLU()(real_pred - ema_fake_pred), 2))- lecam_loss += torch.mean(torch.pow(nn.ReLU()(ema_real_pred - fake_pred), 2))- return lecam_loss+ assert real_pred.ndim == 0 and ema_fake_pred.ndim == 0+ lecam_loss = torch.mean(torch.pow(nn.ReLU()(real_pred - ema_fake_pred), 2))+ lecam_loss += torch.mean(torch.pow(nn.ReLU()(ema_real_pred - fake_pred), 2))+ return lecam_loss+# Open-Sora-Plan# Very bad, do not usedef r1_penalty(real_img, real_pred):"""R1 regularization for discriminator. The core idea is to- penalize the gradient on real data alone: when the- generator distribution produces the true data distribution- and the discriminator is equal to 0 on the data manifold, the- gradient penalty ensures that the discriminator cannot create- a non-zero gradient orthogonal to the data manifold without- suffering a loss in the GAN game.-- Ref:- Eq. 9 in Which training methods for GANs do actually converge.- """+ penalize the gradient on real data alone: when the+ generator distribution produces the true data distribution+ and the discriminator is equal to 0 on the data manifold, the+ gradient penalty ensures that the discriminator cannot create+ a non-zero gradient orthogonal to the data manifold without+ suffering a loss in the GAN game.++ Ref:+ Eq. 9 in Which training methods for GANs do actually converge.+ """grad_real = torch.autograd.grad(outputs=real_pred.sum(), inputs=real_img, create_graph=True)[0]grad_penalty = grad_real.pow(2).view(grad_real.shape[0], -1).sum(1).mean()return grad_penalty+# Open-Sora-Plan# Implementation as described by https://arxiv.org/abs/1704.00028 # TODO: checkout the codesdef gradient_penalty_loss(discriminator, real_data, fake_data, weight=None):@@ -101,7 +109,7 @@ def gradient_penalty_loss(discriminator, real_data, fake_data, weight=None):alpha = real_data.new_tensor(torch.rand(batch_size, 1, 1, 1))# interpolate between real_data and fake_data- interpolates = alpha * real_data + (1. - alpha) * fake_data+ interpolates = alpha * real_data + (1.0 - alpha) * fake_datainterpolates = torch.autograd.Variable(interpolates, requires_grad=True)disc_interpolates = discriminator(interpolates)@@ -111,12 +119,13 @@ def gradient_penalty_loss(discriminator, real_data, fake_data, weight=None):grad_outputs=torch.ones_like(disc_interpolates),create_graph=True,retain_graph=True,- only_inputs=True)[0]+ only_inputs=True,+ )[0]if weight is not None:gradients = gradients * weight- gradients_penalty = ((gradients.norm(2, dim=1) - 1)**2).mean()+ gradients_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()if weight is not None:gradients_penalty /= torch.mean(weight)@@ -126,16 +135,16 @@ def gradient_penalty_loss(discriminator, real_data, fake_data, weight=None):def gradient_penalty_fn(images, output):# batch_size = images.shape[0]gradients = torch.autograd.grad(- outputs = output,- inputs = images,- grad_outputs = torch.ones(output.size(), device = images.device),- create_graph = True,- retain_graph = True,- only_inputs = True+ outputs=output,+ inputs=images,+ grad_outputs=torch.ones(output.size(), device=images.device),+ create_graph=True,+ retain_graph=True,+ only_inputs=True,)[0]- gradients = rearrange(gradients, 'b ... -> b (...)')- return ((gradients.norm(2, dim = 1) - 1) ** 2).mean()+ gradients = rearrange(gradients, "b ... -> b (...)")+ return ((gradients.norm(2, dim=1) - 1) ** 2).mean()# ============== Discriminator Adversarial Loss Functions ==============@@ -145,31 +154,32 @@ def hinge_d_loss(logits_real, logits_fake):d_loss = 0.5 * (loss_real + loss_fake)return d_loss+def vanilla_d_loss(logits_real, logits_fake):d_loss = 0.5 * (- torch.mean(torch.nn.functional.softplus(-logits_real)) +- torch.mean(torch.nn.functional.softplus(logits_fake)))+ torch.mean(torch.nn.functional.softplus(-logits_real)) + torch.mean(torch.nn.functional.softplus(logits_fake))+ )return d_loss+# from MAGVIT, used in place hof hinge_d_lossdef sigmoid_cross_entropy_with_logits(labels, logits):# The final formulation is: max(x, 0) - x * z + log(1 + exp(-abs(x)))zeros = torch.zeros_like(logits, dtype=logits.dtype)- condition = (logits >= zeros)+ condition = logits >= zerosrelu_logits = torch.where(condition, logits, zeros)neg_abs_logits = torch.where(condition, -logits, logits)return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))--def xavier_uniform_weight_init(m):if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):- nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain('relu'))+ nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain("relu"))if m.bias is not None:nn.init.zeros_(m.bias)# print("initialized module to xavier_uniform:", m)+def Sequential(*modules):modules = [*filter(exists, modules)]@@ -178,26 +188,28 @@ def Sequential(*modules):return nn.Sequential(*modules)+def SameConv2d(dim_in, dim_out, kernel_size):kernel_size = cast_tuple(kernel_size, 2)padding = [k // 2 for k in kernel_size]- return nn.Conv2d(dim_in, dim_out, kernel_size = kernel_size, padding = padding)+ return nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, padding=padding)-def adopt_weight(weight, global_step, threshold=0, value=0.):+def adopt_weight(weight, global_step, threshold=0, value=0.0):if global_step < threshold:weight = valuereturn weight+class CausalConv3d(nn.Module):def __init__(self,chan_in,chan_out,kernel_size: Union[int, Tuple[int, int, int]],- pad_mode = 'constant',- strides = None, # allow custom stride- **kwargs+ pad_mode="constant",+ strides=None, # allow custom stride+ **kwargs,):super().__init__()kernel_size = cast_tuple(kernel_size, 3)@@ -206,8 +218,8 @@ class CausalConv3d(nn.Module):assert is_odd(height_kernel_size) and is_odd(width_kernel_size)- dilation = kwargs.pop('dilation', 1)- stride = strides[0] if strides is not None else kwargs.pop('stride', 1)+ dilation = kwargs.pop("dilation", 1)+ stride = strides[0] if strides is not None else kwargs.pop("stride", 1)self.pad_mode = pad_modetime_pad = dilation * (time_kernel_size - 1) + (1 - stride)@@ -222,33 +234,33 @@ class CausalConv3d(nn.Module):# if padding == "same" and not all([pad == 1 for pad in padding]):# padding = "valid"dilation = (dilation, 1, 1)- self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride = stride, dilation = dilation, **kwargs)+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)def forward(self, x):- pad_mode = self.pad_mode if self.time_pad < x.shape[2] else 'constant'+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"- x = F.pad(x, self.time_causal_padding, mode = pad_mode)+ x = F.pad(x, self.time_causal_padding, mode=pad_mode)return self.conv(x)class ResBlock(nn.Module):def __init__(- self,- in_channels, # SCH: added- filters,- conv_fn,- activation_fn=nn.SiLU,- use_conv_shortcut=False,- num_groups=32,- device="cpu",- dtype=torch.bfloat16,+ self,+ in_channels, # SCH: added+ filters,+ conv_fn,+ activation_fn=nn.SiLU,+ use_conv_shortcut=False,+ num_groups=32,+ device="cpu",+ dtype=torch.bfloat16,):super().__init__()self.in_channels = in_channelsself.filters = filtersself.activate = activation_fn()self.use_conv_shortcut = use_conv_shortcut-+# SCH: MAGVIT uses GroupNorm by defaultself.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)self.conv1 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)@@ -256,11 +268,10 @@ class ResBlock(nn.Module):self.conv2 = conv_fn(self.filters, self.filters, kernel_size=(3, 3, 3), bias=False)if in_channels != filters:if self.use_conv_shortcut:- self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)+ self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)else:self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(1, 1, 1), bias=False)-def forward(self, x):# device, dtype = x.device, x.dtype# input_dim = x.shape[1]@@ -271,79 +282,88 @@ class ResBlock(nn.Module):x = self.norm2(x)x = self.activate(x)x = self.conv2(x)- if self.in_channels != self.filters: # SCH: ResBlock X->Y+ if self.in_channels != self.filters: # SCH: ResBlock X->Yresidual = self.conv3(residual)- return x + residual-+ return x + residual++# SCH: own implementation modified on top of: discriminator with anti-aliased downsampling (blurpool Zhang et al.)class BlurPool3D(nn.Module):def __init__(- self,- channels,- pad_type='reflect',- filt_size=3,- stride=2,- pad_off=0,- device="cpu",- dtype=torch.bfloat16,+ self,+ channels,+ pad_type="reflect",+ filt_size=3,+ stride=2,+ pad_off=0,+ device="cpu",+ dtype=torch.bfloat16,):super(BlurPool3D, self).__init__()self.filt_size = filt_sizeself.pad_off = pad_offself.pad_sizes = [- int(1.*(filt_size-1)/2), int(np.ceil(1.*(filt_size-1)/2)),- int(1.*(filt_size-1)/2), int(np.ceil(1.*(filt_size-1)/2)),- int(1.*(filt_size-1)/2), int(np.ceil(1.*(filt_size-1)/2))+ int(1.0 * (filt_size - 1) / 2),+ int(np.ceil(1.0 * (filt_size - 1) / 2)),+ int(1.0 * (filt_size - 1) / 2),+ int(np.ceil(1.0 * (filt_size - 1) / 2)),+ int(1.0 * (filt_size - 1) / 2),+ int(np.ceil(1.0 * (filt_size - 1) / 2)),]- self.pad_sizes = [pad_size+pad_off for pad_size in self.pad_sizes]+ self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]self.stride = stride- self.off = int((self.stride-1)/2.)+ self.off = int((self.stride - 1) / 2.0)self.channels = channels- if(self.filt_size==1):- a = np.array([1.,])- elif(self.filt_size==2):- a = np.array([1., 1.])- elif(self.filt_size==3):- a = np.array([1., 2., 1.])- elif(self.filt_size==4):- a = np.array([1., 3., 3., 1.])- elif(self.filt_size==5):- a = np.array([1., 4., 6., 4., 1.])- elif(self.filt_size==6):- a = np.array([1., 5., 10., 10., 5., 1.])- elif(self.filt_size==7):- a = np.array([1., 6., 15., 20., 15., 6., 1.])-- filt_2d = a[:,None]*a[None,:]+ if self.filt_size == 1:+ a = np.array(+ [+ 1.0,+ ]+ )+ elif self.filt_size == 2:+ a = np.array([1.0, 1.0])+ elif self.filt_size == 3:+ a = np.array([1.0, 2.0, 1.0])+ elif self.filt_size == 4:+ a = np.array([1.0, 3.0, 3.0, 1.0])+ elif self.filt_size == 5:+ a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])+ elif self.filt_size == 6:+ a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])+ elif self.filt_size == 7:+ a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])++ filt_2d = a[:, None] * a[None, :]filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)-- filt = filt_3d/torch.sum(filt_3d) # SCH: modified to it 3D- self.register_buffer('filt', filt[None,None,:,:,:].repeat((self.channels,1,1,1,1)))++ filt = filt_3d / torch.sum(filt_3d) # SCH: modified to it 3D+ self.register_buffer("filt", filt[None, None, :, :, :].repeat((self.channels, 1, 1, 1, 1)))self.pad = get_pad_layer(pad_type)(self.pad_sizes)def forward(self, inp):- if(self.filt_size==1):- if(self.pad_off==0):- return inp[:,:,::self.stride,::self.stride]+ if self.filt_size == 1:+ if self.pad_off == 0:+ return inp[:, :, :: self.stride, :: self.stride]else:- return self.pad(inp)[:,:,::self.stride,::self.stride]+ return self.pad(inp)[:, :, :: self.stride, :: self.stride]else:return F.conv3d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])-++def get_pad_layer(pad_type):- if(pad_type in ['refl','reflect']):+ if pad_type in ["refl", "reflect"]:PadLayer = nn.ReflectionPad3d- elif(pad_type in ['repl','replicate']):+ elif pad_type in ["repl", "replicate"]:PadLayer = nn.ReplicationPad3d- elif(pad_type=='zero'):+ elif pad_type == "zero":PadLayer = nn.ZeroPad3delse:- print('Pad type [%s] not recognized'%pad_type)+ print("Pad type [%s] not recognized" % pad_type)return PadLayer-+class ResBlockDown(nn.Module):"""3D StyleGAN ResBlock for D."""@@ -362,13 +382,19 @@ class ResBlockDown(nn.Module):self.activation_fn = activation_fn# SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code- self.conv1 = nn.Conv3d(in_channels, in_channels, (3,3,3), padding=1, device=device, dtype=dtype) # NOTE: init to xavier_uniform+ self.conv1 = nn.Conv3d(+ in_channels, in_channels, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniformself.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)self.blur = BlurPool3D(in_channels, device=device, dtype=dtype)- self.conv2 = nn.Conv3d(in_channels, self.filters,(1,1,1), bias=False, device=device, dtype=dtype) # NOTE: init to xavier_uniform- self.conv3 = nn.Conv3d(in_channels, self.filters, (3,3,3), padding=1, device=device, dtype=dtype) # NOTE: init to xavier_uniform+ self.conv2 = nn.Conv3d(+ in_channels, self.filters, (1, 1, 1), bias=False, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform+ self.conv3 = nn.Conv3d(+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniformself.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)# self.apply(xavier_uniform_weight_init)@@ -393,14 +419,16 @@ class ResBlockDown(nn.Module):# SCH: taken from Open Sora Plandef n_layer_disc_weights_init(m):classname = m.__class__.__name__- if classname.find('Conv') != -1:+ if classname.find("Conv") != -1:nn.init.normal_(m.weight.data, 0.0, 0.02)- elif classname.find('BatchNorm') != -1:+ elif classname.find("BatchNorm") != -1:nn.init.normal_(m.weight.data, 1.0, 0.02)nn.init.constant_(m.bias.data, 0)+class NLayerDiscriminator3D(nn.Module):"""Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""+def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):"""Construct a 3D PatchGAN discriminator@@ -428,46 +456,60 @@ class NLayerDiscriminator3D(nn.Module):nf_mult_prev = 1for n in range(1, n_layers): # gradually increase the number of filtersnf_mult_prev = nf_mult- nf_mult = min(2 ** n, 8)+ nf_mult = min(2**n, 8)sequence += [- nn.Conv3d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=(1,2,2), padding=padw, bias=use_bias),+ nn.Conv3d(+ ndf * nf_mult_prev,+ ndf * nf_mult,+ kernel_size=(kw, kw, kw),+ stride=(1, 2, 2),+ padding=padw,+ bias=use_bias,+ ),norm_layer(ndf * nf_mult),- nn.LeakyReLU(0.2, True)+ nn.LeakyReLU(0.2, True),]nf_mult_prev = nf_mult- nf_mult = min(2 ** n_layers, 8)+ nf_mult = min(2**n_layers, 8)sequence += [- nn.Conv3d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias),+ nn.Conv3d(+ ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias+ ),norm_layer(ndf * nf_mult),- nn.LeakyReLU(0.2, True)+ nn.LeakyReLU(0.2, True),]- sequence += [nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)] # output 1 channel prediction map+ sequence += [+ nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)+ ] # output 1 channel prediction mapself.main = nn.Sequential(*sequence)def forward(self, input):"""Standard forward."""return self.main(input)-++class StyleGANDiscriminatorBlur(nn.Module):"""StyleGAN Discriminator."""+"""- SCH: NOTE:+ SCH: NOTE:this discriminator requries the num_frames to be fixed during training;- in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!+ in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!"""+def __init__(self,- image_size = (128, 128),- num_frames = 17,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4),+ image_size=(128, 128),+ num_frames=17,+ in_channels=3,+ filters=128,+ channel_multipliers=(2, 4, 4, 4, 4),num_groups=32,- dtype = torch.bfloat16,+ dtype=torch.bfloat16,device="cpu",- ):+ ):super().__init__()self.dtype = dtype@@ -476,42 +518,53 @@ class StyleGANDiscriminatorBlur(nn.Module):self.activation_fn = nn.LeakyReLU(negative_slope=0.2)self.channel_multipliers = channel_multipliers- self.conv1 = nn.Conv3d(in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype) # NOTE: init to xavier_uniform+ self.conv1 = nn.Conv3d(+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform- prev_filters = self.filters # record in_channels+ prev_filters = self.filters # record in_channelsself.num_blocks = len(self.channel_multipliers)self.res_block_list = []for i in range(self.num_blocks):filters = self.filters * self.channel_multipliers[i]- self.res_block_list.append(ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(xavier_uniform_weight_init))- prev_filters = filters # update in_channels+ self.res_block_list.append(+ ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(+ xavier_uniform_weight_init+ )+ )+ prev_filters = filters # update in_channels- self.conv2 = nn.Conv3d(prev_filters, prev_filters, (3,3,3), padding=1, device=device, dtype=dtype) # NOTE: init to xavier_uniform+ self.conv2 = nn.Conv3d(+ prev_filters, prev_filters, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform# torch.nn.init.xavier_uniform_(self.conv2.weight)self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)- scale_factor = 2 ** self.num_blocks- if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage+ scale_factor = 2**self.num_blocks+ if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usagetime_scaled = num_frames // scale_factor + 1else:time_scaled = num_frames / scale_factor-- assert self.input_size[0] % scale_factor == 0, f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"- assert self.input_size[1] % scale_factor == 0, f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"++ assert (+ self.input_size[0] % scale_factor == 0+ ), f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"+ assert (+ self.input_size[1] % scale_factor == 0+ ), f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factorin_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)- self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform- self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform+ self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform+ self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform# self.apply(xavier_uniform_weight_init)def forward(self, x):-x = self.conv1(x)# print("discriminator aft conv:", x.size())x = self.activation_fn(x)-+for i in range(self.num_blocks):x = self.res_block_list[i](x)# print("discriminator resblock down:", x.size())@@ -520,7 +573,7 @@ class StyleGANDiscriminatorBlur(nn.Module):# print("discriminator aft conv2:", x.size())x = self.norm1(x)x = self.activation_fn(x)- x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?+ x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?# print("discriminator reshape:", x.size())x = self.linear1(x)@@ -530,21 +583,24 @@ class StyleGANDiscriminatorBlur(nn.Module):x = self.linear2(x)# print("discriminator aft linear2:", x.size())return x-++class Encoder(nn.Module):"""Encoder Blocks."""- def __init__(self,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm++ def __init__(+ self,+ filters=128,+ num_res_blocks=4,+ channel_multipliers=(1, 2, 2, 4),+ temporal_downsample=(False, True, True),+ num_groups=32, # for nn.GroupNorm# in_out_channels = 3, # SCH: added, in_channels at the start- latent_embed_dim = 512, # num channels for latent vector- # conv_downsample = False,- disable_spatial_downsample = False, # for vae pipeline- custom_conv_padding = None,- activation_fn = 'swish',+ latent_embed_dim=512, # num channels for latent vector+ # conv_downsample = False,+ disable_spatial_downsample=False, # for vae pipeline+ custom_conv_padding=None,+ activation_fn="swish",device="cpu",dtype=torch.bfloat16,):@@ -554,15 +610,15 @@ class Encoder(nn.Module):self.channel_multipliers = channel_multipliersself.temporal_downsample = temporal_downsampleself.num_groups = num_groups-+self.embedding_dim = latent_embed_dimself.disable_spatial_downsample = disable_spatial_downsample# self.conv_downsample = conv_downsampleself.custom_conv_padding = custom_conv_padding- if activation_fn == 'relu':+ if activation_fn == "relu":self.activation_fn = nn.ReLU- elif activation_fn == 'swish':+ elif activation_fn == "swish":self.activation_fn = nn.SiLUelse:raise NotImplementedError@@ -574,7 +630,7 @@ class Encoder(nn.Module):dtype=dtype,device=device,)-+self.block_args = dict(conv_fn=self.conv_fn,dtype=dtype,@@ -583,7 +639,7 @@ class Encoder(nn.Module):num_groups=self.num_groups,device=device,)-+# NOTE: moved to VAE for separate first frame processing# self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)@@ -593,33 +649,38 @@ class Encoder(nn.Module):self.conv_blocks = []filters = self.filters- prev_filters = filters # record for in_channels+ prev_filters = filters # record for in_channelsfor i in range(self.num_blocks):# resblock handling- filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels+ filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channelsblock_items = []for _ in range(self.num_res_blocks):block_items.append(ResBlock(prev_filters, filters, **self.block_args))- prev_filters = filters # update in_channels+ prev_filters = filters # update in_channelsself.block_res_blocks.append(block_items)-- if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x stride s x stride s++ if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x stride s x stride st_stride = 2 if self.temporal_downsample[i] else 1s_stride = 2 if not self.disable_spatial_downsample else 1- self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride))) # SCH: should be same in_channel and out_channel- prev_filters = filters # update in_channels-+ self.conv_blocks.append(+ self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride))+ ) # SCH: should be same in_channel and out_channel+ prev_filters = filters # update in_channels# last layer res blockself.res_blocks = []for _ in range(self.num_res_blocks):self.res_blocks.append(ResBlock(prev_filters, filters, **self.block_args))- prev_filters = filters # update in_channels+ prev_filters = filters # update in_channels# MAGVIT uses Group Normalization- self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, dtype=dtype, device=device) # SCH: separate <prev_filters> channels into 32 groups+ self.norm1 = nn.GroupNorm(+ self.num_groups, prev_filters, dtype=dtype, device=device+ ) # SCH: separate <prev_filters> channels into 32 groups- self.conv2 = nn.Conv3d(prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same")+ self.conv2 = nn.Conv3d(+ prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same"+ )def forward(self, x):# dtype, device = x.dtype, x.device@@ -637,7 +698,6 @@ class Encoder(nn.Module):if i < self.num_blocks - 1:x = self.conv_blocks[i](x)# print("encoder:", x.size())-for i in range(self.num_res_blocks):x = self.res_blocks[i](x)@@ -648,21 +708,24 @@ class Encoder(nn.Module):x = self.conv2(x)# print("encoder:", x.size())return x-++class Decoder(nn.Module):"""Decoder Blocks."""- def __init__(self,- latent_embed_dim = 512,- filters = 128,++ def __init__(+ self,+ latent_embed_dim=512,+ filters=128,# in_out_channels = 4,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm+ num_res_blocks=4,+ channel_multipliers=(1, 2, 2, 4),+ temporal_downsample=(False, True, True),+ num_groups=32, # for nn.GroupNorm# upsample = "nearest+conv", # options: "deconv", "nearest+conv"- disable_spatial_upsample = False, # for vae pipeline- custom_conv_padding = None,- activation_fn = 'swish',+ disable_spatial_upsample=False, # for vae pipeline+ custom_conv_padding=None,+ activation_fn="swish",device="cpu",dtype=torch.bfloat16,):@@ -674,19 +737,19 @@ class Decoder(nn.Module):self.channel_multipliers = channel_multipliersself.temporal_downsample = temporal_downsampleself.num_groups = num_groups-+# self.upsample = upsample- self.s_stride = 1 if disable_spatial_upsample else 2 # spatial stride+ self.s_stride = 1 if disable_spatial_upsample else 2 # spatial strideself.custom_conv_padding = custom_conv_padding# self.norm_type = self.config.vqvae.norm_type# self.num_remat_block = self.config.vqvae.get('num_dec_remat_blocks', 0)- if activation_fn == 'relu':+ if activation_fn == "relu":self.activation_fn = nn.ReLU- elif activation_fn == 'swish':- self.activation_fn = nn.SiLU+ elif activation_fn == "swish":+ self.activation_fn = nn.SiLUelse:- raise NotImplementedError+ raise NotImplementedErrorself.activate = self.activation_fn()self.conv_fn = functools.partial(@@ -722,34 +785,36 @@ class Decoder(nn.Module):# self.upsampler = nn.Upsample(scale_factor=(1,2,2))# ResBlocks and conv upsample- prev_filters = filters # SCH: in_channels+ prev_filters = filters # SCH: in_channelsself.block_res_blocks = []self.num_blocks = len(self.channel_multipliers)self.conv_blocks = []# SCH: reverse to keep track of the in_channels, but append also in a reverse direction- for i in reversed(range(self.num_blocks)):+ for i in reversed(range(self.num_blocks)):filters = self.filters * self.channel_multipliers[i]# resblock handlingblock_items = []for _ in range(self.num_res_blocks):block_items.append(ResBlock(prev_filters, filters, **self.block_args))- prev_filters = filters # SCH: update in_channels- self.block_res_blocks.insert(0, block_items) # SCH: append in front-+ prev_filters = filters # SCH: update in_channels+ self.block_res_blocks.insert(0, block_items) # SCH: append in front+# conv blocks with upsamplingif i > 0:t_stride = 2 if self.temporal_downsample[i - 1] else 1# SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2- self.conv_blocks.insert(0,- self.conv_fn(prev_filters, prev_filters * t_stride * self.s_stride * self.s_stride, kernel_size=(3,3,3))+ self.conv_blocks.insert(+ 0,+ self.conv_fn(+ prev_filters, prev_filters * t_stride * self.s_stride * self.s_stride, kernel_size=(3, 3, 3)+ ),)-+self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)# NOTE: moved to VAE for separate first frame processing# self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))-def forward(self,x,@@ -762,42 +827,49 @@ class Decoder(nn.Module):for i in range(self.num_res_blocks):x = self.res_blocks[i](x)# print("decoder:", x.size())- for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoder+ for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoderfor j in range(self.num_res_blocks):x = self.block_res_blocks[i][j](x)# print("decoder:", x.size())if i > 0:t_stride = 2 if self.temporal_downsample[i - 1] else 1# SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2- x = self.conv_blocks[i-1](x)- x = rearrange(x, "B (C ts hs ws) T H W -> B C (T ts) (H hs) (W ws)", ts=t_stride, hs=self.s_stride, ws=self.s_stride)+ x = self.conv_blocks[i - 1](x)+ x = rearrange(+ x,+ "B (C ts hs ws) T H W -> B C (T ts) (H hs) (W ws)",+ ts=t_stride,+ hs=self.s_stride,+ ws=self.s_stride,+ )# print("decoder:", x.size())x = self.norm1(x)x = self.activate(x)# NOTE: moved to VAE for separate first frame processing- # x = self.conv2(x)+ # x = self.conv2(x)return x@MODELS.register_module()-class VAE_3D_V2(nn.Module): # , ModelMixin- """The 3D VAE """+class VAE_3D_V2(nn.Module): # , ModelMixin+ """The 3D VAE"""+def __init__(- self,- latent_embed_dim = 256,- filters = 128,- num_res_blocks = 2,- separate_first_frame_encoding = False,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (True, True, False),- num_groups = 32, # for nn.GroupNorm- disable_space = False,- custom_conv_padding = None,- activation_fn = 'swish',- in_out_channels = 4,- kl_embed_dim = 64,- encoder_double_z = True,+ self,+ latent_embed_dim=256,+ filters=128,+ num_res_blocks=2,+ separate_first_frame_encoding=False,+ channel_multipliers=(1, 2, 2, 4),+ temporal_downsample=(True, True, False),+ num_groups=32, # for nn.GroupNorm+ disable_space=False,+ custom_conv_padding=None,+ activation_fn="swish",+ in_out_channels=4,+ kl_embed_dim=64,+ encoder_double_z=True,device="cpu",dtype="bf16",):@@ -809,12 +881,11 @@ class VAE_3D_V2(nn.Module): # , ModelMixinelif dtype == "fp16":dtype = torch.float16else:- raise NotImplementedError(f'dtype: {dtype}')-+ raise NotImplementedError(f"dtype: {dtype}")# ==== Model Params ====# self.image_size = cast_tuple(image_size, 2)- self.time_downsample_factor = 2**sum(temporal_downsample)+ self.time_downsample_factor = 2 ** sum(temporal_downsample)self.time_padding = self.time_downsample_factor - 1self.separate_first_frame_encoding = separate_first_frame_encoding@@ -825,50 +896,52 @@ class VAE_3D_V2(nn.Module): # , ModelMixin# ==== Model Initialization ====# encoder & decoder first and last conv layer- # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv- self.conv_in = CausalConv3d(in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device)+ # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv+ self.conv_in = CausalConv3d(+ in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device+ )self.conv_in_first_frame = nn.Identity()self.conv_out_first_frame = nn.Identity()if separate_first_frame_encoding:- self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3,3))- self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3,3))+ self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3, 3))+ self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3, 3))self.conv_out = CausalConv3d(filters, in_out_channels, 3, dtype=dtype, device=device)self.encoder = Encoder(- filters = filters,- num_res_blocks=num_res_blocks,- channel_multipliers=channel_multipliers,+ filters=filters,+ num_res_blocks=num_res_blocks,+ channel_multipliers=channel_multipliers,temporal_downsample=temporal_downsample,- num_groups = num_groups, # for nn.GroupNorm+ num_groups=num_groups, # for nn.GroupNorm# in_out_channels = in_out_channels,- latent_embed_dim = latent_embed_dim * 2 if encoder_double_z else latent_embed_dim,- # conv_downsample = conv_downsample,+ latent_embed_dim=latent_embed_dim * 2 if encoder_double_z else latent_embed_dim,+ # conv_downsample = conv_downsample,disable_spatial_downsample=disable_space,- custom_conv_padding = custom_conv_padding,- activation_fn = activation_fn,- device = device,- dtype = dtype,+ custom_conv_padding=custom_conv_padding,+ activation_fn=activation_fn,+ device=device,+ dtype=dtype,)self.decoder = Decoder(- latent_embed_dim = latent_embed_dim,- filters = filters,- # in_out_channels = in_out_channels,- num_res_blocks = num_res_blocks,- channel_multipliers = channel_multipliers,- temporal_downsample = temporal_downsample,- num_groups = num_groups, # for nn.GroupNorm+ latent_embed_dim=latent_embed_dim,+ filters=filters,+ # in_out_channels = in_out_channels,+ num_res_blocks=num_res_blocks,+ channel_multipliers=channel_multipliers,+ temporal_downsample=temporal_downsample,+ num_groups=num_groups, # for nn.GroupNorm# upsample = upsample, # options: "deconv", "nearest+conv"disable_spatial_upsample=disable_space,- custom_conv_padding = custom_conv_padding,- activation_fn = activation_fn,- device = device,- dtype = dtype,+ custom_conv_padding=custom_conv_padding,+ activation_fn=activation_fn,+ device=device,+ dtype=dtype,)-+if encoder_double_z:- self.quant_conv = nn.Conv3d(2*latent_embed_dim, 2*kl_embed_dim, 1, device=device, dtype=dtype)+ self.quant_conv = nn.Conv3d(2 * latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)else:- self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1, device=device, dtype=dtype)+ self.quant_conv = nn.Conv3d(latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)def get_latent_size(self, input_size):@@ -876,33 +949,33 @@ class VAE_3D_V2(nn.Module): # , ModelMixinassert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"input_size = [input_size[i] // self.patch_size[i] for i in range(3)]return input_size-+def encode(self,video,- video_contains_first_frame = True,+ video_contains_first_frame=True,):encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame# whether to pad video or notif video_contains_first_frame:video_len = video.shape[2]- video = pad_at_dim(video, (self.time_padding, 0), value = 0., dim = 2)+ video = pad_at_dim(video, (self.time_padding, 0), value=0.0, dim=2)video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]# print("pre-encoder:", video.size())# NOTE: moved encoder conv1 here for separate first frame encodingif encode_first_frame_separately:- pad, first_frame, video = unpack(video, video_packed_shape, 'b c * h w')+ pad, first_frame, video = unpack(video, video_packed_shape, "b c * h w")first_frame = self.conv_in_first_frame(first_frame)video = self.conv_in(video)# print("pre-encoder:", video.size())if encode_first_frame_separately:- video, _ = pack([first_frame, video], 'b c * h w')- video = pad_at_dim(video, (self.time_padding, 0), dim = 2)+ video, _ = pack([first_frame, video], "b c * h w")+ video = pad_at_dim(video, (self.time_padding, 0), dim=2)encoded_feature = self.encoder(video)@@ -915,16 +988,15 @@ class VAE_3D_V2(nn.Module): # , ModelMixin# print("after encoder moments:", moments.size())return posterior-+def decode(self,z,- video_contains_first_frame = True,- ):+ video_contains_first_frame=True,+ ):# dtype = z.dtypedecode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame-z = self.post_quant_conv(z)# print("pre decoder, post quant conv:", z.size())@@ -933,25 +1005,28 @@ class VAE_3D_V2(nn.Module): # , ModelMixin# SCH: moved decoder last conv layer here for separate first frame decodingif decode_first_frame_separately:- left_pad, dec_ff, dec = dec[:, :, :self.time_padding], dec[:, :, self.time_padding], dec[:, :, (self.time_padding + 1):]+ left_pad, dec_ff, dec = (+ dec[:, :, : self.time_padding],+ dec[:, :, self.time_padding],+ dec[:, :, (self.time_padding + 1) :],+ )out = self.conv_out(dec)outff = self.conv_out_first_frame(dec_ff)- video, _ = pack([outff, out], 'b c * h w')+ video, _ = pack([outff, out], "b c * h w")else:video = self.conv_out(dec)# if video were padded, remove paddingif video_contains_first_frame:- video = video[:, :, self.time_padding:]+ video = video[:, :, self.time_padding :]# print("conv out:", video.size())return video-+def get_last_layer(self):# CausalConv3d wraps the convreturn self.conv_out.conv.weight--+# def parameters(self):# return [# *self.conv_in.parameters(),@@ -966,22 +1041,22 @@ class VAE_3D_V2(nn.Module): # , ModelMixin# def disc_parameters(self):# return self.discriminator.parameters()-+def forward(self,video,sample_posterior=True,- video_contains_first_frame = True,+ video_contains_first_frame=True,# split = "train",-- ):-+ ):batch, channels, frames = video.shape[:3]- assert divisible_by(frames - int(video_contains_first_frame), self.time_downsample_factor), f'number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}'+ assert divisible_by(+ frames - int(video_contains_first_frame), self.time_downsample_factor+ ), f"number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}"posterior = self.encode(video,- video_contains_first_frame = video_contains_first_frame,+ video_contains_first_frame=video_contains_first_frame,)if sample_posterior:@@ -989,12 +1064,8 @@ class VAE_3D_V2(nn.Module): # , ModelMixinelse:z = posterior.mode()+ recon_video = self.decode(z, video_contains_first_frame=video_contains_first_frame)- recon_video = self.decode(- z,- video_contains_first_frame = video_contains_first_frame- )-return recon_video, posterior@@ -1002,12 +1073,12 @@ class VEALoss(nn.Module):def __init__(self,logvar_init=0.0,- perceptual_loss_weight = 0.1,- kl_loss_weight = 0.000001,+ perceptual_loss_weight=0.1,+ kl_loss_weight=0.000001,# vgg=None,# vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,- device = "cpu",- dtype = "bf16"+ device="cpu",+ dtype="bf16",):super().__init__()@@ -1017,8 +1088,8 @@ class VEALoss(nn.Module):elif dtype == "fp16":dtype = torch.float16else:- raise NotImplementedError(f'dtype: {dtype}')-+ raise NotImplementedError(f"dtype: {dtype}")+# KL Lossself.kl_loss_weight = kl_loss_weight# Perceptual Loss@@ -1034,7 +1105,6 @@ class VEALoss(nn.Module):# )# vgg.classifier = Sequential(*vgg.classifier[:-2])# self.vgg = vgg.to(device, dtype).eval() # SCH: added eval-def forward(self,@@ -1042,8 +1112,8 @@ class VEALoss(nn.Module):recon_video,posterior,nll_weights=None,- split = "train",- ):+ split="train",+ ):video = rearrange(video, "b c t h w -> (b t) c h w").contiguous()recon_video = rearrange(recon_video, "b c t h w -> (b t) c h w").contiguous()@@ -1052,13 +1122,13 @@ class VEALoss(nn.Module):# perceptual lossif self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:- # handle channels+ # handle channelschannels = video.shape[1]- assert channels in {1,3,4}+ assert channels in {1, 3, 4}if channels == 1:- input_vgg_input = repeat(video, 'b 1 h w -> b c h w', c = 3)- recon_vgg_input = repeat(recon_video, 'b 1 h w -> b c h w', c = 3)- elif channels == 4: # SCH: take the first 3 for perceptual loss calc+ input_vgg_input = repeat(video, "b 1 h w -> b c h w", c=3)+ recon_vgg_input = repeat(recon_video, "b 1 h w -> b c h w", c=3)+ elif channels == 4: # SCH: take the first 3 for perceptual loss calcinput_vgg_input = video[:, :3]recon_vgg_input = recon_video[:, :3]else:@@ -1067,7 +1137,7 @@ class VEALoss(nn.Module):perceptual_loss = self.perceptual_loss_fn(input_vgg_input, recon_vgg_input)recon_loss = recon_loss + self.perceptual_loss_weight * perceptual_loss-+nll_loss = recon_loss / torch.exp(self.logvar) + self.logvarweighted_nll_loss = nll_lossif nll_weights is not None:@@ -1118,14 +1188,14 @@ class VEALoss(nn.Module):# }# return nll_loss, weighted_kl_loss-+class AdversarialLoss(nn.Module):def __init__(self,- discriminator_factor = 1.0,- discriminator_start = 50001,- generator_factor = 0.5,+ discriminator_factor=1.0,+ discriminator_start=50001,+ generator_factor=0.5,generator_loss_type="non-saturating",):super().__init__()@@ -1133,10 +1203,12 @@ class AdversarialLoss(nn.Module):self.discriminator_start = discriminator_startself.generator_factor = generator_factorself.generator_loss_type = generator_loss_type-+def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added create_graph=True+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[+ 0+ ] # SCH: TODO: debug added create_graph=Trued_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()d_weight = d_weight * self.generator_factor@@ -1148,8 +1220,8 @@ class AdversarialLoss(nn.Module):nll_loss,last_layer,global_step,- is_training = True,- ):+ is_training=True,+ ):# NOTE: following MAGVIT to allow non_saturatingassert self.generator_loss_type in ["hinge", "vanilla", "non-saturating"]@@ -1157,15 +1229,13 @@ class AdversarialLoss(nn.Module):gen_loss = -torch.mean(fake_logits)elif self.generator_loss_type == "non-saturating":gen_loss = torch.mean(- sigmoid_cross_entropy_with_logits(- labels=torch.ones_like(fake_logits), logits=fake_logits)- )+ sigmoid_cross_entropy_with_logits(labels=torch.ones_like(fake_logits), logits=fake_logits)+ )else:raise ValueError("Generator loss {} not supported".format(self.generator_loss_type))-if self.discriminator_factor is not None and self.discriminator_factor > 0.0:- try:+ try:d_weight = self.calculate_adaptive_weight(nll_loss, gen_loss, last_layer)except RuntimeError:assert not is_training@@ -1177,55 +1247,49 @@ class AdversarialLoss(nn.Module):weighted_gen_loss = d_weight * disc_factor * gen_lossreturn weighted_gen_loss-++class LeCamEMA:- def __init__(- self,- ema_real = 0.0,- ema_fake = 0.0,- decay=0.999,- dtype=torch.bfloat16,- device="cpu"- ):+ def __init__(self, ema_real=0.0, ema_fake=0.0, decay=0.999, dtype=torch.bfloat16, device="cpu"):self.decay = decayself.ema_real = torch.tensor(ema_real).to(device, dtype)self.ema_fake = torch.tensor(ema_fake).to(device, dtype)-+def update(self, ema_real, ema_fake):- self.ema_real = self.ema_real * self.decay + ema_real * (1-self.decay)- self.ema_fake = self.ema_fake * self.decay + ema_fake * (1-self.decay)+ self.ema_real = self.ema_real * self.decay + ema_real * (1 - self.decay)+ self.ema_fake = self.ema_fake * self.decay + ema_fake * (1 - self.decay)def get(self):return self.ema_real, self.ema_fake-++class DiscriminatorLoss(nn.Module):def __init__(self,- discriminator_factor = 1.0,- discriminator_start = 50001,+ discriminator_factor=1.0,+ discriminator_start=50001,discriminator_loss_type="non-saturating",lecam_loss_weight=None,- gradient_penalty_loss_weight=None, # SCH: following MAGVIT config.vqgan.grad_penalty_cost+ gradient_penalty_loss_weight=None, # SCH: following MAGVIT config.vqgan.grad_penalty_cost):super().__init__()-+assert discriminator_loss_type in ["hinge", "vanilla", "non-saturating"]self.discriminator_factor = discriminator_factorself.discriminator_start = discriminator_startself.lecam_loss_weight = lecam_loss_weightself.gradient_penalty_loss_weight = gradient_penalty_loss_weightself.discriminator_loss_type = discriminator_loss_type-def forward(self,real_logits,fake_logits,global_step,- lecam_ema_real = None,- lecam_ema_fake = None,- real_video = None,- split = "train",+ lecam_ema_real=None,+ lecam_ema_fake=None,+ real_video=None,+ split="train",):if self.discriminator_factor is not None and self.discriminator_factor > 0.0:disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)@@ -1241,7 +1305,8 @@ class DiscriminatorLoss(nn.Module):real_loss = 0.0if fake_logits is not None:fake_loss = sigmoid_cross_entropy_with_logits(- labels=torch.zeros_like(fake_logits), logits=fake_logits)+ labels=torch.zeros_like(fake_logits), logits=fake_logits+ )else:fake_loss = 0.0disc_loss = 0.5 * (torch.mean(real_loss) + torch.mean(fake_loss))@@ -1259,18 +1324,16 @@ class DiscriminatorLoss(nn.Module):if self.lecam_loss_weight is not None and self.lecam_loss_weight > 0.0:real_pred = torch.mean(real_logits)fake_pred = torch.mean(fake_logits)- lecam_loss = lecam_reg(real_pred, fake_pred,- lecam_ema_real,- lecam_ema_fake)+ lecam_loss = lecam_reg(real_pred, fake_pred, lecam_ema_real, lecam_ema_fake)lecam_loss = lecam_loss * self.lecam_loss_weight-+gradient_penalty = torch.tensor(0.0)if self.gradient_penalty_loss_weight is not None and self.gradient_penalty_loss_weight > 0.0:assert real_video is not Nonegradient_penalty = gradient_penalty_fn(real_video, real_logits)# gradient_penalty = r1_penalty(real_video, real_logits) # MAGVIT uses r1 penaltygradient_penalty *= self.gradient_penalty_loss_weight-+# discriminator_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty# log = {@@ -1281,6 +1344,7 @@ class DiscriminatorLoss(nn.Module):return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)+@MODELS.register_module("VAE_MAGVIT_V2")def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):model = VAE_3D_V2(**kwargs)@@ -1288,11 +1352,12 @@ def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):load_checkpoint(model, from_pretrained, model_name="model")return model+@MODELS.register_module("DISCRIMINATOR_3D")def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):model = StyleGANDiscriminatorBlur(**kwargs).apply(xavier_uniform_weight_init)- # model = StyleGANDiscriminator(**kwargs).apply(xavier_uniform_weight_init) # SCH: DEBUG: to change back- # model = NLayerDiscriminator3D(input_nc=3, n_layers=3,).apply(n_layer_disc_weights_init)+ # model = StyleGANDiscriminator(**kwargs).apply(xavier_uniform_weight_init) # SCH: DEBUG: to change back+ # model = NLayerDiscriminator3D(input_nc=3, n_layers=3,).apply(n_layer_disc_weights_init)if from_pretrained is not None:if use_pretrained:if inflate_from_2d:@@ -1302,28 +1367,27 @@ def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrainedprint(f"loaded discriminator")else:print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")-+return modeldef load_checkpoint_with_inflation(model, ckpt_path):"""- pre-train using image, then inflate to 3D videos+ pre-train using image, then inflate to 3D videos"""if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):state_dict = find_model(ckpt_path)with torch.no_grad():for key in state_dict:if key in model:- # central inflation- if state_dict[key].size() == model[key][:, :, 0, :, :].size():- # temporal dimension- val = torch.zeros_like(model[key])- centre = int(model[key].size(2) // 2)- val[:, :, centre, :, :] = state_dict[key]+ # central inflation+ if state_dict[key].size() == model[key][:, :, 0, :, :].size():+ # temporal dimension+ val = torch.zeros_like(model[key])+ centre = int(model[key].size(2) // 2)+ val[:, :, centre, :, :] = state_dict[key]missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)print(f"Missing keys: {missing_keys}")print(f"Unexpected keys: {unexpected_keys}")else:- load_checkpoint(model, ckpt_path) # use the default function-\ No newline at end of file+ load_checkpoint(model, ckpt_path) # use the default functiondiff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pyindex 4967935..c65c3b3 100644--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@ -1,10 +1,8 @@-from copy import deepcopy+import os+from glob import globimport colossalaiimport torch-import torch.nn as nn-from torch.optim import Optimizer-from torch.optim.lr_scheduler import _LRSchedulerimport torch.distributed as distimport wandbfrom colossalai.booster import Booster@@ -12,11 +10,8 @@ from colossalai.booster.plugin import LowLevelZeroPluginfrom colossalai.cluster import DistCoordinatorfrom colossalai.nn.optimizer import HybridAdamfrom colossalai.utils import get_current_device-from tqdm import tqdm-import osfrom einops import rearrange-import numpy as np-from glob import glob+from tqdm import tqdmfrom opensora.acceleration.checkpoint import set_grad_checkpointfrom opensora.acceleration.parallel_states import (@@ -25,28 +20,17 @@ from opensora.acceleration.parallel_states import (set_sequence_parallel_group,)from opensora.acceleration.plugin import ZeroSeqParallelPlugin-from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader-from opensora.registry import MODELS, SCHEDULERS, build_module-from opensora.utils.ckpt_utils import create_logger, load_json, save_json, load, model_sharding, record_model_param_shape, save+from opensora.datasets import prepare_dataloader, prepare_variable_dataloader+from opensora.models.vae.vae_3d_v2 import AdversarialLoss, DiscriminatorLoss, LeCamEMA, VEALoss, pad_at_dim+from opensora.registry import DATASETS, MODELS, build_module+from opensora.utils.ckpt_utils import create_logger, load_json, save_jsonfrom opensora.utils.config_utils import (create_experiment_workspace,create_tensorboard_writer,parse_configs,save_training_config,)-from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, requires_grad, to_torch_dtype-from opensora.utils.train_utils import update_ema, MaskGenerator-from opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, LeCamEMA, pad_at_dim----# efficiency-# from torch.profiler import profile, record_function, ProfilerActivity--def trace_handler(p):- output = p.key_averages().table(sort_by="self_cuda_time_total", row_limit=5)- print(output)- # p.export_chrome_trace("/home/shenchenhui/Open-Sora-dev/outputs/traces/trace_" + str(p.step_num) + ".json")+from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, to_torch_dtypedef main():@@ -59,20 +43,20 @@ def main():# 2. runtime variables & colossalai launch# ======================================================assert torch.cuda.is_available(), "Training currently requires at least one GPU."-+# 2.1. colossalai init distributed trainingcolossalai.launch_from_torch({})coordinator = DistCoordinator()exp_dir = None- if coordinator.is_master(): # only create directory for master+ if coordinator.is_master(): # only create directory for masterexp_name, exp_dir = create_experiment_workspace(cfg)save_training_config(cfg._cfg_dict, exp_dir)dist.barrier()# get exp dir for non-master processif exp_dir is None:- experiment_index = len(glob(f"{cfg.outputs}/*"))-1+ experiment_index = len(glob(f"{cfg.outputs}/*")) - 1model_name = cfg.model["type"].replace("/", "-")exp_name = f"{experiment_index:03d}-F{cfg.num_frames}S{cfg.frame_interval}-{model_name}"exp_dir = f"{cfg.outputs}/{exp_name}"@@ -123,31 +107,30 @@ def main():# ======================================================# 3. build dataset and dataloader# ======================================================- dataset = DatasetFromCSV(- cfg.data_path,- transform=(- get_transforms_video(cfg.image_size[0])- if not cfg.use_image_transform- else get_transforms_image(cfg.image_size[0])- ),- num_frames=cfg.num_frames,- frame_interval=cfg.frame_interval,- root=cfg.root,- )-- dataloader = prepare_dataloader(- dataset,+ dataset = build_module(cfg.dataset, DATASETS)+ logger.info(f"Dataset contains {len(dataset)} samples.")+ dataloader_args = dict(+ dataset=dataset,batch_size=cfg.batch_size,num_workers=cfg.num_workers,+ seed=cfg.seed,shuffle=True,drop_last=True,pin_memory=True,process_group=get_data_parallel_group(),)- logger.info(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")-- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size- logger.info(f"Total batch size: {total_batch_size}")+ # TODO: use plugin's prepare dataloader+ if cfg.bucket_config is None:+ dataloader = prepare_dataloader(**dataloader_args)+ else:+ dataloader = prepare_variable_dataloader(+ bucket_config=cfg.bucket_config,+ num_bucket_build_workers=cfg.num_bucket_build_workers,+ **dataloader_args,+ )+ if cfg.dataset.type == "VideoTextDataset":+ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size+ logger.info(f"Total batch size: {total_batch_size}")# ======================================================# 4. build model@@ -163,7 +146,8 @@ def main():logger.info(f"Trainable vae params: {format_numel_str(vae_numel_trainable)}, Total model params: {format_numel_str(vae_numel)}")-+ breakpoint()+discriminator = build_module(cfg.discriminator, MODELS, device=device)discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)logger.info(@@ -175,12 +159,11 @@ def main():# 4.3. move to deviceif cfg.get("use_pipeline") == True:- vae_2d.to(device, dtype).eval() # eval mode, not training!+ vae_2d.to(device, dtype).eval() # eval mode, not training!vae = vae.to(device, dtype)discriminator = discriminator.to(device, dtype)-# 4.5. setup optimizer# vae optimizeroptimizer = HybridAdam(@@ -200,7 +183,6 @@ def main():vae.train()discriminator.train()-# =======================================================# 5. boost model for distributed training with colossalai# =======================================================@@ -212,13 +194,11 @@ def main():num_steps_per_epoch = len(dataloader)logger.info("Boost vae for distributed training")-discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(model=discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler)logger.info("Boost discriminator for distributed training")-# =======================================================# 6. training loop# =======================================================@@ -226,7 +206,6 @@ def main():running_loss = 0.0running_disc_loss = 0.0-# 6.1. resume trainingif cfg.load is not None:logger.info("Loading checkpoint")@@ -244,11 +223,17 @@ def main():if cfg.lecam_loss_weight is not None and os.path.exists(lecam_path):lecam_state = load_json(lecam_path)lecam_ema_real, lecam_ema_fake = lecam_state["lecam_ema_real"], lecam_state["lecam_ema_fake"]- lecam_ema = LeCamEMA(decay=cfg.ema_decay, ema_real=lecam_ema_real, ema_fake=lecam_ema_fake, dtype=dtype, device=device)+ lecam_ema = LeCamEMA(+ decay=cfg.ema_decay, ema_real=lecam_ema_real, ema_fake=lecam_ema_fake, dtype=dtype, device=device+ )running_states = load_json(os.path.join(cfg.load, "running_states.json"))dist.barrier()- start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]+ start_epoch, start_step, sampler_start_idx = (+ running_states["epoch"],+ running_states["step"],+ running_states["sample_start_index"],+ )logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")@@ -257,25 +242,25 @@ def main():# 6.2 Define loss functionsvae_loss_fn = VEALoss(logvar_init=cfg.logvar_init,- perceptual_loss_weight = cfg.perceptual_loss_weight,- kl_loss_weight = cfg.kl_loss_weight,+ perceptual_loss_weight=cfg.perceptual_loss_weight,+ kl_loss_weight=cfg.kl_loss_weight,device=device,dtype=dtype,)adversarial_loss_fn = AdversarialLoss(- discriminator_factor = cfg.discriminator_factor,- discriminator_start = cfg.discriminator_start,- generator_factor = cfg.generator_factor,- generator_loss_type = cfg.generator_loss_type,+ discriminator_factor=cfg.discriminator_factor,+ discriminator_start=cfg.discriminator_start,+ generator_factor=cfg.generator_factor,+ generator_loss_type=cfg.generator_loss_type,)disc_loss_fn = DiscriminatorLoss(- discriminator_factor = cfg.discriminator_factor,- discriminator_start = cfg.discriminator_start,- discriminator_loss_type = cfg.discriminator_loss_type,- lecam_loss_weight = cfg.lecam_loss_weight,- gradient_penalty_loss_weight = cfg.gradient_penalty_loss_weight,+ discriminator_factor=cfg.discriminator_factor,+ discriminator_start=cfg.discriminator_start,+ discriminator_loss_type=cfg.discriminator_loss_type,+ lecam_loss_weight=cfg.lecam_loss_weight,+ gradient_penalty_loss_weight=cfg.gradient_penalty_loss_weight,)# 6.3. training loop@@ -288,14 +273,11 @@ def main():disc_time_padding = 0video_contains_first_frame = cfg.video_contains_first_frame-for epoch in range(start_epoch, cfg.epochs):dataloader.sampler.set_epoch(epoch)dataloader_iter = iter(dataloader)logger.info(f"Beginning epoch {epoch}...")--with tqdm(range(start_step, num_steps_per_epoch),desc=f"Epoch {epoch}",@@ -303,7 +285,6 @@ def main():total=num_steps_per_epoch,initial=start_step,) as pbar:-# with profile(# activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],# schedule=torch.profiler.schedule(@@ -317,199 +298,203 @@ def main():# record_shapes=True,# profile_memory=True,# ) as p: # trace efficiency-- for step in pbar:-- # with profile(- # activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],- # with_stack=True,- # ) as p: # trace efficiency-- # SCH: calc global step at the start- global_step = epoch * num_steps_per_epoch + step-- batch = next(dataloader_iter)- x = batch["video"].to(device, dtype) # [B, C, T, H, W]-- # supprt for image or video inputs- assert x.ndim in {4, 5}, f"received input of {x.ndim} dimensions" # either image or video- assert x.shape[-2:] == cfg.image_size, f"received input size {x.shape[-2:]}, but config image size is {cfg.image_size}"- is_image = x.ndim == 4- if is_image:- video = rearrange(x, 'b c ... -> b c 1 ...')- video_contains_first_frame = True- else:- video = x-- # ===== Spatial VAE =====- if cfg.get("use_pipeline") == True:- with torch.no_grad():- video = vae_2d.encode(video)-- # ====== VAE ======- recon_video, posterior = vae(- video,- video_contains_first_frame = video_contains_first_frame,- )- # ====== Generator Loss ======- # simple nll loss- nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(- video,- recon_video,- posterior,- split = "train"+ for step in pbar:+ # with profile(+ # activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],+ # with_stack=True,+ # ) as p: # trace efficiency++ # SCH: calc global step at the start+ global_step = epoch * num_steps_per_epoch + step++ batch = next(dataloader_iter)+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]++ # supprt for image or video inputs+ assert x.ndim in {4, 5}, f"received input of {x.ndim} dimensions" # either image or video+ assert (+ x.shape[-2:] == cfg.image_size+ ), f"received input size {x.shape[-2:]}, but config image size is {cfg.image_size}"+ is_image = x.ndim == 4+ if is_image:+ video = rearrange(x, "b c ... -> b c 1 ...")+ video_contains_first_frame = True+ else:+ video = x++ # ===== Spatial VAE =====+ if cfg.get("use_pipeline") == True:+ with torch.no_grad():+ video = vae_2d.encode(video)++ # ====== VAE ======+ recon_video, posterior = vae(+ video,+ video_contains_first_frame=video_contains_first_frame,+ )++ # ====== Generator Loss ======+ # simple nll loss+ nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(+ video, recon_video, posterior, split="train"+ )++ adversarial_loss = torch.tensor(0.0)+ # adversarial loss+ if global_step > cfg.discriminator_start:+ # padded videos for GAN+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)+ fake_logits = discriminator(fake_video.contiguous())+ adversarial_loss = adversarial_loss_fn(+ fake_logits,+ nll_loss,+ vae.module.get_last_layer(),+ global_step,+ is_training=vae.training,+ )++ vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss++ optimizer.zero_grad()+ # Backward & update+ booster.backward(loss=vae_loss, optimizer=optimizer)+ # # NOTE: clip gradients? this is done in Open-Sora-Plan+ # torch.nn.utils.clip_grad_norm_(vae.parameters(), 1) # NOTE: done by grad_clip+ optimizer.step()++ # Log loss values:+ all_reduce_mean(vae_loss) # NOTE: this is to get average loss for logging+ running_loss += vae_loss.item()++ # ====== Discriminator Loss ======+ if global_step > cfg.discriminator_start:+ # if video_contains_first_frame:+ # Since we don't have enough T frames, pad anyways+ real_video = pad_at_dim(video, (disc_time_padding, 0), value=0.0, dim=2)+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)++ if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:+ real_video = real_video.requires_grad_()+ real_logits = discriminator(+ real_video.contiguous()+ ) # SCH: not detached for now for gradient_penalty calculation+ else:+ real_logits = discriminator(real_video.contiguous().detach())++ fake_logits = discriminator(fake_video.contiguous().detach())++ lecam_ema_real, lecam_ema_fake = lecam_ema.get()++ weighted_d_adversarial_loss, lecam_loss, gradient_penalty_loss = disc_loss_fn(+ real_logits,+ fake_logits,+ global_step,+ lecam_ema_real=lecam_ema_real,+ lecam_ema_fake=lecam_ema_fake,+ real_video=real_video if cfg.gradient_penalty_loss_weight is not None else None,+ )+ disc_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty_loss+ if cfg.lecam_loss_weight is not None:+ ema_real = torch.mean(real_logits.clone().detach()).to(device, dtype)+ ema_fake = torch.mean(fake_logits.clone().detach()).to(device, dtype)+ all_reduce_mean(ema_real)+ all_reduce_mean(ema_fake)+ lecam_ema.update(ema_real, ema_fake)++ disc_optimizer.zero_grad()+ # Backward & update+ booster.backward(loss=disc_loss, optimizer=disc_optimizer)+ # # NOTE: TODO: clip gradients? this is done in Open-Sora-Plan+ # torch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1) # NOTE: done by grad_clip+ disc_optimizer.step()++ # Log loss values:+ all_reduce_mean(disc_loss)+ running_disc_loss += disc_loss.item()+ else:+ disc_loss = torch.tensor(0.0)+ weighted_d_adversarial_loss = torch.tensor(0.0)+ lecam_loss = torch.tensor(0.0)+ gradient_penalty_loss = torch.tensor(0.0)++ log_step += 1++ # Log to tensorboard+ if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:+ avg_loss = running_loss / log_step+ avg_disc_loss = running_disc_loss / log_step+ pbar.set_postfix(+ {"loss": avg_loss, "disc_loss": avg_disc_loss, "step": step, "global_step": global_step}+ )+ running_loss = 0+ log_step = 0+ running_disc_loss = 0+ writer.add_scalar("loss", vae_loss.item(), global_step)+ if cfg.wandb:+ wandb.log(+ {+ "iter": global_step,+ "num_samples": global_step * total_batch_size,+ "epoch": epoch,+ "loss": vae_loss.item(),+ "kl_loss": weighted_kl_loss.item(),+ "gen_adv_loss": adversarial_loss.item(),+ "disc_loss": disc_loss.item(),+ "lecam_loss": lecam_loss.item(),+ "r1_grad_penalty": gradient_penalty_loss.item(),+ "nll_loss": weighted_nll_loss.item(),+ "avg_loss": avg_loss,+ },+ step=global_step,)- adversarial_loss = torch.tensor(0.0)- # adversarial loss- if global_step > cfg.discriminator_start:- # padded videos for GAN- fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)- fake_logits = discriminator(fake_video.contiguous())- adversarial_loss = adversarial_loss_fn(- fake_logits,- nll_loss,- vae.module.get_last_layer(),- global_step,- is_training = vae.training,- )-- vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss-- optimizer.zero_grad()- # Backward & update- booster.backward(loss=vae_loss, optimizer=optimizer)- # # NOTE: clip gradients? this is done in Open-Sora-Plan- # torch.nn.utils.clip_grad_norm_(vae.parameters(), 1) # NOTE: done by grad_clip- optimizer.step()-- # Log loss values:- all_reduce_mean(vae_loss) # NOTE: this is to get average loss for logging- running_loss += vae_loss.item()--- # ====== Discriminator Loss ======- if global_step > cfg.discriminator_start:- # if video_contains_first_frame:- # Since we don't have enough T frames, pad anyways- real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)- fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)-- if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:- real_video = real_video.requires_grad_()- real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculation- else:- real_logits = discriminator(real_video.contiguous().detach())-- fake_logits = discriminator(fake_video.contiguous().detach())--- lecam_ema_real, lecam_ema_fake = lecam_ema.get()-- weighted_d_adversarial_loss, lecam_loss, gradient_penalty_loss = disc_loss_fn(- real_logits,- fake_logits,- global_step,- lecam_ema_real = lecam_ema_real,- lecam_ema_fake = lecam_ema_fake,- real_video = real_video if cfg.gradient_penalty_loss_weight is not None else None,- )- disc_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty_loss- if cfg.lecam_loss_weight is not None:- ema_real = torch.mean(real_logits.clone().detach()).to(device, dtype)- ema_fake = torch.mean(fake_logits.clone().detach()).to(device, dtype)- all_reduce_mean(ema_real)- all_reduce_mean(ema_fake)- lecam_ema.update(ema_real, ema_fake)-- disc_optimizer.zero_grad()- # Backward & update- booster.backward(loss=disc_loss, optimizer=disc_optimizer)- # # NOTE: TODO: clip gradients? this is done in Open-Sora-Plan- # torch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1) # NOTE: done by grad_clip- disc_optimizer.step()-- # Log loss values:- all_reduce_mean(disc_loss)- running_disc_loss += disc_loss.item()- else:- disc_loss = torch.tensor(0.0)- weighted_d_adversarial_loss = torch.tensor(0.0)- lecam_loss = torch.tensor(0.0)- gradient_penalty_loss = torch.tensor(0.0)-- log_step += 1-- # Log to tensorboard- if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:- avg_loss = running_loss / log_step- avg_disc_loss = running_disc_loss / log_step- pbar.set_postfix({"loss": avg_loss, "disc_loss": avg_disc_loss, "step": step, "global_step": global_step})- running_loss = 0- log_step = 0- running_disc_loss = 0- writer.add_scalar("loss", vae_loss.item(), global_step)- if cfg.wandb:- wandb.log(- {- "iter": global_step,- "num_samples": global_step * total_batch_size,- "epoch": epoch,- "loss": vae_loss.item(),- "kl_loss": weighted_kl_loss.item(),- "gen_adv_loss": adversarial_loss.item(),- "disc_loss": disc_loss.item(),- "lecam_loss": lecam_loss.item(),- "r1_grad_penalty": gradient_penalty_loss.item(),- "nll_loss": weighted_nll_loss.item(),- "avg_loss": avg_loss,- },- step=global_step,- )-- # Save checkpoint- if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:- save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")- os.makedirs(os.path.join(save_dir, "model"), exist_ok=True) # already handled in booster save_model- booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)- booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)- booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)- booster.save_optimizer(disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096)-- if lr_scheduler is not None:- booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))- if disc_lr_scheduler is not None:- booster.save_lr_scheduler(disc_lr_scheduler, os.path.join(save_dir, "disc_lr_scheduler"))-- running_states = {- "epoch": epoch,- "step": step+1,- "global_step": global_step+1,- "sample_start_index": (step+1) * cfg.batch_size,- }-- lecam_ema_real, lecam_ema_fake = lecam_ema.get()- lecam_state = {- "lecam_ema_real": lecam_ema_real.item(),- "lecam_ema_fake": lecam_ema_fake.item(),- }- if coordinator.is_master():- save_json(running_states, os.path.join(save_dir, "running_states.json"))- if cfg.lecam_loss_weight is not None:- save_json(lecam_state, os.path.join(save_dir, "lecam_states.json"))- dist.barrier()-- logger.info(- f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"- )-- # print(p.key_averages().table(sort_by="self_cuda_time_total", row_limit=-1))-+ # Save checkpoint+ if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:+ save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")+ os.makedirs(os.path.join(save_dir, "model"), exist_ok=True) # already handled in booster save_model+ booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)+ booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)+ booster.save_optimizer(+ optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096+ )+ booster.save_optimizer(+ disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096+ )++ if lr_scheduler is not None:+ booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))+ if disc_lr_scheduler is not None:+ booster.save_lr_scheduler(disc_lr_scheduler, os.path.join(save_dir, "disc_lr_scheduler"))++ running_states = {+ "epoch": epoch,+ "step": step + 1,+ "global_step": global_step + 1,+ "sample_start_index": (step + 1) * cfg.batch_size,+ }++ lecam_ema_real, lecam_ema_fake = lecam_ema.get()+ lecam_state = {+ "lecam_ema_real": lecam_ema_real.item(),+ "lecam_ema_fake": lecam_ema_fake.item(),+ }+ if coordinator.is_master():+ save_json(running_states, os.path.join(save_dir, "running_states.json"))+ if cfg.lecam_loss_weight is not None:+ save_json(lecam_state, os.path.join(save_dir, "lecam_states.json"))+ dist.barrier()++ logger.info(+ f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"+ )++ # print(p.key_averages().table(sort_by="self_cuda_time_total", row_limit=-1))# the continue epochs are not resumed, so we need to reset the sampler start index and start stepdataloader.sampler.set_start_index(0)start_step = 0+if __name__ == "__main__":main()