Open-Sora: Democratizing Efficient Video Production for All
Python
[wip] try vae
zhengzangwb2f302b379be
diff --git a/.gitignore b/.gitignoreindex 4d5dc97..b9f8121 100644--- a/.gitignore+++ b/.gitignore@@ -181,4 +181,3 @@ cache/hostfilegradio_cached_examples/wandb/-taming/diff --git a/configs/vae/train/17x256x256.py b/configs/vae/train/17x256x256.pyindex d5a49d7..2b24de2 100644--- a/configs/vae/train/17x256x256.py+++ b/configs/vae/train/17x256x256.py@@ -1,31 +1,22 @@num_frames = 17image_size = (256, 256)++# Define datasetdataset = dict(type="VideoTextDataset",data_path=None,num_frames=num_frames,frame_interval=1,image_size=image_size,- get_text=False,)# Define acceleration-num_workers = 4+num_workers = 16dtype = "bf16"grad_checkpoint = Trueplugin = "zero2"sp_size = 1-use_pipeline = True-video_contains_first_frame = True--# Define model-# vae_2d = dict(-# type="VideoAutoencoderKL",-# from_pretrained="stabilityai/sd-vae-ft-ema",-# # SDXL-# )-# latestvae_2d = dict(type="VideoAutoencoderKL",@@ -36,33 +27,20 @@ vae_2d = dict()model = dict(- type="VAE_MAGVIT_V2",- in_out_channels=4,- latent_embed_dim=64,- filters=128,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- kl_embed_dim=4,- activation_fn="swish",- separate_first_frame_encoding=False,- disable_space=True,- encoder_double_z=True,- custom_conv_padding=None,+ type="VAE_Temporal_SD",)-discriminator = dict(- type="DISCRIMINATOR_3D",- image_size=image_size, # NOTE: here image size is different- num_frames=num_frames,- in_channels=3,- filters=128,- use_pretrained=True, # NOTE: set to False only if we want to disable load- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z-)+# discriminator = dict(+# type="DISCRIMINATOR_3D",+# image_size=image_size, # NOTE: here image size is different+# num_frames=num_frames,+# in_channels=3,+# filters=128,+# use_pretrained=True, # NOTE: set to False only if we want to disable load+# channel_multipliers=(2, 4, 4, 4, 4), # (2,4,4,4) for 64x64 resolution+# # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z+# )# loss weights@@ -76,7 +54,7 @@ discriminator_loss_type = "non-saturating"generator_loss_type = "non-saturating"# discriminator_loss_type="hinge"# generator_loss_type="hinge"-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch+discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epochgradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't useema_decay = 0.999 # ema decay factor for generator@@ -95,11 +73,9 @@ magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128epochs = 100log_every = 1--ckpt_every = 500+ckpt_every = 1000load = None-batch_size = 1--lr = 1e-4+batch_size = 4+lr = 1e-5grad_clip = 1.0diff --git a/configs/vae/train/1x256x256.py b/configs/vae/train/1x256x256.pyindex 91d99a9..582bbcb 100644--- a/configs/vae/train/1x256x256.py+++ b/configs/vae/train/1x256x256.py@@ -1,31 +1,22 @@num_frames = 1image_size = (256, 256)++# Define datasetdataset = dict(type="VideoTextDataset",data_path=None,num_frames=num_frames,frame_interval=1,image_size=image_size,- get_text=False,)# Define acceleration-num_workers = 4+num_workers = 16dtype = "bf16"-grad_checkpoint = True+grad_checkpoint = Falseplugin = "zero2"sp_size = 1-use_pipeline = True-video_contains_first_frame = True--# Define model-# vae_2d = dict(-# type="VideoAutoencoderKL",-# from_pretrained="stabilityai/sd-vae-ft-ema",-# # SDXL-# )-# latestvae_2d = dict(type="VideoAutoencoderKL",@@ -36,33 +27,19 @@ vae_2d = dict()model = dict(- type="VAE_MAGVIT_V2",- in_out_channels=4,- latent_embed_dim=64,- filters=128,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- kl_embed_dim=4,- activation_fn="swish",- separate_first_frame_encoding=False,- disable_space=True,- custom_conv_padding=None,- encoder_double_z=True,+ type="VAE_Temporal_SD",)--discriminator = dict(- type="DISCRIMINATOR_3D",- image_size=image_size, # NOTE: here image size is different- num_frames=num_frames,- in_channels=3,- filters=128,- use_pretrained=True, # NOTE: set to False only if we want to disable load- channel_multipliers=(2, 4, 4, 4, 4), # (2,4,4,4) for 64x64 resolution- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z-)+# discriminator = dict(+# type="DISCRIMINATOR_3D",+# image_size=image_size, # NOTE: here image size is different+# num_frames=num_frames,+# in_channels=3,+# filters=128,+# use_pretrained=True, # NOTE: set to False only if we want to disable load+# channel_multipliers=(2, 4, 4, 4, 4), # (2,4,4,4) for 64x64 resolution+# # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z+# )# loss weights@@ -70,14 +47,12 @@ logvar_init = 0.0kl_loss_weight = 0.000001perceptual_loss_weight = 0.1 # use vgg is not None and more than 0discriminator_factor = 1.0 # for discriminator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1+generator_factor = 0.1 # generator adversarial loss, MAGVIT v2 uses 0.1lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001discriminator_loss_type = "non-saturating"generator_loss_type = "non-saturating"-# discriminator_loss_type="hinge"-# generator_loss_type="hinge"-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use+discriminator_start = 2000+gradient_penalty_loss_weight = None # MAGVIT uses 10, opensora plan doesn't useema_decay = 0.999 # ema decay factor for generator@@ -95,11 +70,9 @@ magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128epochs = 100log_every = 1--ckpt_every = 500+ckpt_every = 1000load = None-batch_size = 1--lr = 1e-4+batch_size = 4+lr = 1e-5grad_clip = 1.0diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.pyindex d320ff7..b9d7f05 100644--- a/opensora/datasets/datasets.py+++ b/opensora/datasets/datasets.py@@ -29,10 +29,10 @@ class VideoTextDataset(torch.utils.data.Dataset):frame_interval=1,image_size=(256, 256),transform_name="center",- get_text=True,):self.data_path = data_pathself.data = read_file(data_path)+ self.get_text = "text" in self.data.columnsself.num_frames = num_framesself.frame_interval = frame_intervalself.image_size = image_size@@ -40,7 +40,6 @@ class VideoTextDataset(torch.utils.data.Dataset):"image": get_transforms_image(transform_name, image_size),"video": get_transforms_video(transform_name, image_size),}- self.get_text = get_textdef _print_data_number(self):num_videos = 0@@ -63,7 +62,6 @@ class VideoTextDataset(torch.utils.data.Dataset):def getitem(self, index):sample = self.data.iloc[index]path = sample["path"]- text = sample["text"] if self.get_text else Nonefile_type = self.get_type(path)if file_type == "video":@@ -90,10 +88,10 @@ class VideoTextDataset(torch.utils.data.Dataset):# TCHW -> CTHWvideo = video.permute(1, 0, 2, 3)+ ret = {"video": video}if self.get_text:- return {"video": video, "text": text}- else:- return {"video": video}+ ret["text"] = sample["text"]+ return retdef __getitem__(self, index):for _ in range(10):@@ -135,7 +133,6 @@ class VariableVideoTextDataset(VideoTextDataset):sample = self.data.iloc[index]path = sample["path"]- text = sample["text"]file_type = self.get_type(path)ar = width / height@@ -166,12 +163,14 @@ class VariableVideoTextDataset(VideoTextDataset):# TCHW -> CTHWvideo = video.permute(1, 0, 2, 3)- return {+ ret = {"video": video,- "text": text,"num_frames": num_frames,"height": height,"width": width,"ar": ar,"fps": video_fps,}+ if self.get_text:+ ret["text"] = sample["text"]+ return retdiff --git a/opensora/models/vae/__init__.py b/opensora/models/vae/__init__.pyindex c46c015..78a24fd 100644--- a/opensora/models/vae/__init__.py+++ b/opensora/models/vae/__init__.py@@ -1,2 +1,3 @@+from .discriminator import DISCRIMINATOR_3Dfrom .vae import VideoAutoencoderKL, VideoAutoencoderKLTemporalDecoder-from .vae_3d import VAE_3D_V2, VAE_MAGVIT_V2+from .vae_3d import VAE_Temporaldiff --git a/opensora/models/vae/discriminator.py b/opensora/models/vae/discriminator.pynew file mode 100644index 0000000..e510387--- /dev/null+++ b/opensora/models/vae/discriminator.py@@ -0,0 +1,364 @@+import functools+import math++import numpy as np+import torch+import torch.nn as nn+import torch.nn.functional as F++from opensora.registry import MODELS+from opensora.utils.ckpt_utils import find_model, load_checkpoint+++def cast_tuple(t, length=1):+ return t if isinstance(t, tuple) else ((t,) * length)+++def xavier_uniform_weight_init(m):+ if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):+ nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain("relu"))+ if m.bias is not None:+ nn.init.zeros_(m.bias)+ # print("initialized module to xavier_uniform:", m)+++# SCH: taken from Open Sora Plan+def n_layer_disc_weights_init(m):+ classname = m.__class__.__name__+ if classname.find("Conv") != -1:+ nn.init.normal_(m.weight.data, 0.0, 0.02)+ elif classname.find("BatchNorm") != -1:+ nn.init.normal_(m.weight.data, 1.0, 0.02)+ nn.init.constant_(m.bias.data, 0)+++# SCH: own implementation modified on top of: discriminator with anti-aliased downsampling (blurpool Zhang et al.)+class BlurPool3D(nn.Module):+ def __init__(+ self,+ channels,+ pad_type="reflect",+ filt_size=3,+ stride=2,+ pad_off=0,+ device="cpu",+ dtype=torch.bfloat16,+ ):+ super(BlurPool3D, self).__init__()+ self.filt_size = filt_size+ self.pad_off = pad_off+ self.pad_sizes = [+ int(1.0 * (filt_size - 1) / 2),+ int(np.ceil(1.0 * (filt_size - 1) / 2)),+ int(1.0 * (filt_size - 1) / 2),+ int(np.ceil(1.0 * (filt_size - 1) / 2)),+ int(1.0 * (filt_size - 1) / 2),+ int(np.ceil(1.0 * (filt_size - 1) / 2)),+ ]+ self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]+ self.stride = stride+ self.off = int((self.stride - 1) / 2.0)+ self.channels = channels++ if self.filt_size == 1:+ a = np.array(+ [+ 1.0,+ ]+ )+ elif self.filt_size == 2:+ a = np.array([1.0, 1.0])+ elif self.filt_size == 3:+ a = np.array([1.0, 2.0, 1.0])+ elif self.filt_size == 4:+ a = np.array([1.0, 3.0, 3.0, 1.0])+ elif self.filt_size == 5:+ a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])+ elif self.filt_size == 6:+ a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])+ elif self.filt_size == 7:+ a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])++ filt_2d = a[:, None] * a[None, :]+ filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)++ filt = filt_3d / torch.sum(filt_3d) # SCH: modified to it 3D+ self.register_buffer("filt", filt[None, None, :, :, :].repeat((self.channels, 1, 1, 1, 1)))++ self.pad = get_pad_layer(pad_type)(self.pad_sizes)++ def forward(self, inp):+ if self.filt_size == 1:+ if self.pad_off == 0:+ return inp[:, :, :: self.stride, :: self.stride]+ else:+ return self.pad(inp)[:, :, :: self.stride, :: self.stride]+ else:+ return F.conv3d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])+++class ResBlockDown(nn.Module):+ """3D StyleGAN ResBlock for D."""++ def __init__(+ self,+ in_channels,+ filters,+ activation_fn,+ num_groups=32,+ device="cpu",+ dtype=torch.bfloat16,+ ):+ super().__init__()++ self.filters = filters+ self.activation_fn = activation_fn++ # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code+ self.conv1 = nn.Conv3d(+ in_channels, in_channels, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform+ self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)++ self.blur = BlurPool3D(in_channels, device=device, dtype=dtype)++ self.conv2 = nn.Conv3d(+ in_channels, self.filters, (1, 1, 1), bias=False, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform+ self.conv3 = nn.Conv3d(+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform+ self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)++ # self.apply(xavier_uniform_weight_init)++ def forward(self, x):+ residual = x+ x = self.conv1(x)+ x = self.norm1(x)+ x = self.activation_fn(x)++ residual = self.blur(residual)+ residual = self.conv2(residual)++ x = self.blur(x)+ x = self.conv3(x)+ x = self.norm2(x)+ x = self.activation_fn(x)+ out = (residual + x) / math.sqrt(2)+ return out+++class NLayerDiscriminator3D(nn.Module):+ """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""++ def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):+ """+ Construct a 3D PatchGAN discriminator++ Parameters:+ input_nc (int) -- the number of channels in input volumes+ ndf (int) -- the number of filters in the last conv layer+ n_layers (int) -- the number of conv layers in the discriminator+ use_actnorm (bool) -- flag to use actnorm instead of batchnorm+ """+ super(NLayerDiscriminator3D, self).__init__()+ if not use_actnorm:+ norm_layer = nn.BatchNorm3d+ else:+ raise NotImplementedError("Not implemented.")+ if type(norm_layer) == functools.partial:+ use_bias = norm_layer.func != nn.BatchNorm3d+ else:+ use_bias = norm_layer != nn.BatchNorm3d++ kw = 4+ padw = 1+ sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]+ nf_mult = 1+ nf_mult_prev = 1+ for n in range(1, n_layers): # gradually increase the number of filters+ nf_mult_prev = nf_mult+ nf_mult = min(2**n, 8)+ sequence += [+ nn.Conv3d(+ ndf * nf_mult_prev,+ ndf * nf_mult,+ kernel_size=(kw, kw, kw),+ stride=(1, 2, 2),+ padding=padw,+ bias=use_bias,+ ),+ norm_layer(ndf * nf_mult),+ nn.LeakyReLU(0.2, True),+ ]++ nf_mult_prev = nf_mult+ nf_mult = min(2**n_layers, 8)+ sequence += [+ nn.Conv3d(+ ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias+ ),+ norm_layer(ndf * nf_mult),+ nn.LeakyReLU(0.2, True),+ ]++ sequence += [+ nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)+ ] # output 1 channel prediction map+ self.main = nn.Sequential(*sequence)++ def forward(self, input):+ """Standard forward."""+ return self.main(input)+++class StyleGANDiscriminatorBlur(nn.Module):+ """StyleGAN Discriminator.++ SCH: NOTE:+ this discriminator requries the num_frames to be fixed during training;+ in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!+ """++ def __init__(+ self,+ image_size=(128, 128),+ num_frames=17,+ in_channels=3,+ filters=128,+ channel_multipliers=(2, 4, 4, 4, 4),+ num_groups=32,+ dtype=torch.bfloat16,+ device="cpu",+ ):+ super().__init__()++ self.dtype = dtype+ self.input_size = cast_tuple(image_size, 2)+ self.filters = filters+ self.activation_fn = nn.LeakyReLU(negative_slope=0.2)+ self.channel_multipliers = channel_multipliers++ self.conv1 = nn.Conv3d(+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform++ prev_filters = self.filters # record in_channels+ self.num_blocks = len(self.channel_multipliers)+ self.res_block_list = nn.ModuleList([])+ for i in range(self.num_blocks):+ filters = self.filters * self.channel_multipliers[i]+ self.res_block_list.append(+ ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(+ xavier_uniform_weight_init+ )+ )+ prev_filters = filters # update in_channels++ self.conv2 = nn.Conv3d(+ prev_filters, prev_filters, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform+ # torch.nn.init.xavier_uniform_(self.conv2.weight)++ self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)++ scale_factor = 2**self.num_blocks+ if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage+ time_scaled = num_frames // scale_factor + 1+ else:+ time_scaled = num_frames / scale_factor++ assert (+ self.input_size[0] % scale_factor == 0+ ), f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"+ assert (+ self.input_size[1] % scale_factor == 0+ ), f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"+ w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor+ in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)+ self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform+ self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform++ # self.apply(xavier_uniform_weight_init)++ def forward(self, x):+ x = self.conv1(x)+ # print("discriminator aft conv:", x.size())+ x = self.activation_fn(x)++ for i in range(self.num_blocks):+ x = self.res_block_list[i](x)+ # print("discriminator resblock down:", x.size())++ x = self.conv2(x)+ # print("discriminator aft conv2:", x.size())+ x = self.norm1(x)+ x = self.activation_fn(x)+ x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?++ # print("discriminator reshape:", x.size())+ x = self.linear1(x)+ # print("discriminator aft linear1:", x.size())++ x = self.activation_fn(x)+ x = self.linear2(x)+ # print("discriminator aft linear2:", x.size())+ return x+++def load_checkpoint_with_inflation(model, ckpt_path):+ """+ pre-train using image, then inflate to 3D videos+ """+ if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):+ state_dict = find_model(ckpt_path)+ with torch.no_grad():+ for key in state_dict:+ if key in model:+ # central inflation+ if state_dict[key].size() == model[key][:, :, 0, :, :].size():+ # temporal dimension+ val = torch.zeros_like(model[key])+ centre = int(model[key].size(2) // 2)+ val[:, :, centre, :, :] = state_dict[key]+ missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)+ print(f"Missing keys: {missing_keys}")+ print(f"Unexpected keys: {unexpected_keys}")+ else:+ load_checkpoint(model, ckpt_path) # use the default function++[email protected]_module("DISCRIMINATOR_3D")+def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):+ model = StyleGANDiscriminatorBlur(**kwargs).apply(xavier_uniform_weight_init)+ if from_pretrained is not None:+ if use_pretrained:+ if inflate_from_2d:+ load_checkpoint_with_inflation(model, from_pretrained)+ else:+ load_checkpoint(model, from_pretrained, model_name="discriminator")+ print("loaded discriminator")+ else:+ print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")++ return model++[email protected]_module("N_Layer_DISCRIMINATOR_3D")+def DISCRIMINATOR_3D_N_Layer(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):+ model = NLayerDiscriminator3D(+ input_nc=3,+ n_layers=3,+ ).apply(n_layer_disc_weights_init)+ if from_pretrained is not None:+ if use_pretrained:+ if inflate_from_2d:+ load_checkpoint_with_inflation(model, from_pretrained)+ else:+ load_checkpoint(model, from_pretrained, model_name="discriminator")+ print("loaded discriminator")+ else:+ print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")++ return modeldiff --git a/opensora/models/vae/losses.py b/opensora/models/vae/losses.pyindex 4adc85c..68d555f 100644--- a/opensora/models/vae/losses.py+++ b/opensora/models/vae/losses.py@@ -1,9 +1,11 @@-import torch.nn as nnimport torch+import torch.nn as nnimport torch.nn.functional as F-from .lpips import LPIPSfrom einops import rearrange, repeat+from .lpips import LPIPS++def hinge_d_loss(logits_real, logits_fake):loss_real = torch.mean(F.relu(1.0 - logits_real))loss_fake = torch.mean(F.relu(1.0 + logits_fake))@@ -27,12 +29,14 @@ def sigmoid_cross_entropy_with_logits(labels, logits):neg_abs_logits = torch.where(condition, -logits, logits)return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))+def lecam_reg(real_pred, fake_pred, ema_real_pred, ema_fake_pred):assert real_pred.ndim == 0 and ema_fake_pred.ndim == 0lecam_loss = torch.mean(torch.pow(nn.ReLU()(real_pred - ema_fake_pred), 2))lecam_loss += torch.mean(torch.pow(nn.ReLU()(ema_real_pred - fake_pred), 2))return lecam_loss+def gradient_penalty_fn(images, output):gradients = torch.autograd.grad(outputs=output,@@ -46,7 +50,8 @@ def gradient_penalty_fn(images, output):gradients = rearrange(gradients, "b ... -> b (...)")return ((gradients.norm(2, dim=1) - 1) ** 2).mean()-class VEALoss(nn.Module):++class VAELoss(nn.Module):def __init__(self,logvar_init=0.0,@@ -72,7 +77,6 @@ class VEALoss(nn.Module):self.perceptual_loss_weight = perceptual_loss_weightself.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)-def forward(self,video,@@ -111,7 +115,6 @@ class VEALoss(nn.Module):weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]-# KL Lossweighted_kl_loss = 0if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:@@ -121,11 +124,13 @@ class VEALoss(nn.Module):return nll_loss, weighted_nll_loss, weighted_kl_loss+def adopt_weight(weight, global_step, threshold=0, value=0.0):if global_step < threshold:weight = valuereturn weight+class AdversarialLoss(nn.Module):def __init__(self,@@ -141,10 +146,8 @@ class AdversarialLoss(nn.Module):self.generator_loss_type = generator_loss_typedef calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[- 0- ]+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()d_weight = d_weight * self.generator_factor@@ -269,4 +272,4 @@ class DiscriminatorLoss(nn.Module):gradient_penalty = gradient_penalty_fn(real_video, real_logits)gradient_penalty *= self.gradient_penalty_loss_weight- return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)\ No newline at end of file+ return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)diff --git a/opensora/models/vae/lpips.py b/opensora/models/vae/lpips.pyindex b22cd0d..d1f00c7 100644--- a/opensora/models/vae/lpips.py+++ b/opensora/models/vae/lpips.py@@ -1,28 +1,26 @@+import hashlib+import os+from collections import namedtuple++import requestsimport torchimport torch.nn as nnfrom torchvision import models-from collections import namedtuple-import os, hashlib-import requestsfrom tqdm import tqdm-URL_MAP = {- "vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"-}+URL_MAP = {"vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"}-CKPT_MAP = {- "vgg_lpips": "vgg.pth"-}+CKPT_MAP = {"vgg_lpips": "vgg.pth"}++MD5_MAP = {"vgg_lpips": "d507d7349b931f0638a25a48a722f98a"}-MD5_MAP = {- "vgg_lpips": "d507d7349b931f0638a25a48a722f98a"-}def md5_hash(path):with open(path, "rb") as f:content = f.read()return hashlib.md5(content).hexdigest()+def download(url, local_path, chunk_size=1024):os.makedirs(os.path.split(local_path)[0], exist_ok=True)with requests.get(url, stream=True) as r:@@ -34,6 +32,7 @@ def download(url, local_path, chunk_size=1024):f.write(data)pbar.update(chunk_size)+def get_ckpt_path(name, root, check=False):assert name in URL_MAPpath = os.path.join(root, CKPT_MAP[name])@@ -45,14 +44,13 @@ def get_ckpt_path(name, root, check=False):return path-class LPIPS(nn.Module):# Learned perceptual metricdef __init__(self, use_dropout=True):super().__init__()self.scaling_layer = ScalingLayer()self.chns = [64, 128, 256, 512, 512] # vg16 features- self.net = vgg16(pretrained=True, requires_grad=False) # NOTE: TODO: need in_channels = 4 to use+ self.net = vgg16(pretrained=True, requires_grad=False) # NOTE: TODO: need in_channels = 4 to useself.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout)self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout)self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout)@@ -63,7 +61,7 @@ class LPIPS(nn.Module):param.requires_grad = Falsedef load_from_pretrained(self, name="vgg_lpips"):- ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips")+ ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips", root="pretrained_models")self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)# print("loaded pretrained LPIPS loss from {}".format(ckpt))@@ -92,23 +90,32 @@ class LPIPS(nn.Module):return val-# SCH: TODO: this channel shift & scale may need to be changed+# SCH: TODO: this channel shift & scale may need to be changedclass ScalingLayer(nn.Module):def __init__(self):super(ScalingLayer, self).__init__()- self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None])- self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None])+ self.register_buffer("shift", torch.Tensor([-0.030, -0.088, -0.188])[None, :, None, None])+ self.register_buffer("scale", torch.Tensor([0.458, 0.448, 0.450])[None, :, None, None])def forward(self, inp):return (inp - self.shift) / self.scaleclass NetLinLayer(nn.Module):- """ A single linear layer which does a 1x1 conv """+ """A single linear layer which does a 1x1 conv"""+def __init__(self, chn_in, chn_out=1, use_dropout=False):super(NetLinLayer, self).__init__()- layers = [nn.Dropout(), ] if (use_dropout) else []- layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ]+ layers = (+ [+ nn.Dropout(),+ ]+ if (use_dropout)+ else []+ )+ layers += [+ nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False),+ ]self.model = nn.Sequential(*layers)@@ -147,15 +154,15 @@ class vgg16(torch.nn.Module):h_relu4_3 = hh = self.slice5(h)h_relu5_3 = h- vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3'])+ vgg_outputs = namedtuple("VggOutputs", ["relu1_2", "relu2_2", "relu3_3", "relu4_3", "relu5_3"])out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3)return out-def normalize_tensor(x,eps=1e-10):- norm_factor = torch.sqrt(torch.sum(x**2,dim=1,keepdim=True))- return x/(norm_factor+eps)+def normalize_tensor(x, eps=1e-10):+ norm_factor = torch.sqrt(torch.sum(x**2, dim=1, keepdim=True))+ return x / (norm_factor + eps)def spatial_average(x, keepdim=True):- return x.mean([2,3],keepdim=keepdim)\ No newline at end of file+ return x.mean([2, 3], keepdim=keepdim)diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.pyindex d4a47ad..db0db61 100644--- a/opensora/models/vae/vae_3d.py+++ b/opensora/models/vae/vae_3d.py@@ -1,15 +1,13 @@import functools-import mathfrom typing import Tuple, Union-import numpy as npimport torchimport torch.nn as nnimport torch.nn.functional as F-from einops import pack, rearrange+from einops import rearrangefrom opensora.registry import MODELS-from opensora.utils.ckpt_utils import find_model, load_checkpoint+from opensora.utils.ckpt_utils import load_checkpointfrom .utils import DiagonalGaussianDistribution@@ -29,7 +27,7 @@ def is_odd(n):def pad_at_dim(t, pad, dim=-1, value=0.0):dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)zeros = (0, 0) * dims_from_right- return F.pad(t, (*zeros, *pad), value=value)+ return F.pad(t, (*zeros, *pad), mode="replicate")def pick_video_frame(video, frame_indices):@@ -47,29 +45,6 @@ def exists(v):return v is not None-def xavier_uniform_weight_init(m):- if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):- nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain("relu"))- if m.bias is not None:- nn.init.zeros_(m.bias)- # print("initialized module to xavier_uniform:", m)---def Sequential(*modules):- modules = [*filter(exists, modules)]-- if len(modules) == 0:- return nn.Identity()-- return nn.Sequential(*modules)---def SameConv2d(dim_in, dim_out, kernel_size):- kernel_size = cast_tuple(kernel_size, 2)- padding = [k // 2 for k in kernel_size]- return nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, padding=padding)--class CausalConv3d(nn.Module):def __init__(self,@@ -99,17 +74,14 @@ class CausalConv3d(nn.Module):self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)stride = strides if strides is not None else (stride, 1, 1)- # padding = kwargs.pop('padding', 0)- # if padding == "same" and not all([pad == 1 for pad in padding]):- # padding = "valid"dilation = (dilation, 1, 1)self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)def forward(self, x):- pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"-+ pad_mode = "replicate"x = F.pad(x, self.time_causal_padding, mode=pad_mode)- return self.conv(x)+ x = self.conv(x)+ return xclass ResBlock(nn.Module):@@ -156,302 +128,14 @@ class ResBlock(nn.Module):return x + residual-# SCH: own implementation modified on top of: discriminator with anti-aliased downsampling (blurpool Zhang et al.)-class BlurPool3D(nn.Module):- def __init__(- self,- channels,- pad_type="reflect",- filt_size=3,- stride=2,- pad_off=0,- device="cpu",- dtype=torch.bfloat16,- ):- super(BlurPool3D, self).__init__()- self.filt_size = filt_size- self.pad_off = pad_off- self.pad_sizes = [- int(1.0 * (filt_size - 1) / 2),- int(np.ceil(1.0 * (filt_size - 1) / 2)),- int(1.0 * (filt_size - 1) / 2),- int(np.ceil(1.0 * (filt_size - 1) / 2)),- int(1.0 * (filt_size - 1) / 2),- int(np.ceil(1.0 * (filt_size - 1) / 2)),- ]- self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]- self.stride = stride- self.off = int((self.stride - 1) / 2.0)- self.channels = channels-- if self.filt_size == 1:- a = np.array(- [- 1.0,- ]- )- elif self.filt_size == 2:- a = np.array([1.0, 1.0])- elif self.filt_size == 3:- a = np.array([1.0, 2.0, 1.0])- elif self.filt_size == 4:- a = np.array([1.0, 3.0, 3.0, 1.0])- elif self.filt_size == 5:- a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])- elif self.filt_size == 6:- a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])- elif self.filt_size == 7:- a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])-- filt_2d = a[:, None] * a[None, :]- filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)-- filt = filt_3d / torch.sum(filt_3d) # SCH: modified to it 3D- self.register_buffer("filt", filt[None, None, :, :, :].repeat((self.channels, 1, 1, 1, 1)))-- self.pad = get_pad_layer(pad_type)(self.pad_sizes)-- def forward(self, inp):- if self.filt_size == 1:- if self.pad_off == 0:- return inp[:, :, :: self.stride, :: self.stride]- else:- return self.pad(inp)[:, :, :: self.stride, :: self.stride]- else:- return F.conv3d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])---def get_pad_layer(pad_type):- if pad_type in ["refl", "reflect"]:- PadLayer = nn.ReflectionPad3d- elif pad_type in ["repl", "replicate"]:- PadLayer = nn.ReplicationPad3d- elif pad_type == "zero":- PadLayer = nn.ZeroPad3d+def get_activation_fn(activation):+ if activation == "relu":+ activation_fn = nn.ReLU+ elif activation == "swish":+ activation_fn = nn.SiLUelse:- print("Pad type [%s] not recognized" % pad_type)- return PadLayer---class ResBlockDown(nn.Module):- """3D StyleGAN ResBlock for D."""-- def __init__(- self,- in_channels,- filters,- activation_fn,- num_groups=32,- device="cpu",- dtype=torch.bfloat16,- ):- super().__init__()-- self.filters = filters- self.activation_fn = activation_fn-- # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code- self.conv1 = nn.Conv3d(- in_channels, in_channels, (3, 3, 3), padding=1, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform- self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)-- self.blur = BlurPool3D(in_channels, device=device, dtype=dtype)-- self.conv2 = nn.Conv3d(- in_channels, self.filters, (1, 1, 1), bias=False, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform- self.conv3 = nn.Conv3d(- in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform- self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)-- # self.apply(xavier_uniform_weight_init)-- def forward(self, x):- residual = x- x = self.conv1(x)- x = self.norm1(x)- x = self.activation_fn(x)-- residual = self.blur(residual)- residual = self.conv2(residual)-- x = self.blur(x)- x = self.conv3(x)- x = self.norm2(x)- x = self.activation_fn(x)- out = (residual + x) / math.sqrt(2)- return out---# SCH: taken from Open Sora Plan-def n_layer_disc_weights_init(m):- classname = m.__class__.__name__- if classname.find("Conv") != -1:- nn.init.normal_(m.weight.data, 0.0, 0.02)- elif classname.find("BatchNorm") != -1:- nn.init.normal_(m.weight.data, 1.0, 0.02)- nn.init.constant_(m.bias.data, 0)---class NLayerDiscriminator3D(nn.Module):- """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""-- def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):- """- Construct a 3D PatchGAN discriminator-- Parameters:- input_nc (int) -- the number of channels in input volumes- ndf (int) -- the number of filters in the last conv layer- n_layers (int) -- the number of conv layers in the discriminator- use_actnorm (bool) -- flag to use actnorm instead of batchnorm- """- super(NLayerDiscriminator3D, self).__init__()- if not use_actnorm:- norm_layer = nn.BatchNorm3d- else:- raise NotImplementedError("Not implemented.")- if type(norm_layer) == functools.partial:- use_bias = norm_layer.func != nn.BatchNorm3d- else:- use_bias = norm_layer != nn.BatchNorm3d-- kw = 4- padw = 1- sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]- nf_mult = 1- nf_mult_prev = 1- for n in range(1, n_layers): # gradually increase the number of filters- nf_mult_prev = nf_mult- nf_mult = min(2**n, 8)- sequence += [- nn.Conv3d(- ndf * nf_mult_prev,- ndf * nf_mult,- kernel_size=(kw, kw, kw),- stride=(1, 2, 2),- padding=padw,- bias=use_bias,- ),- norm_layer(ndf * nf_mult),- nn.LeakyReLU(0.2, True),- ]-- nf_mult_prev = nf_mult- nf_mult = min(2**n_layers, 8)- sequence += [- nn.Conv3d(- ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias- ),- norm_layer(ndf * nf_mult),- nn.LeakyReLU(0.2, True),- ]-- sequence += [- nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)- ] # output 1 channel prediction map- self.main = nn.Sequential(*sequence)-- def forward(self, input):- """Standard forward."""- return self.main(input)---class StyleGANDiscriminatorBlur(nn.Module):- """StyleGAN Discriminator."""-- """- SCH: NOTE:- this discriminator requries the num_frames to be fixed during training;- in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!- """-- def __init__(- self,- image_size=(128, 128),- num_frames=17,- in_channels=3,- filters=128,- channel_multipliers=(2, 4, 4, 4, 4),- num_groups=32,- dtype=torch.bfloat16,- device="cpu",- ):- super().__init__()-- self.dtype = dtype- self.input_size = cast_tuple(image_size, 2)- self.filters = filters- self.activation_fn = nn.LeakyReLU(negative_slope=0.2)- self.channel_multipliers = channel_multipliers-- self.conv1 = nn.Conv3d(- in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform-- prev_filters = self.filters # record in_channels- self.num_blocks = len(self.channel_multipliers)- self.res_block_list = nn.ModuleList([])- for i in range(self.num_blocks):- filters = self.filters * self.channel_multipliers[i]- self.res_block_list.append(- ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(- xavier_uniform_weight_init- )- )- prev_filters = filters # update in_channels-- self.conv2 = nn.Conv3d(- prev_filters, prev_filters, (3, 3, 3), padding=1, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform- # torch.nn.init.xavier_uniform_(self.conv2.weight)-- self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)-- scale_factor = 2**self.num_blocks- if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage- time_scaled = num_frames // scale_factor + 1- else:- time_scaled = num_frames / scale_factor-- assert (- self.input_size[0] % scale_factor == 0- ), f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"- assert (- self.input_size[1] % scale_factor == 0- ), f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"- w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor- in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)- self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform- self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform-- # self.apply(xavier_uniform_weight_init)-- def forward(self, x):- x = self.conv1(x)- # print("discriminator aft conv:", x.size())- x = self.activation_fn(x)-- for i in range(self.num_blocks):- x = self.res_block_list[i](x)- # print("discriminator resblock down:", x.size())-- x = self.conv2(x)- # print("discriminator aft conv2:", x.size())- x = self.norm1(x)- x = self.activation_fn(x)- x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?-- # print("discriminator reshape:", x.size())- x = self.linear1(x)- # print("discriminator aft linear1:", x.size())-- x = self.activation_fn(x)- x = self.linear2(x)- # print("discriminator aft linear2:", x.size())- return x+ raise NotImplementedError+ return activation_fnclass Encoder(nn.Module):@@ -459,16 +143,13 @@ class Encoder(nn.Module):def __init__(self,+ in_out_channels=4,+ latent_embed_dim=512, # num channels for latent vectorfilters=128,num_res_blocks=4,channel_multipliers=(1, 2, 2, 4),temporal_downsample=(False, True, True),num_groups=32, # for nn.GroupNorm- # in_out_channels = 3, # SCH: added, in_channels at the start- latent_embed_dim=512, # num channels for latent vector- # conv_downsample = False,- disable_spatial_downsample=False, # for vae pipeline- custom_conv_padding=None,activation_fn="swish",device="cpu",dtype=torch.bfloat16,@@ -476,30 +157,19 @@ class Encoder(nn.Module):super().__init__()self.filters = filtersself.num_res_blocks = num_res_blocks+ self.num_blocks = len(channel_multipliers)self.channel_multipliers = channel_multipliersself.temporal_downsample = temporal_downsampleself.num_groups = num_groups-self.embedding_dim = latent_embed_dim- self.disable_spatial_downsample = disable_spatial_downsample- # self.conv_downsample = conv_downsample- self.custom_conv_padding = custom_conv_padding-- if activation_fn == "relu":- self.activation_fn = nn.ReLU- elif activation_fn == "swish":- self.activation_fn = nn.SiLU- else:- raise NotImplementedError- self.activate = self.activation_fn()+ self.activation_fn = get_activation_fn(activation_fn)+ self.activate = self.activation_fn()self.conv_fn = functools.partial(CausalConv3d,- # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorchdtype=dtype,device=device,)-self.block_args = dict(conv_fn=self.conv_fn,dtype=dtype,@@ -509,40 +179,44 @@ class Encoder(nn.Module):device=device,)- # NOTE: moved to VAE for separate first frame processing- # self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)+ # first layer conv+ self.conv_in = self.conv_fn(+ in_out_channels,+ filters,+ kernel_size=(3, 3, 3),+ bias=False,+ dtype=dtype,+ device=device,+ )# ResBlocks and conv downsampleself.block_res_blocks = nn.ModuleList([])- self.num_blocks = len(self.channel_multipliers)self.conv_blocks = nn.ModuleList([])filters = self.filtersprev_filters = filters # record for in_channelsfor i in range(self.num_blocks):- # resblock handling- filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels+ filters = self.filters * self.channel_multipliers[i]block_items = nn.ModuleList([])for _ in range(self.num_res_blocks):block_items.append(ResBlock(prev_filters, filters, **self.block_args))prev_filters = filters # update in_channelsself.block_res_blocks.append(block_items)- if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x stride s x stride s- if self.temporal_downsample[i]:+ if i < self.num_blocks - 1:+ if self.temporal_downsample[i]:t_stride = 2 if self.temporal_downsample[i] else 1- s_stride = 2 if not self.disable_spatial_downsample else 1+ s_stride = 1self.conv_blocks.append(- self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride))- ) # SCH: should be same in_channel and out_channel- prev_filters = filters # update in_channels- else: # if no t downsample, don't add since this does nothing for pipeline models- self.conv_blocks.append( # Identity- nn.Identity(prev_filters)+ self.conv_fn(+ prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride)+ ))prev_filters = filters # update in_channels--+ else:+ # if no t downsample, don't add since this does nothing for pipeline models+ self.conv_blocks.append(nn.Identity(prev_filters)) # Identity+ prev_filters = filters # update in_channels# last layer res blockself.res_blocks = nn.ModuleList([])@@ -553,40 +227,26 @@ class Encoder(nn.Module):# MAGVIT uses Group Normalizationself.norm1 = nn.GroupNorm(self.num_groups, prev_filters, dtype=dtype, device=device- ) # SCH: separate <prev_filters> channels into 32 groups+ ) # separate <prev_filters> channels into 32 groups- self.conv2 = nn.Conv3d(+ self.conv2 = self.conv_fn(prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same")def forward(self, x):- # dtype, device = x.dtype, x.device-- # NOTE: moved to VAE for separate first frame processing- # x = self.conv1(x)-- # print("encoder:", x.size())-- try:- for i in range(self.num_blocks):- for j in range(self.num_res_blocks):- x = self.block_res_blocks[i][j](x)- # print("encoder:", x.size())+ x = self.conv_in(x)- if i < self.num_blocks - 1:- x = self.conv_blocks[i](x)- # print("encoder:", x.size())-- for i in range(self.num_res_blocks):- x = self.res_blocks[i](x)- # print("encoder:", x.size())- except:- breakpoint()+ for i in range(self.num_blocks):+ for j in range(self.num_res_blocks):+ x = self.block_res_blocks[i][j](x)+ if i < self.num_blocks - 1:+ x = self.conv_blocks[i](x)+ for i in range(self.num_res_blocks):+ x = self.res_blocks[i](x)x = self.norm1(x)x = self.activate(x)x = self.conv2(x)- # print("encoder:", x.size())return x@@ -595,50 +255,34 @@ class Decoder(nn.Module):def __init__(self,+ in_out_channels=4,latent_embed_dim=512,filters=128,- # in_out_channels = 4,num_res_blocks=4,channel_multipliers=(1, 2, 2, 4),temporal_downsample=(False, True, True),num_groups=32, # for nn.GroupNorm- # upsample = "nearest+conv", # options: "deconv", "nearest+conv"- disable_spatial_upsample=False, # for vae pipeline- custom_conv_padding=None,activation_fn="swish",device="cpu",dtype=torch.bfloat16,):super().__init__()- # self.output_dim = in_out_channels- self.embedding_dim = latent_embed_dimself.filters = filtersself.num_res_blocks = num_res_blocks+ self.num_blocks = len(channel_multipliers)self.channel_multipliers = channel_multipliersself.temporal_downsample = temporal_downsampleself.num_groups = num_groups+ self.embedding_dim = latent_embed_dim+ self.s_stride = 1- # self.upsample = upsample- self.s_stride = 1 if disable_spatial_upsample else 2 # spatial stride- self.custom_conv_padding = custom_conv_padding- # self.norm_type = self.config.vqvae.norm_type- # self.num_remat_block = self.config.vqvae.get('num_dec_remat_blocks', 0)-- if activation_fn == "relu":- self.activation_fn = nn.ReLU- elif activation_fn == "swish":- self.activation_fn = nn.SiLU- else:- raise NotImplementedError+ self.activation_fn = get_activation_fn(activation_fn)self.activate = self.activation_fn()-self.conv_fn = functools.partial(CausalConv3d,dtype=dtype,- # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorchdevice=device,)-self.block_args = dict(conv_fn=self.conv_fn,activation_fn=self.activation_fn,@@ -647,10 +291,11 @@ class Decoder(nn.Module):device=device,dtype=dtype,)- self.num_blocks = len(self.channel_multipliers)filters = self.filters * self.channel_multipliers[-1]+ prev_filters = filters+ # last convself.conv1 = self.conv_fn(self.embedding_dim, filters, kernel_size=(3, 3, 3), bias=True)# last layer res block@@ -658,18 +303,11 @@ class Decoder(nn.Module):for _ in range(self.num_res_blocks):self.res_blocks.append(ResBlock(filters, filters, **self.block_args))- # TODO: do I need to add adaptive GroupNorm in between each block?-- # # NOTE: upsample, dimensions T, H, W- # self.upsampler_with_t = nn.Upsample(scale_factor=(2,2,2))- # self.upsampler = nn.Upsample(scale_factor=(1,2,2))-# ResBlocks and conv upsample- prev_filters = filters # SCH: in_channelsself.block_res_blocks = nn.ModuleList([])self.num_blocks = len(self.channel_multipliers)self.conv_blocks = nn.ModuleList([])- # SCH: reverse to keep track of the in_channels, but append also in a reverse direction+ # reverse to keep track of the in_channels, but append also in a reverse directionfor i in reversed(range(self.num_blocks)):filters = self.filters * self.channel_multipliers[i]# resblock handling@@ -681,7 +319,7 @@ class Decoder(nn.Module):# conv blocks with upsamplingif i > 0:- if self.temporal_downsample[i]:+ if self.temporal_downsample[i - 1]:t_stride = 2 if self.temporal_downsample[i - 1] else 1# SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2self.conv_blocks.insert(@@ -693,36 +331,22 @@ class Decoder(nn.Module):else:self.conv_blocks.insert(0,- nn.Identity(- prev_filters- ),+ nn.Identity(prev_filters),)-self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)- # NOTE: moved to VAE for separate first frame processing- # self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))-- def forward(- self,- x,- **kwargs,- ):- # dtype, device = x.dtype, x.device+ self.conv_out = self.conv_fn(filters, in_out_channels, 3, dtype=dtype, device=device)+ def forward(self, x):x = self.conv1(x)- # print("decoder:", x.size())for i in range(self.num_res_blocks):x = self.res_blocks[i](x)- # print("decoder:", x.size())- for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoder+ for i in reversed(range(self.num_blocks)):for j in range(self.num_res_blocks):x = self.block_res_blocks[i][j](x)- # print("decoder:", x.size())if i > 0:t_stride = 2 if self.temporal_downsample[i - 1] else 1- # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2x = self.conv_blocks[i - 1](x)x = rearrange(x,@@ -731,278 +355,105 @@ class Decoder(nn.Module):hs=self.s_stride,ws=self.s_stride,)- # print("decoder:", x.size())x = self.norm1(x)x = self.activate(x)- # NOTE: moved to VAE for separate first frame processing- # x = self.conv2(x)+ x = self.conv_out(x)return x@MODELS.register_module()-class VAE_3D_V2(nn.Module): # , ModelMixin- """The 3D VAE"""-+class VAE_Temporal(nn.Module):def __init__(self,- latent_embed_dim=256,+ in_out_channels=4,+ latent_embed_dim=4,+ embed_dim=4,filters=128,- num_res_blocks=2,- separate_first_frame_encoding=False,+ num_res_blocks=4,channel_multipliers=(1, 2, 2, 4),temporal_downsample=(True, True, False),num_groups=32, # for nn.GroupNorm- disable_space=False,- custom_conv_padding=None,activation_fn="swish",- in_out_channels=4,- kl_embed_dim=64,- encoder_double_z=True,device="cpu",- dtype="bf16",+ dtype=torch.bfloat16,):super().__init__()- if type(dtype) == str:- if dtype == "bf16":- dtype = torch.bfloat16- elif dtype == "fp16":- dtype = torch.float16- else:- raise NotImplementedError(f"dtype: {dtype}")-- # ==== Model Params ====- # self.image_size = cast_tuple(image_size, 2)self.time_downsample_factor = 2 ** sum(temporal_downsample)self.time_padding = self.time_downsample_factor - 1- self.separate_first_frame_encoding = separate_first_frame_encoding-- image_down = 2 ** len(temporal_downsample)- t_down = 2 ** len([x for x in temporal_downsample if x == True])- self.patch_size = (t_down, image_down, image_down)-- # ==== Model Initialization ====-- # encoder & decoder first and last conv layer- # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv- self.conv_in = CausalConv3d(- in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device- )- self.conv_in_first_frame = nn.Identity()- self.conv_out_first_frame = nn.Identity()- if separate_first_frame_encoding:- self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3, 3))- self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3, 3))- self.conv_out = CausalConv3d(filters, in_out_channels, 3, dtype=dtype, device=device)+ self.patch_size = (self.time_downsample_factor, 1, 1)+ # NOTE: following MAGVIT, conv in bias=False in encoder first convself.encoder = Encoder(+ in_out_channels=in_out_channels,+ latent_embed_dim=latent_embed_dim * 2,filters=filters,num_res_blocks=num_res_blocks,channel_multipliers=channel_multipliers,temporal_downsample=temporal_downsample,num_groups=num_groups, # for nn.GroupNorm- # in_out_channels = in_out_channels,- latent_embed_dim=latent_embed_dim * 2 if encoder_double_z else latent_embed_dim,- # conv_downsample = conv_downsample,- disable_spatial_downsample=disable_space,- custom_conv_padding=custom_conv_padding,activation_fn=activation_fn,device=device,dtype=dtype,)+ self.quant_conv = CausalConv3d(2 * latent_embed_dim, 2 * embed_dim, 1, device=device, dtype=dtype)++ self.post_quant_conv = CausalConv3d(embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)self.decoder = Decoder(+ in_out_channels=in_out_channels,latent_embed_dim=latent_embed_dim,filters=filters,- # in_out_channels = in_out_channels,num_res_blocks=num_res_blocks,channel_multipliers=channel_multipliers,temporal_downsample=temporal_downsample,num_groups=num_groups, # for nn.GroupNorm- # upsample = upsample, # options: "deconv", "nearest+conv"- disable_spatial_upsample=disable_space,- custom_conv_padding=custom_conv_padding,activation_fn=activation_fn,device=device,dtype=dtype,)- if encoder_double_z:- self.quant_conv = nn.Conv3d(2 * latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)- else:- self.quant_conv = nn.Conv3d(latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)- self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)-def get_latent_size(self, input_size):for i in range(len(input_size)):assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"input_size = [input_size[i] // self.patch_size[i] for i in range(3)]return input_size- def encode(- self,- video,- video_contains_first_frame=True,- ):- self.separate_first_frame_encoding and video_contains_first_frame-- # whether to pad video or not- # if video_contains_first_frame:- # video_len = video.shape[2]- # video = pad_at_dim(video, (self.time_padding, 0), value=0.0, dim=2)- # video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]-- # print("pre-encoder:", video.size())-- # NOTE: moved encoder conv1 here for separate first frame encoding- # if encode_first_frame_separately:- # pad, first_frame, video = unpack(video, video_packed_shape, "b c * h w")- # first_frame = self.conv_in_first_frame(first_frame)- video = self.conv_in(video)-- # print("pre-encoder:", video.size())-- # if encode_first_frame_separately:- # video, _ = pack([first_frame, video], "b c * h w")- # video = pad_at_dim(video, (self.time_padding, 0), dim=2)-- encoded_feature = self.encoder(video)-- # print("after encoder:", encoded_feature.size())-- # NOTE: TODO: do we include this before gaussian distri? or go directly to Gaussian distribution- moments = self.quant_conv(encoded_feature).to(video.dtype)+ def encode(self, x):+ x = pad_at_dim(x, (self.time_padding, 0), dim=2)+ encoded_feature = self.encoder(x)+ moments = self.quant_conv(encoded_feature).to(x.dtype)posterior = DiagonalGaussianDistribution(moments)-- # print("after encoder moments:", moments.size())-return posterior- def decode(- self,- z,- video_contains_first_frame=True,- ):- # dtype = z.dtype- decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame-+ def decode(self, z):z = self.post_quant_conv(z)- # print("pre decoder, post quant conv:", z.size())-- dec = self.decoder(z)- # print("post decoder:", dec.size())-- # SCH: moved decoder last conv layer here for separate first frame decoding- if decode_first_frame_separately:- left_pad, dec_ff, dec = (- dec[:, :, : self.time_padding],- dec[:, :, self.time_padding],- dec[:, :, (self.time_padding + 1) :],- )- out = self.conv_out(dec)- outff = self.conv_out_first_frame(dec_ff)- video, _ = pack([outff, out], "b c * h w")- else:- video = self.conv_out(dec)- # if video were padded, remove padding- if video_contains_first_frame:- video = video[:, :, self.time_padding :]-- # print("conv out:", video.size())-- return video-- def get_last_layer(self):- # CausalConv3d wraps the conv- return self.conv_out.conv.weight-- def forward(- self,- video,- sample_posterior=True,- video_contains_first_frame=True,- # split = "train",- ):- batch, channels, frames = video.shape[:3]- # assert divisible_by(- # frames - int(video_contains_first_frame), self.time_downsample_factor- # ), f"number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}"-- posterior = self.encode(- video,- video_contains_first_frame=video_contains_first_frame,- )+ x = self.decoder(z)+ x = x[:, :, self.time_padding :]+ return x+ def forward(self, x, sample_posterior=True):+ posterior = self.encode(x)if sample_posterior:z = posterior.sample()else:z = posterior.mode()+ recon_video = self.decode(z)+ return recon_video, posterior, z- recon_video = self.decode(z, video_contains_first_frame=video_contains_first_frame)-- return recon_video, posterior--[email protected]_module("VAE_MAGVIT_V2")-def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):- model = VAE_3D_V2(**kwargs)- if from_pretrained is not None:- load_checkpoint(model, from_pretrained, model_name="model")- return model--[email protected]_module("DISCRIMINATOR_3D")-def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):- model = StyleGANDiscriminatorBlur(**kwargs).apply(xavier_uniform_weight_init)- if from_pretrained is not None:- if use_pretrained:- if inflate_from_2d:- load_checkpoint_with_inflation(model, from_pretrained)- else:- load_checkpoint(model, from_pretrained, model_name="discriminator")- print(f"loaded discriminator")- else:- print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")- return model--[email protected]_module("N_Layer_DISCRIMINATOR_3D")-def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):- model = NLayerDiscriminator3D(- input_nc=3,- n_layers=3,- ).apply(n_layer_disc_weights_init)[email protected]_module("VAE_Temporal_SD")+def VAE_Temporal_SD(from_pretrained=None, **kwargs):+ model = VAE_Temporal(+ in_out_channels=4,+ latent_embed_dim=4,+ embed_dim=4,+ filters=128,+ num_res_blocks=3,+ channel_multipliers=(1, 2, 2),+ temporal_downsample=(True, True),+ **kwargs,+ )if from_pretrained is not None:- if use_pretrained:- if inflate_from_2d:- load_checkpoint_with_inflation(model, from_pretrained)- else:- load_checkpoint(model, from_pretrained, model_name="discriminator")- print(f"loaded discriminator")- else:- print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")-+ load_checkpoint(model, from_pretrained)return model---def load_checkpoint_with_inflation(model, ckpt_path):- """- pre-train using image, then inflate to 3D videos- """- if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):- state_dict = find_model(ckpt_path)- with torch.no_grad():- for key in state_dict:- if key in model:- # central inflation- if state_dict[key].size() == model[key][:, :, 0, :, :].size():- # temporal dimension- val = torch.zeros_like(model[key])- centre = int(model[key].size(2) // 2)- val[:, :, centre, :, :] = state_dict[key]- missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)- print(f"Missing keys: {missing_keys}")- print(f"Unexpected keys: {unexpected_keys}")- else:- load_checkpoint(model, ckpt_path) # use the default functiondiff --git a/scripts/inference-vae.py b/scripts/inference-vae.pyindex 63030db..7011807 100644--- a/scripts/inference-vae.py+++ b/scripts/inference-vae.py@@ -9,8 +9,8 @@ from tqdm import tqdmfrom opensora.acceleration.parallel_states import get_data_parallel_groupfrom opensora.datasets import prepare_dataloader, save_sample-from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, VEALoss-from opensora.models.vae.vae_3d import LeCamEMA, pad_at_dim+from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, LeCamEMA, VAELoss+from opensora.models.vae.vae_3d import pad_at_dimfrom opensora.registry import DATASETS, MODELS, build_modulefrom opensora.utils.config_utils import parse_configsfrom opensora.utils.misc import to_torch_dtype@@ -87,7 +87,7 @@ def main():# define loss functionif cfg.calc_loss:- vae_loss_fn = VEALoss(+ vae_loss_fn = VAELoss(logvar_init=cfg.logvar_init,perceptual_loss_weight=cfg.perceptual_loss_weight,kl_loss_weight=cfg.kl_loss_weight,diff --git a/scripts/train-vae.py b/scripts/train-vae.pyindex aef65ac..5200cba 100644--- a/scripts/train-vae.py+++ b/scripts/train-vae.py@@ -1,18 +1,17 @@import os-from glob import glob+from datetime import timedelta+from pprint import pprint-import colossalaiimport torchimport torch.distributed as dist-import wandbfrom colossalai.booster import Boosterfrom colossalai.booster.plugin import LowLevelZeroPluginfrom colossalai.cluster import DistCoordinatorfrom colossalai.nn.optimizer import HybridAdam-from colossalai.utils import get_current_device-from einops import rearrange+from colossalai.utils import get_current_device, set_seedfrom tqdm import tqdm+import wandbfrom opensora.acceleration.checkpoint import set_grad_checkpointfrom opensora.acceleration.parallel_states import (get_data_parallel_group,@@ -20,9 +19,8 @@ from opensora.acceleration.parallel_states import (set_sequence_parallel_group,)from opensora.acceleration.plugin import ZeroSeqParallelPlugin-from opensora.datasets import prepare_dataloader, prepare_variable_dataloader-from opensora.models.vae.vae_3d import pad_at_dim-from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, VEALoss, LeCamEMA+from opensora.datasets import prepare_dataloader+from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, VAELossfrom opensora.registry import DATASETS, MODELS, build_modulefrom opensora.utils.ckpt_utils import create_logger, load_json, save_jsonfrom opensora.utils.config_utils import (@@ -39,49 +37,36 @@ def main():# 1. args & cfg# ======================================================cfg = parse_configs(training=True)+ exp_name, exp_dir = create_experiment_workspace(cfg)+ save_training_config(cfg._cfg_dict, exp_dir)# ======================================================# 2. runtime variables & colossalai launch# ======================================================assert torch.cuda.is_available(), "Training currently requires at least one GPU."+ assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"# 2.1. colossalai init distributed training- colossalai.launch_from_torch({})+ # we set a very large timeout to avoid some processes exit early+ dist.init_process_group(backend="nccl", timeout=timedelta(hours=24))+ torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count())+ set_seed(1024)coordinator = DistCoordinator()-- exp_dir = None- if coordinator.is_master(): # only create directory for master- exp_name, exp_dir = create_experiment_workspace(cfg)- save_training_config(cfg._cfg_dict, exp_dir)- dist.barrier()-- # get exp dir for non-master process- if exp_dir is None:- # experiment_index = len(glob(f"{cfg.outputs}/*")) - 1- # model_name = cfg.model["type"].replace("/", "-")- # exp_name = f"{experiment_index:03d}-{model_name}"- # exp_dir = f"{cfg.outputs}/{exp_name}"- exp_name, exp_dir = create_experiment_workspace(cfg, get_last_workspace=True)- assert os.path.exists(exp_dir)-device = get_current_device()- assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"dtype = to_torch_dtype(cfg.dtype)# 2.2. init logger, tensorboard & wandbif not coordinator.is_master():logger = create_logger(None)else:- print(cfg)+ print("Training configuration:")+ pprint(cfg._cfg_dict)logger = create_logger(exp_dir)logger.info(f"Experiment directory created at {exp_dir}")writer = create_tensorboard_writer(exp_dir)if cfg.wandb:- # wandb.init(project="opensora-vae", name=exp_name, config=cfg._cfg_dict)- # NOTE: here we use the outputs folder name to store running records of different experiments (since frequent interruption)- name = os.path.basename(os.path.normpath(cfg.outputs))- wandb.init(project="opensora-vae", name=name, config=cfg._cfg_dict)+ wandb.init(project="minisora", name=exp_name, config=cfg._cfg_dict)# 2.3. initialize ColossalAI boosterif cfg.plugin == "zero2":@@ -109,6 +94,7 @@ def main():# ======================================================# 3. build dataset and dataloader# ======================================================+ assert cfg.dataset.type == "VideoTextDataset", "Only support VideoTextDataset for now"dataset = build_module(cfg.dataset, DATASETS)logger.info(f"Dataset contains {len(dataset)} samples.")dataloader_args = dict(@@ -122,111 +108,129 @@ def main():process_group=get_data_parallel_group(),)# TODO: use plugin's prepare dataloader- if cfg.bucket_config is None:- dataloader = prepare_dataloader(**dataloader_args)- else:- dataloader = prepare_variable_dataloader(- bucket_config=cfg.bucket_config,- num_bucket_build_workers=cfg.num_bucket_build_workers,- **dataloader_args,- )- if cfg.dataset.type == "VideoTextDataset":- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size- logger.info(f"Total batch size: {total_batch_size}")+ dataloader = prepare_dataloader(**dataloader_args)+ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size+ logger.info(f"Total batch size: {total_batch_size}")# ======================================================# 4. build model# ======================================================# 4.1. build model-- if cfg.get("use_pipeline") == True:- # use 2D VAE, then temporal VAE+ if cfg.get("vae_2d", None) is not None:vae_2d = build_module(cfg.vae_2d, MODELS)+ vae_2d.to(device, dtype).eval()- vae = build_module(cfg.model, MODELS, device=device)- vae_numel, vae_numel_trainable = get_model_numel(vae)- logger.info(- f"Trainable vae params: {format_numel_str(vae_numel_trainable)}, Total model params: {format_numel_str(vae_numel)}"+ model = build_module(+ cfg.model,+ MODELS,+ device=device,+ dtype=dtype,)-- discriminator = build_module(cfg.discriminator, MODELS, device=device)- discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)+ model_numel, model_numel_trainable = get_model_numel(model)logger.info(- f"Trainable discriminator params: {format_numel_str(discriminator_numel_trainable)}, Total model params: {format_numel_str(discriminator_numel)}"+ f"Trainable model params: {format_numel_str(model_numel_trainable)}, Total model params: {format_numel_str(model_numel)}")- # LeCam Initialization- lecam_ema = LeCamEMA(decay=cfg.ema_decay, dtype=dtype, device=device)+ # discriminator = build_module(cfg.discriminator, MODELS, device=device)+ # discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)+ # logger.info(+ # f"Trainable discriminator params: {format_numel_str(discriminator_numel_trainable)}, Total model params: {format_numel_str(discriminator_numel)}"+ # )++ # # LeCam Initialization+ # lecam_ema = LeCamEMA(decay=cfg.ema_decay, dtype=dtype, device=device)# 4.3. move to device- if cfg.get("use_pipeline") == True:- vae_2d.to(device, dtype).eval() # eval mode, not training!+ model = model.to(device, dtype)+ # discriminator = discriminator.to(device, dtype)++ # 4.4 loss functions+ vae_loss_fn = VAELoss(+ logvar_init=cfg.logvar_init,+ perceptual_loss_weight=cfg.perceptual_loss_weight,+ kl_loss_weight=cfg.kl_loss_weight,+ device=device,+ dtype=dtype,+ )++ adversarial_loss_fn = AdversarialLoss(+ discriminator_factor=cfg.discriminator_factor,+ discriminator_start=cfg.discriminator_start,+ generator_factor=cfg.generator_factor,+ generator_loss_type=cfg.generator_loss_type,+ )- vae = vae.to(device, dtype)- discriminator = discriminator.to(device, dtype)+ disc_loss_fn = DiscriminatorLoss(+ discriminator_factor=cfg.discriminator_factor,+ discriminator_start=cfg.discriminator_start,+ discriminator_loss_type=cfg.discriminator_loss_type,+ lecam_loss_weight=cfg.lecam_loss_weight,+ gradient_penalty_loss_weight=cfg.gradient_penalty_loss_weight,+ )# 4.5. setup optimizer# vae optimizeroptimizer = HybridAdam(- filter(lambda p: p.requires_grad, vae.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True+ filter(lambda p: p.requires_grad, model.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True)lr_scheduler = None# disc optimizer- disc_optimizer = HybridAdam(- filter(lambda p: p.requires_grad, discriminator.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True- )- disc_lr_scheduler = None+ # disc_optimizer = HybridAdam(+ # filter(lambda p: p.requires_grad, discriminator.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True+ # )+ # disc_lr_scheduler = None# 4.6. prepare for trainingif cfg.grad_checkpoint:- set_grad_checkpoint(vae)- set_grad_checkpoint(discriminator)- vae.train()- discriminator.train()+ set_grad_checkpoint(model)+ # set_grad_checkpoint(discriminator)+ model.train()+ # discriminator.train()# =======================================================# 5. boost model for distributed training with colossalai# =======================================================torch.set_default_dtype(dtype)- vae, optimizer, _, dataloader, lr_scheduler = booster.boost(- model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader+ model, optimizer, _, dataloader, lr_scheduler = booster.boost(+ model=model,+ optimizer=optimizer,+ lr_scheduler=lr_scheduler,+ dataloader=dataloader,)torch.set_default_dtype(torch.float)num_steps_per_epoch = len(dataloader)- logger.info("Boost vae for distributed training")+ logger.info("Boost model for distributed training")+ num_steps_per_epoch = len(dataloader)- discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(- model=discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler- )- logger.info("Boost discriminator for distributed training")+ # discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(+ # model=discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler+ # )+ # logger.info("Boost discriminator for distributed training")# =======================================================# 6. training loop# =======================================================start_epoch = start_step = log_step = sampler_start_idx = 0+ acc_step = 0running_loss = 0.0running_disc_loss = 0.0-# 6.1. resume trainingif cfg.load is not None:logger.info("Loading checkpoint")- booster.load_model(vae, os.path.join(cfg.load, "model"))- booster.load_model(discriminator, os.path.join(cfg.load, "discriminator"))+ booster.load_model(model, os.path.join(cfg.load, "model"))booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))- booster.load_optimizer(disc_optimizer, os.path.join(cfg.load, "disc_optimizer"))- if lr_scheduler is not None:- booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))- if disc_lr_scheduler is not None:- booster.load_lr_scheduler(disc_lr_scheduler, os.path.join(cfg.load, "disc_lr_scheduler"))++ # booster.load_model(discriminator, os.path.join(cfg.load, "discriminator"))+ # booster.load_optimizer(disc_optimizer, os.path.join(cfg.load, "disc_optimizer"))# LeCam EMA for discriminator- lecam_path = os.path.join(cfg.load, "lecam_states.json")- if cfg.lecam_loss_weight is not None and os.path.exists(lecam_path):- lecam_state = load_json(lecam_path)- lecam_ema_real, lecam_ema_fake = lecam_state["lecam_ema_real"], lecam_state["lecam_ema_fake"]- lecam_ema = LeCamEMA(- decay=cfg.ema_decay, ema_real=lecam_ema_real, ema_fake=lecam_ema_fake, dtype=dtype, device=device- )+ # lecam_path = os.path.join(cfg.load, "lecam_states.json")+ # if cfg.lecam_loss_weight is not None and os.path.exists(lecam_path):+ # lecam_state = load_json(lecam_path)+ # lecam_ema_real, lecam_ema_fake = lecam_state["lecam_ema_real"], lecam_state["lecam_ema_fake"]+ # lecam_ema = LeCamEMA(+ # decay=cfg.ema_decay, ema_real=lecam_ema_real, ema_fake=lecam_ema_fake, dtype=dtype, device=device+ # )running_states = load_json(os.path.join(cfg.load, "running_states.json"))dist.barrier()@@ -240,39 +244,15 @@ def main():dataloader.sampler.set_start_index(sampler_start_idx)- # 6.2 Define loss functions- vae_loss_fn = VEALoss(- logvar_init=cfg.logvar_init,- perceptual_loss_weight=cfg.perceptual_loss_weight,- kl_loss_weight=cfg.kl_loss_weight,- device=device,- dtype=dtype,- )-- adversarial_loss_fn = AdversarialLoss(- discriminator_factor=cfg.discriminator_factor,- discriminator_start=cfg.discriminator_start,- generator_factor=cfg.generator_factor,- generator_loss_type=cfg.generator_loss_type,- )-- disc_loss_fn = DiscriminatorLoss(- discriminator_factor=cfg.discriminator_factor,- discriminator_start=cfg.discriminator_start,- discriminator_loss_type=cfg.discriminator_loss_type,- lecam_loss_weight=cfg.lecam_loss_weight,- gradient_penalty_loss_weight=cfg.gradient_penalty_loss_weight,- )-# 6.3. training loop# calculate discriminator_time_padding- disc_time_downsample_factor = 2 ** len(cfg.discriminator.channel_multipliers)- if cfg.dataset.num_frames % disc_time_downsample_factor != 0:- disc_time_padding = disc_time_downsample_factor - cfg.dataset.num_frames % disc_time_downsample_factor- else:- disc_time_padding = 0- video_contains_first_frame = cfg.video_contains_first_frame+ # disc_time_downsample_factor = 2 ** len(cfg.discriminator.channel_multipliers)+ # if cfg.dataset.num_frames % disc_time_downsample_factor != 0:+ # disc_time_padding = disc_time_downsample_factor - cfg.dataset.num_frames % disc_time_downsample_factor+ # else:+ # disc_time_padding = 0+ # video_contains_first_frame = cfg.video_contains_first_framefor epoch in range(start_epoch, cfg.epochs):dataloader.sampler.set_epoch(epoch)@@ -280,88 +260,47 @@ def main():logger.info(f"Beginning epoch {epoch}...")with tqdm(- range(start_step, num_steps_per_epoch),+ enumerate(dataloader_iter, start=start_step),desc=f"Epoch {epoch}",disable=not coordinator.is_master(),total=num_steps_per_epoch,initial=start_step,) as pbar:- # with profile(- # activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],- # schedule=torch.profiler.schedule(- # wait=1,- # warmup=1,- # active=2,- # repeat=2,- # ),- # on_trace_ready=torch.profiler.tensorboard_trace_handler('/home/shenchenhui/log'),- # with_stack=True,- # record_shapes=True,- # profile_memory=True,- # ) as p: # trace efficiency-- for step in pbar:- # with profile(- # activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],- # with_stack=True,- # ) as p: # trace efficiency-- # SCH: calc global step at the start- global_step = epoch * num_steps_per_epoch + step-- batch = next(dataloader_iter)+ for step, batch in pbar:x = batch["video"].to(device, dtype) # [B, C, T, H, W]- # supprt for image or video inputs- assert x.ndim in {4, 5}, f"received input of {x.ndim} dimensions" # either image or video- assert (- x.shape[-2:] == cfg.dataset.image_size- ), f"received input size {x.shape[-2:]}, but config image size is {cfg.dataset.image_size}"- is_image = x.ndim == 4- if is_image:- video = rearrange(x, "b c ... -> b c 1 ...")- video_contains_first_frame = True- else:- video = x-# ===== Spatial VAE =====- if cfg.get("use_pipeline") == True:+ if cfg.get("vae_2d", None) is not None:with torch.no_grad():- t_video = vae_2d.encode(video)+ x_z = vae_2d.encode(x)# ====== VAE ======- t_recon_video, posterior = vae(- t_video,- video_contains_first_frame=video_contains_first_frame,- )-- recon_video = vae_2d.decode(t_recon_video)+ x_z_rec, posterior, z = model(x_z)+ x_rec = vae_2d.decode(x_z_rec)# ====== Generator Loss ======# simple nll loss- nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(- video, recon_video, posterior- )-- _, weighted_z_nll_loss, _ = vae_loss_fn(- t_video, t_recon_video, posterior- )+ _, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(x, x_rec, posterior)+ _, weighted_z_nll_loss, _ = vae_loss_fn(x_z, x_z_rec, posterior)+ # _, image_identity_loss, _ = vae_loss_fn(x_z, z, posterior)- adversarial_loss = torch.tensor(0.0)+ # adversarial_loss = torch.tensor(0.0)# adversarial loss- if global_step > cfg.discriminator_start:- # padded videos for GAN- fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)- fake_logits = discriminator(fake_video.contiguous())- adversarial_loss = adversarial_loss_fn(- fake_logits,- nll_loss,- vae.module.get_last_layer(),- global_step,- is_training=vae.training,- )-- vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss + weighted_z_nll_loss+ # if global_step > cfg.discriminator_start:+ # # padded videos for GAN+ # fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)+ # fake_logits = discriminator(fake_video.contiguous())+ # adversarial_loss = adversarial_loss_fn(+ # fake_logits,+ # nll_loss,+ # vae.module.get_last_layer(),+ # global_step,+ # is_training=vae.training,+ # )++ # vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss + weighted_z_nll_loss+ # vae_loss = weighted_nll_loss + weighted_kl_loss + weighted_z_nll_loss + image_identity_loss+ vae_loss = weighted_nll_loss + weighted_kl_loss + weighted_z_nll_lossoptimizer.zero_grad()# Backward & update@@ -373,59 +312,60 @@ def main():# Log loss values:all_reduce_mean(vae_loss) # NOTE: this is to get average loss for loggingrunning_loss += vae_loss.item()+ global_step = epoch * num_steps_per_epoch + step+ log_step += 1+ acc_step += 1# ====== Discriminator Loss ======- if global_step > cfg.discriminator_start:- # if video_contains_first_frame:- # Since we don't have enough T frames, pad anyways- real_video = pad_at_dim(video, (disc_time_padding, 0), value=0.0, dim=2)- fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)-- if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:- real_video = real_video.requires_grad_()- real_logits = discriminator(- real_video.contiguous()- ) # SCH: not detached for now for gradient_penalty calculation- else:- real_logits = discriminator(real_video.contiguous().detach())-- fake_logits = discriminator(fake_video.contiguous().detach())-- lecam_ema_real, lecam_ema_fake = lecam_ema.get()-- weighted_d_adversarial_loss, lecam_loss, gradient_penalty_loss = disc_loss_fn(- real_logits,- fake_logits,- global_step,- lecam_ema_real=lecam_ema_real,- lecam_ema_fake=lecam_ema_fake,- real_video=real_video if cfg.gradient_penalty_loss_weight is not None else None,- )- disc_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty_loss- if cfg.lecam_loss_weight is not None:- ema_real = torch.mean(real_logits.clone().detach()).to(device, dtype)- ema_fake = torch.mean(fake_logits.clone().detach()).to(device, dtype)- all_reduce_mean(ema_real)- all_reduce_mean(ema_fake)- lecam_ema.update(ema_real, ema_fake)-- disc_optimizer.zero_grad()- # Backward & update- booster.backward(loss=disc_loss, optimizer=disc_optimizer)- # # NOTE: TODO: clip gradients? this is done in Open-Sora-Plan- # torch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1) # NOTE: done by grad_clip- disc_optimizer.step()-- # Log loss values:- all_reduce_mean(disc_loss)- running_disc_loss += disc_loss.item()- else:- disc_loss = torch.tensor(0.0)- weighted_d_adversarial_loss = torch.tensor(0.0)- lecam_loss = torch.tensor(0.0)- gradient_penalty_loss = torch.tensor(0.0)-- log_step += 1+ # if global_step > cfg.discriminator_start:+ # # if video_contains_first_frame:+ # # Since we don't have enough T frames, pad anyways+ # real_video = pad_at_dim(video, (disc_time_padding, 0), value=0.0, dim=2)+ # fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)++ # if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:+ # real_video = real_video.requires_grad_()+ # real_logits = discriminator(+ # real_video.contiguous()+ # ) # SCH: not detached for now for gradient_penalty calculation+ # else:+ # real_logits = discriminator(real_video.contiguous().detach())++ # fake_logits = discriminator(fake_video.contiguous().detach())++ # lecam_ema_real, lecam_ema_fake = lecam_ema.get()++ # weighted_d_adversarial_loss, lecam_loss, gradient_penalty_loss = disc_loss_fn(+ # real_logits,+ # fake_logits,+ # global_step,+ # lecam_ema_real=lecam_ema_real,+ # lecam_ema_fake=lecam_ema_fake,+ # real_video=real_video if cfg.gradient_penalty_loss_weight is not None else None,+ # )+ # disc_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty_loss+ # if cfg.lecam_loss_weight is not None:+ # ema_real = torch.mean(real_logits.clone().detach()).to(device, dtype)+ # ema_fake = torch.mean(fake_logits.clone().detach()).to(device, dtype)+ # all_reduce_mean(ema_real)+ # all_reduce_mean(ema_fake)+ # lecam_ema.update(ema_real, ema_fake)++ # disc_optimizer.zero_grad()+ # # Backward & update+ # booster.backward(loss=disc_loss, optimizer=disc_optimizer)+ # # # NOTE: TODO: clip gradients? this is done in Open-Sora-Plan+ # # torch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1) # NOTE: done by grad_clip+ # disc_optimizer.step()++ # # Log loss values:+ # all_reduce_mean(disc_loss)+ # running_disc_loss += disc_loss.item()+ # else:+ # disc_loss = torch.tensor(0.0)+ # weighted_d_adversarial_loss = torch.tensor(0.0)+ # lecam_loss = torch.tensor(0.0)+ # gradient_penalty_loss = torch.tensor(0.0)# Log to tensorboardif coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:@@ -446,11 +386,13 @@ def main():"epoch": epoch,"loss": vae_loss.item(),"kl_loss": weighted_kl_loss.item(),- "gen_adv_loss": adversarial_loss.item(),- "disc_loss": disc_loss.item(),- "lecam_loss": lecam_loss.item(),- "r1_grad_penalty": gradient_penalty_loss.item(),+ # "gen_adv_loss": adversarial_loss.item(),+ # "disc_loss": disc_loss.item(),+ # "lecam_loss": lecam_loss.item(),+ # "r1_grad_penalty": gradient_penalty_loss.item(),"nll_loss": weighted_nll_loss.item(),+ "z_nll_loss": weighted_z_nll_loss.item(),+ # "image_identity_loss": image_identity_loss.item(),"avg_loss": avg_loss,},step=global_step,@@ -460,19 +402,14 @@ def main():if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")os.makedirs(os.path.join(save_dir, "model"), exist_ok=True) # already handled in booster save_model- booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)- booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)+ booster.save_model(model, os.path.join(save_dir, "model"), shard=True)+ # booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)- booster.save_optimizer(- disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096- )-- if lr_scheduler is not None:- booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))- if disc_lr_scheduler is not None:- booster.save_lr_scheduler(disc_lr_scheduler, os.path.join(save_dir, "disc_lr_scheduler"))+ # booster.save_optimizer(+ # disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096+ # )running_states = {"epoch": epoch,@@ -481,15 +418,15 @@ def main():"sample_start_index": (step + 1) * cfg.batch_size,}- lecam_ema_real, lecam_ema_fake = lecam_ema.get()- lecam_state = {- "lecam_ema_real": lecam_ema_real.item(),- "lecam_ema_fake": lecam_ema_fake.item(),- }+ # lecam_ema_real, lecam_ema_fake = lecam_ema.get()+ # lecam_state = {+ # "lecam_ema_real": lecam_ema_real.item(),+ # "lecam_ema_fake": lecam_ema_fake.item(),+ # }if coordinator.is_master():save_json(running_states, os.path.join(save_dir, "running_states.json"))- if cfg.lecam_loss_weight is not None:- save_json(lecam_state, os.path.join(save_dir, "lecam_states.json"))+ # if cfg.lecam_loss_weight is not None:+ # save_json(lecam_state, os.path.join(save_dir, "lecam_states.json"))dist.barrier()logger.info(