Open-Sora: Democratizing Efficient Video Production for All
Python
add logvar
Shen-Chenhui235d195daca5
diff --git a/configs/vae_magvit_v2/train/16x128x128.py b/configs/vae_magvit_v2/train/16x128x128.pynew file mode 100644index 0000000..706b4ce--- /dev/null+++ b/configs/vae_magvit_v2/train/16x128x128.py@@ -0,0 +1,81 @@+num_frames = 16+frame_interval = 3+image_size = (128, 128)++# Define dataset+root = None+data_path = "CSV_PATH"+use_image_transform = False+num_workers = 4+video_contains_first_frame = False++# Define acceleration+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1+++# Define model++model = dict(+ type="VAE_MAGVIT_V2",+ in_out_channels = 3,+ latent_embed_dim = 256,+ filters = 128,+ num_res_blocks = 4,+ channel_multipliers = (1, 2, 2, 4),+ temporal_downsample = (False, True, True),+ num_groups = 32, # for nn.GroupNorm+ kl_embed_dim = 64,+ activation_fn = 'swish',+ separate_first_frame_encoding = False,+ custom_conv_padding = None+)+++discriminator = dict(+ type="DISCRIMINATOR_3D",+ image_size = image_size,+ num_frames = num_frames,+ in_channels = 3,+ filters = 128,+ channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution+)+++# loss weights+logvar_init=0.0+kl_loss_weight = 0.000001+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0+discriminator_factor = 1.0 # for discriminator adversarial loss+# discriminator_loss_weight = 0.5 # for generator adversarial loss+generator_factor = 0.5 # for generator adversarial loss+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight+discriminator_loss_type="non-saturating"+generator_loss_type="non-saturating"+discriminator_start = 50000 # 50000 NOTE: change to correct val, debug use -1 for now+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use+ema_decay = 0.999 # ema decay factor for generator+++# Others+seed = 42+outputs = "outputs"+wandb = False++# Training+''' NOTE:+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],+3-6 epochs for pexel, from pexel observation its correct+'''++epochs = 10+log_every = 1+ckpt_every = 1000+load = None++batch_size = 4+lr = 1e-4+grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.pyindex 8684ba2..d357911 100644--- a/configs/vae_magvit_v2/train/17x128x128.py+++ b/configs/vae_magvit_v2/train/17x128x128.py@@ -31,16 +31,6 @@ model = dict(activation_fn = 'swish',separate_first_frame_encoding = False,custom_conv_padding = None- # num_frames = num_frames,- # image_size = image_size,- # kl_loss_weight = 0.000001,- # perceptual_loss_weight = 0.1, # use vgg is not None and more than 0- # discriminator_factor = 1.0,- # discriminator_in_channels = 3,- # discriminator_filters = 128,- # discriminator_channel_multipliers = (2,4,4,4,4),- # discriminator_loss="hinge",- # discriminator_start = 50001,)@@ -57,12 +47,14 @@ discriminator = dict(# loss weightskl_loss_weight = 0.000001perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0-discriminator_loss_weight = 0.5+discriminator_factor = 1.0 # for discriminator adversarial loss+# discriminator_loss_weight = 0.5 # for generator adversarial loss+generator_factor = 0.5 # for generator adversarial losslecam_loss_weight = None # TODO: not clear in MAGVIT what is the weight-discriminator_loss="non-saturating"+discriminator_loss_type="non-saturating"+generator_loss_type="non-saturating"discriminator_start = 50000 # 50000 TODO: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: following MAGVIT config.vqgan.grad_penalty_cost, 10+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't useema_decay = 0.999 # ema decay factor for generatordiff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex 6a8ec42..5fdd923 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -12,12 +12,13 @@ from einops import rearrange, repeat, pack, unpackimport torch.nn.functional as Fimport torchvisionfrom torchvision.models import VGG16_Weights-from collections import namedtuplefrom taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers-from torch import nn, einsum, Tensor-from kornia.filters import filter3d+from torch import nnimport math+# from diffusers.models.modeling_utils import ModelMixin++"""Encoder and Decoder stuctures with 3D CNNs.""""""@@ -28,8 +29,6 @@ NOTE:!!! opensora read video into [B,C,T,H,W] format output-TODO:- check data dimensions format"""def cast_tuple(t, length = 1):return t if isinstance(t, tuple) else ((t,) * length)@@ -83,8 +82,8 @@ def gradient_penalty_fn(images, output):# ============== Discriminator Adversarial Loss Functions ==============def hinge_d_loss(logits_real, logits_fake):- loss_real = torch.mean(F.relu(1. - logits_real))- loss_fake = torch.mean(F.relu(1. + logits_fake))+ loss_real = torch.mean(F.relu(1.0 - logits_real))+ loss_fake = torch.mean(F.relu(1.0 + logits_fake))d_loss = 0.5 * (loss_real + loss_fake)return d_loss@@ -261,7 +260,7 @@ class BlurPool3D(nn.Module):filt_2d = a[:,None]*a[None,:]filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)- filt = filt_3d/torch.sum(filt_3d) # TODO: modify to make it 3D+ filt = filt_3d/torch.sum(filt_3d) # SCH: modified to it 3Dself.register_buffer('filt', filt[None,None,:,:,:].repeat((self.channels,1,1,1,1)))self.pad = get_pad_layer(pad_type)(self.pad_sizes)@@ -657,7 +656,7 @@ class Decoder(nn.Module):@MODELS.register_module()-class VAE_3D_V2(nn.Module):+class VAE_3D_V2(nn.Module): # , ModelMixin"""The 3D VAE """def __init__(self,@@ -870,10 +869,11 @@ class VAE_3D_V2(nn.Module):class VEALoss(nn.Module):def __init__(self,+ logvar_init=0.0,perceptual_loss_weight = 0.1,kl_loss_weight = 0.000001,- vgg=None,- vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,+ # vgg=None,+ # vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,device = "cpu",dtype = "bf16"):@@ -890,15 +890,18 @@ class VEALoss(nn.Module):# KL Lossself.kl_loss_weight = kl_loss_weight# Perceptual Loss- self.vgg = None+ self.perceptual_loss_fn = LPIPS().eval()self.perceptual_loss_weight = perceptual_loss_weight- if perceptual_loss_weight is not None and perceptual_loss_weight > 0.0:- if not exists(vgg):- vgg = torchvision.models.vgg16(- weights = vgg_weights- )- vgg.classifier = Sequential(*vgg.classifier[:-2])- self.vgg = vgg.to(device, dtype).eval() # SCH: added eval+ self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)++ # self.vgg = None+ # if perceptual_loss_weight is not None and perceptual_loss_weight > 0.0:+ # if not exists(vgg):+ # vgg = torchvision.models.vgg16(+ # weights = vgg_weights+ # )+ # vgg.classifier = Sequential(*vgg.classifier[:-2])+ # self.vgg = vgg.to(device, dtype).eval() # SCH: added evaldef forward(@@ -906,40 +909,69 @@ class VEALoss(nn.Module):video,recon_video,posterior,+ nll_weights=None,split = "train",- ):- # reconstruction loss- recon_loss = F.mse_loss(video, recon_video)- nll_loss = recon_loss+ ):+ video = rearrange(video, "b c t h w -> (b t) c h w").contiguous()+ recon_video = rearrange(recon_video, "b c t h w -> (b t) c h w").contiguous()- # KL Loss- weighted_kl_loss = 0- if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:- kl_loss = posterior.kl()- # NOTE: since we use MSE, here use mean as well, else use sum- kl_loss = torch.mean(kl_loss) / kl_loss.shape[0]- weighted_kl_loss = kl_loss * self.kl_loss_weight- nll_loss += weighted_kl_loss+ # reconstruction loss+ recon_loss = torch.abs(video - recon_video)# perceptual loss- weighted_perceptual_loss = 0if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:- assert exists(self.vgg)- batch, channels, frames = video.shape[:3]- frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices- input_vgg_input = pick_video_frame(video, frame_indices)- recon_vgg_input = pick_video_frame(recon_video, frame_indices)+ # handle channels+ channels = video.shape[1]if channels == 1:input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)elif channels == 4: # SCH: take the first 3 for perceptual loss calcinput_vgg_input = input_vgg_input[:, :3]recon_vgg_input = recon_vgg_input[:, :3]- input_vgg_feats = self.vgg(input_vgg_input)- recon_vgg_feats = self.vgg(recon_vgg_input)- perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)- weighted_perceptual_loss = perceptual_loss * self.perceptual_loss_weight- nll_loss += weighted_perceptual_loss+ perceptual_loss = self.perceptual_loss_fn(video, recon_video)+ recon_loss = recon_loss + self.perceptual_loss_weight * perceptual_loss++ nll_loss = recon_loss / torch.exp(self.logvar) + self.logvar+ weighted_nll_loss = nll_loss+ if nll_weights is not None:+ weighted_nll_loss = nll_weights * nll_loss+ weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]+ nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]++ # recon_loss = F.mse_loss(video, recon_video)+ # nll_loss = recon_loss++ # KL Loss+ weighted_kl_loss = 0+ if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:+ kl_loss = posterior.kl()+ # # NOTE: since we use MSE, here use mean as well, else use sum+ # kl_loss = torch.mean(kl_loss) / kl_loss.shape[0]+ kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]+ weighted_kl_loss = kl_loss * self.kl_loss_weight++ return nll_loss, weighted_nll_loss, weighted_kl_loss++ # # perceptual loss+ # # TODO: use all frames and take average instead of sampling+ # weighted_perceptual_loss = 0+ # if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:+ # assert exists(self.vgg)+ # batch, channels, frames = video.shape[:3]+ # frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices+ # input_vgg_input = pick_video_frame(video, frame_indices)+ # recon_vgg_input = pick_video_frame(recon_video, frame_indices)+ # if channels == 1:+ # input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)+ # recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)+ # elif channels == 4: # SCH: take the first 3 for perceptual loss calc+ # input_vgg_input = input_vgg_input[:, :3]+ # recon_vgg_input = recon_vgg_input[:, :3]+ # input_vgg_feats = self.vgg(input_vgg_input)+ # recon_vgg_feats = self.vgg(recon_vgg_input)+ # perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)+ # weighted_perceptual_loss = perceptual_loss * self.perceptual_loss_weight+ # nll_loss += weighted_perceptual_loss# log = {# "{}/total_loss".format(split): nll_loss.clone().detach().mean(),@@ -948,7 +980,7 @@ class VEALoss(nn.Module):# "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),# }- return nll_loss+ # return nll_loss, weighted_kl_lossclass AdversarialLoss(nn.Module):@@ -956,19 +988,21 @@ class AdversarialLoss(nn.Module):self,discriminator_factor = 1.0,discriminator_start = 50001,- discriminator_loss_weight = 0.5,+ generator_factor = 0.5,+ generator_loss_type="non-saturating",):super().__init__()self.discriminator_factor = discriminator_factorself.discriminator_start = discriminator_start- self.discriminator_loss_weight = discriminator_loss_weight+ self.generator_factor = generator_factor+ self.generator_loss_type = generator_loss_typedef calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creatg_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added create_graph=Trued_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()- d_weight = d_weight * self.discriminator_loss_weight+ d_weight = d_weight * self.generator_factorreturn d_weightdef forward(@@ -979,12 +1013,22 @@ class AdversarialLoss(nn.Module):global_step,is_training = True,):- gan_loss = -torch.mean(fake_logits)+ # NOTE: following MAGVIT to allow non_saturating+ assert self.generator_loss_type in ["hinge", "vanilla", "non-saturating"]+ if self.generator_loss_type == "hinge":+ gen_loss = -torch.mean(fake_logits)+ elif self.generator_loss_type == "non-saturating":+ gen_loss = torch.mean(+ sigmoid_cross_entropy_with_logits(+ labels=torch.ones_like(fake_logits), logits=fake_logits)+ )+ else:+ raise ValueError("Generator loss {} not supported".format(self.generator_loss_type))++if self.discriminator_factor is not None and self.discriminator_factor > 0.0:- # DEBUG: NOTE: something about this d_weigth calculation is causing issue- # d_weight = 1try:- d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, last_layer)+ d_weight = self.calculate_adaptive_weight(nll_loss, gen_loss, last_layer)except RuntimeError:assert not is_trainingd_weight = torch.tensor(0.0)@@ -992,9 +1036,9 @@ class AdversarialLoss(nn.Module):d_weight = torch.tensor(0.0)disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)- weighted_gan_loss = d_weight * disc_factor * gan_loss+ weighted_gen_loss = d_weight * disc_factor * gen_loss- return weighted_gan_loss+ return weighted_gen_loss@@ -1003,18 +1047,18 @@ class DiscriminatorLoss(nn.Module):self,discriminator_factor = 1.0,discriminator_start = 50001,- discriminator_loss="non-saturating",- lecam_loss_weight=0,- gradient_penalty_loss_weight=10, # SCH: following MAGVIT config.vqgan.grad_penalty_cost+ discriminator_loss_type="non-saturating",+ lecam_loss_weight=None,+ gradient_penalty_loss_weight=None, # SCH: following MAGVIT config.vqgan.grad_penalty_cost):super().__init__()- assert discriminator_loss in ["hinge", "vanilla", "non-saturating"]+ assert discriminator_loss_type in ["hinge", "vanilla", "non-saturating"]self.discriminator_factor = discriminator_factorself.discriminator_start = discriminator_startself.lecam_loss_weight = lecam_loss_weightself.gradient_penalty_loss_weight = gradient_penalty_loss_weight- self.discriminator_loss_type = discriminator_loss+ self.discriminator_loss_type = discriminator_loss_typedef forward(diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pyindex 88a11f8..a210193 100644--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@ -37,6 +37,8 @@ from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numfrom opensora.utils.train_utils import update_ema, MaskGeneratorfrom opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, pad_at_dim++# efficiency# from torch.profiler import profile, record_function, ProfilerActivity@@ -218,7 +220,8 @@ def main():dataloader.sampler.set_start_index(sampler_start_idx)# 6.2 Define loss functions- nll_loss_fn = VEALoss(+ vae_loss_fn = VEALoss(+ logvar_init=cfg.logvar_init,perceptual_loss_weight = cfg.perceptual_loss_weight,kl_loss_weight = cfg.kl_loss_weight,device=device,@@ -228,12 +231,14 @@ def main():adversarial_loss_fn = AdversarialLoss(discriminator_factor = cfg.discriminator_factor,discriminator_start = cfg.discriminator_start,+ generator_factor = cfg.generator_factor,+ generator_loss_type = cfg.generator_loss_type,)disc_loss_fn = DiscriminatorLoss(discriminator_factor = cfg.discriminator_factor,discriminator_start = cfg.discriminator_start,- discriminator_loss = cfg.discriminator_loss,+ discriminator_loss_type = cfg.discriminator_loss_type,lecam_loss_weight = cfg.lecam_loss_weight,gradient_penalty_loss_weight = cfg.gradient_penalty_loss_weight,)@@ -303,7 +308,6 @@ def main():video = x# ====== VAE ======- optimizer.zero_grad()recon_video, posterior = vae(video,video_contains_first_frame = video_contains_first_frame,@@ -311,19 +315,18 @@ def main():# ====== Generator Loss ======# simple nll loss- nll_loss = nll_loss_fn(+ nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(video,recon_video,posterior,split = "train")- vae_loss = nll_loss-+# adversarial lossif global_step > cfg.discriminator_start:# padded videos for GANfake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)- fake_logits = discriminator(fake_video) # TODO: take out contiguous?+ fake_logits = discriminator(fake_video.contiguous()) # TODO: take out contiguous?adversarial_loss = adversarial_loss_fn(fake_logits,nll_loss,@@ -331,12 +334,16 @@ def main():global_step,is_training = vae.training,)- vae_loss += adversarial_loss- # Backward & update++ vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss++ optimizer.zero_grad()+ # Backward & updatebooster.backward(loss=vae_loss, optimizer=optimizer)# NOTE: clip gradients? this is done in Open-Sora-Plantorch.nn.utils.clip_grad_norm_(vae.parameters(), 1)optimizer.step()+# Log loss values:all_reduce_mean(vae_loss)running_loss += vae_loss.item()@@ -345,7 +352,7 @@ def main():# ====== Discriminator Loss ======if global_step > cfg.discriminator_start:- disc_optimizer.zero_grad()+# if video_contains_first_frame:# Since we don't have enough T frames, pad anyways@@ -354,11 +361,9 @@ def main():if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:real_video = real_video.requires_grad_()-- real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculation-- if cfg.gradient_penalty_loss_weight is None:- real_logits = real_logits.detach()+ real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculation+ else:+ real_logits = discriminator(real_video.contiguous().detach())fake_logits = discriminator(fake_video.contiguous().detach())disc_loss = disc_loss_fn(@@ -375,11 +380,11 @@ def main():lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(real_logits.clone().detach())lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(fake_logits.clone().detach())+ disc_optimizer.zero_grad()# Backward & updatebooster.backward(loss=disc_loss, optimizer=disc_optimizer)# NOTE: TODO: clip gradients? this is done in Open-Sora-Plantorch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1)-disc_optimizer.step()# Log loss values: