Open-Sora: Democratizing Efficient Video Production for All
Python
debug
Shen-Chenhui686829969f78
diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex e85dcda..98db5f7 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -868,7 +868,6 @@ class VAE_3D_V2(nn.Module):# recon_loss = F.mse_loss(video, recon_video)- # breakpoint()# total_loss = recon_loss# # KL Loss@@ -924,7 +923,6 @@ class VAE_3D_V2(nn.Module):# disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)# weighted_gan_loss = d_weight * disc_factor * gan_loss- # breakpoint()# total_loss += weighted_gan_loss# log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),@@ -1057,8 +1055,8 @@ class AdversarialLoss(nn.Module):self.discriminator_loss_weight = discriminator_loss_weightdef calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added create_graph=Trued_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()d_weight = d_weight * self.discriminator_loss_weight@@ -1074,6 +1072,8 @@ class AdversarialLoss(nn.Module):):gan_loss = -torch.mean(fake_logits)if self.discriminator_factor is not None and self.discriminator_factor > 0.0:+ # DEBUG: NOTE: something about this d_weigth calculation is causing issue+ # d_weight = 1try:d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, last_layer)except RuntimeError:diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pyindex adeb112..7f764bd 100644--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@ -235,8 +235,8 @@ def main():disc_time_padding = disc_time_downsample_factor - cfg.num_frames % disc_time_downsample_factorvideo_contains_first_frame = cfg.video_contains_first_frame- lecam_ema_real = np.asarray(0)- lecam_ema_fake = np.asarray(0)+ lecam_ema_real = torch.tensor(0.0)+ lecam_ema_fake = torch.tensor(0.0)for epoch in range(start_epoch, cfg.epochs):dataloader.sampler.set_epoch(epoch)@@ -268,7 +268,6 @@ def main():else:video = x-# ====== VAE ======# this is essential for the first iteration after OOM# optimizer._grad_store.reset_all_gradients()@@ -281,11 +280,16 @@ def main():# optimizer._bucket_store._grad_in_bucket[rank] = []# optimizer._bucket_store.offset_list = [0]# optimizer.zero_grad()+optimizer.zero_grad()recon_video, posterior = vae(video,video_contains_first_frame = video_contains_first_frame,)++++ # ====== Generator Loss ======# simple nll lossnll_loss = nll_loss_fn(video,@@ -310,21 +314,25 @@ def main():vae_loss += adversarial_loss# Backward & updatebooster.backward(loss=vae_loss, optimizer=optimizer)+ # NOTE: clip gradients? this is done in Open-Sora-Plan+ torch.nn.utils.clip_grad_norm_(vae.parameters(), 1)optimizer.step()# Log loss values:all_reduce_mean(vae_loss)running_loss += vae_loss.item()- # ====== Discriminator ======+ # TODO: debug, move after generator loss later+ # ====== Discriminator Loss ======if global_step > cfg.discriminator_start:disc_optimizer.zero_grad()# if video_contains_first_frame:# Since we don't have enough T frames, pad anywaysreal_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:real_video = real_video.requires_grad_()- real_logits = discriminator(real_video.contiguous().detach())+ real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculationfake_logits = discriminator(fake_video.contiguous().detach())disc_loss = disc_loss_fn(real_logits,@@ -337,12 +345,14 @@ def main():if cfg.ema_decay is not None:# SCH: TODO: is this written properly like this for moving average? e.g. distributed training etc.- lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * np.mean(real_logits.clone().detach())- lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * np.mean(fake_logits.clone().detach())+ lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(real_logits.clone().detach())+ lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(fake_logits.clone().detach())-# Backward & updatebooster.backward(loss=disc_loss, optimizer=disc_optimizer)+ # NOTE: TODO: clip gradients? this is done in Open-Sora-Plan+ torch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1)+disc_optimizer.step()# Log loss values: