Open-Sora: Democratizing Efficient Video Production for All
Python
enable r1 gradient penalty
Shen-Chenhui48e3a84ba6b2
diff --git a/configs/vae_magvit_v2/train/16x128x128_z4_LC.py b/configs/vae_magvit_v2/train/16x128x128_z4_LC.pyindex 9310257..2cb7cb6 100644--- a/configs/vae_magvit_v2/train/16x128x128_z4_LC.py+++ b/configs/vae_magvit_v2/train/16x128x128_z4_LC.py@@ -56,7 +56,7 @@ lecam_loss_weight = 0.001 # NOTE: MAVGIT v2 use 0.001discriminator_loss_type="non-saturating"generator_loss_type="non-saturating"discriminator_start = -1 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use+gradient_penalty_loss_weight = 10 # 10 # SCH: MAGVIT uses 10, opensora plan doesn't useema_decay = 0.999 # ema decay for LeCamdiff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.mdindex e180eea..a0bf1ac 100644--- a/opensora/models/vae/README.md+++ b/opensora/models/vae/README.md@@ -5,6 +5,7 @@* https://github.com/google-research/magvit* https://github.com/CompVis/taming-transformers* https://github.com/adobe/antialiased-cnns/pull/39/commits/3d6f02b6943c58b68c19c07bc26fad57492ff3bc+* https://github.com/PKU-YuanGroup/Open-Sora-Plan## 1. VAE 3Ddiff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex 628c013..b834513 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -64,6 +64,63 @@ def lecam_reg(real_pred, fake_pred, ema_real_pred, ema_fake_pred):lecam_loss += torch.mean(torch.pow(nn.ReLU()(ema_real_pred - fake_pred), 2))return lecam_loss+# Open-Sora-Plan+def r1_penalty(real_img, real_pred):+ """R1 regularization for discriminator. The core idea is to+ penalize the gradient on real data alone: when the+ generator distribution produces the true data distribution+ and the discriminator is equal to 0 on the data manifold, the+ gradient penalty ensures that the discriminator cannot create+ a non-zero gradient orthogonal to the data manifold without+ suffering a loss in the GAN game.++ Ref:+ Eq. 9 in Which training methods for GANs do actually converge.+ """+ grad_real = torch.autograd.grad(outputs=real_pred.sum(), inputs=real_img, create_graph=True)[0]+ grad_penalty = grad_real.pow(2).view(grad_real.shape[0], -1).sum(1).mean()+ return grad_penalty++# Open-Sora-Plan+def gradient_penalty_loss(discriminator, real_data, fake_data, weight=None):+ """Calculate gradient penalty for wgan-gp.++ Args:+ discriminator (nn.Module): Network for the discriminator.+ real_data (Tensor): Real input data.+ fake_data (Tensor): Fake input data.+ weight (Tensor): Weight tensor. Default: None.++ Returns:+ Tensor: A tensor for gradient penalty.+ """++ batch_size = real_data.size(0)+ alpha = real_data.new_tensor(torch.rand(batch_size, 1, 1, 1))++ # interpolate between real_data and fake_data+ interpolates = alpha * real_data + (1. - alpha) * fake_data+ interpolates = torch.autograd.Variable(interpolates, requires_grad=True)++ disc_interpolates = discriminator(interpolates)+ gradients = torch.autograd.grad(+ outputs=disc_interpolates,+ inputs=interpolates,+ grad_outputs=torch.ones_like(disc_interpolates),+ create_graph=True,+ retain_graph=True,+ only_inputs=True)[0]++ if weight is not None:+ gradients = gradients * weight++ gradients_penalty = ((gradients.norm(2, dim=1) - 1)**2).mean()+ if weight is not None:+ gradients_penalty /= torch.mean(weight)++ return gradients_penalty++def gradient_penalty_fn(images, output):# batch_size = images.shape[0]gradients = torch.autograd.grad(@@ -1207,7 +1264,8 @@ class DiscriminatorLoss(nn.Module):gradient_penalty = 0.0if self.gradient_penalty_loss_weight is not None and self.gradient_penalty_loss_weight > 0.0:assert real_video is not None- gradient_penalty = gradient_penalty_fn(real_video, real_logits)+ # gradient_penalty = gradient_penalty_fn(real_video, real_logits)+ gradient_penalty = r1_penalty(real_video, real_logits) # MAGVIT uses r1 penaltygradient_penalty *= self.gradient_penalty_loss_weightdiscriminator_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty