Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

debug

Shen-Chenhui686829969f78
diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.py
index e85dcda..98db5f7 100644
--- a/opensora/models/vae/vae_3d_v2.py
+++ b/opensora/models/vae/vae_3d_v2.py
@@ -868,7 +868,6 @@ class VAE_3D_V2(nn.Module):
# recon_loss = F.mse_loss(video, recon_video)
- # breakpoint()
# total_loss = recon_loss
# # KL Loss
@@ -924,7 +923,6 @@ class VAE_3D_V2(nn.Module):
# disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
# weighted_gan_loss = d_weight * disc_factor * gan_loss
- # breakpoint()
# total_loss += weighted_gan_loss
# log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),
@@ -1057,8 +1055,8 @@ class AdversarialLoss(nn.Module):
self.discriminator_loss_weight = discriminator_loss_weight
def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):
- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]
- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]
+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat
+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added create_graph=True
d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
d_weight = d_weight * self.discriminator_loss_weight
@@ -1074,6 +1072,8 @@ class AdversarialLoss(nn.Module):
):
gan_loss = -torch.mean(fake_logits)
if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
+ # DEBUG: NOTE: something about this d_weigth calculation is causing issue
+ # d_weight = 1
try:
d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, last_layer)
except RuntimeError:
diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.py
index adeb112..7f764bd 100644
--- a/scripts/train-vae-v2.py
+++ b/scripts/train-vae-v2.py
@@ -235,8 +235,8 @@ def main():
disc_time_padding = disc_time_downsample_factor - cfg.num_frames % disc_time_downsample_factor
video_contains_first_frame = cfg.video_contains_first_frame
- lecam_ema_real = np.asarray(0)
- lecam_ema_fake = np.asarray(0)
+ lecam_ema_real = torch.tensor(0.0)
+ lecam_ema_fake = torch.tensor(0.0)
for epoch in range(start_epoch, cfg.epochs):
dataloader.sampler.set_epoch(epoch)
@@ -268,7 +268,6 @@ def main():
else:
video = x
-
# ====== VAE ======
# this is essential for the first iteration after OOM
# optimizer._grad_store.reset_all_gradients()
@@ -281,11 +280,16 @@ def main():
# optimizer._bucket_store._grad_in_bucket[rank] = []
# optimizer._bucket_store.offset_list = [0]
# optimizer.zero_grad()
+
optimizer.zero_grad()
recon_video, posterior = vae(
video,
video_contains_first_frame = video_contains_first_frame,
)
+
+
+
+ # ====== Generator Loss ======
# simple nll loss
nll_loss = nll_loss_fn(
video,
@@ -310,21 +314,25 @@ def main():
vae_loss += adversarial_loss
# Backward & update
booster.backward(loss=vae_loss, optimizer=optimizer)
+ # NOTE: clip gradients? this is done in Open-Sora-Plan
+ torch.nn.utils.clip_grad_norm_(vae.parameters(), 1)
optimizer.step()
# Log loss values:
all_reduce_mean(vae_loss)
running_loss += vae_loss.item()
- # ====== Discriminator ======
+ # TODO: debug, move after generator loss later
+ # ====== Discriminator Loss ======
if global_step > cfg.discriminator_start:
disc_optimizer.zero_grad()
# if video_contains_first_frame:
# Since we don't have enough T frames, pad anyways
real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)
+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)
if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:
real_video = real_video.requires_grad_()
- real_logits = discriminator(real_video.contiguous().detach())
+ real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculation
fake_logits = discriminator(fake_video.contiguous().detach())
disc_loss = disc_loss_fn(
real_logits,
@@ -337,12 +345,14 @@ def main():
if cfg.ema_decay is not None:
# SCH: TODO: is this written properly like this for moving average? e.g. distributed training etc.
- lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * np.mean(real_logits.clone().detach())
- lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * np.mean(fake_logits.clone().detach())
+ lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(real_logits.clone().detach())
+ lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(fake_logits.clone().detach())
-
# Backward & update
booster.backward(loss=disc_loss, optimizer=disc_optimizer)
+ # NOTE: TODO: clip gradients? this is done in Open-Sora-Plan
+ torch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1)
+
disc_optimizer.step()
# Log loss values: