Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

gan optimizer

Shen-Chenhuie004d43f6987
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.py
index 048fd77..23e07a7 100644
--- a/configs/vae_magvit_v2/train/17x128x128.py
+++ b/configs/vae_magvit_v2/train/17x128x128.py
@@ -1,5 +1,4 @@
num_frames = 17
-video_contains_first_frame = True
frame_interval = 3
image_size = (128, 128)
diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.py
index 46ccef8..a5499ea 100644
--- a/opensora/models/vae/vae_3d_v2.py
+++ b/opensora/models/vae/vae_3d_v2.py
@@ -9,7 +9,6 @@ from opensora.models.vae import model_utils
from opensora.registry import MODELS
from opensora.utils.ckpt_utils import load_checkpoint
from einops import rearrange, repeat, pack, unpack
-from einops.layers.torch import Rearrange
import torch.nn.functional as F
import torchvision
from torchvision.models import VGG16_Weights
@@ -18,7 +17,6 @@ from taming.modules.losses.lpips import LPIPS # need to pip install https://gith
from torch import nn, einsum, Tensor
from kornia.filters import filter3d
import math
-from torch.autograd import grad as torch_grad
"""Encoder and Decoder stuctures with 3D CNNs."""
@@ -799,26 +797,31 @@ class VAE_3D_V2(nn.Module):
def get_last_layer(self):
return self.conv_out.weight
+ def parameters(self):
+ return [
+ *self.conv_in.parameters(),
+ *self.conv_in_first_frame.parameters(),
+ *self.encoder.paramters(),
+ *self.quant_conv.parameters(),
+ *self.post_quant_conv.parameters(),
+ *self.decoder.parameters(),
+ *self.conv_out_first_frame.parametrs(),
+ *self.conv_out.parameters()
+ ]
+
+ def disc_parameters(self):
+ return self.discriminator.parameters()
+
def forward(
self,
- input,
- optimizer_idx,
+ video,
+ # optimizer_idx,
global_step,
sample_posterior=True,
video_contains_first_frame = True,
split = "train",
):
- assert input.ndim in {4, 5}, f"received input of {input.ndim} dimensions" # either image or video
- assert input.shape[-2:] == self.image_size, f"received input size {input.shape[-2:]}, but config image size is {self.image_size}"
-
- is_image = input.ndim == 4
-
- if is_image:
- video = rearrange(input, 'b c ... -> b c 1 ...')
- video_contains_first_frame = True
- else:
- video = input
batch, channels, frames = video.shape[:3]
assert divisible_by(frames - int(video_contains_first_frame), self.time_downsample_factor), f'number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}'
@@ -877,63 +880,88 @@ class VAE_3D_V2(nn.Module):
# GAN
- if optimizer_idx == 0: # generator update
- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
- try:
- d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, self.get_last_layer())
- except RuntimeError:
- assert not self.training
- d_weight = torch.tensor(0.0)
- # if video_contains_first_frame:
- # Since we don't have enough T frames, pad anyways
- fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)
- # real_logits = self.discriminator(real_video)
- fake_logits = self.discriminator(fake_video.contiguous())
- gan_loss = -torch.mean(fake_logits)
- else:
+ # if optimizer_idx == 0: # generator update
+ if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
+ try:
+ d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, self.get_last_layer())
+ except RuntimeError:
+ assert not self.training
d_weight = torch.tensor(0.0)
- gan_loss = torch.tensor(0.0)
+ # if video_contains_first_frame:
+ # Since we don't have enough T frames, pad anyways
+ fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)
+ fake_logits = self.discriminator(fake_video.contiguous()) # DETACH?
+ gan_loss = -torch.mean(fake_logits)
+ else:
+ d_weight = torch.tensor(0.0)
+ gan_loss = torch.tensor(0.0)
+ disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)
+ weighted_gan_loss = d_weight * disc_factor * gan_loss
+ breakpoint()
+ total_loss += weighted_gan_loss
+
+ log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),
+ "{}/nll_loss".format(split): nll_loss.detach().mean(),
+ "{}/recon_loss".format(split): recon_loss.detach().mean(),
+ "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),
+ "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),
+ "{}/weighted_gan_loss".format(split): weighted_gan_loss.detach().mean(),
+ "{}gan_adaptive_weight".format(split): d_weight.detach(),
+ "{}/disc_factor".format(split): torch.tensor(disc_factor),
+ }
+ return total_loss, recon_video, log
+
+ ## SCH: move to a different function call
+ # if optimizer_idx == 1: # second pass for discriminator update
+ # if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
+ # # if video_contains_first_frame:
+ # # Since we don't have enough T frames, pad anyways
+ # real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)
+ # fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)
+ # real_logits = self.discriminator(real_video.contiguous.detach())
+ # fake_logits = self.discriminator(fake_video.contiguous.detach())
+ # disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)
+ # weight_discriminator_loss = disc_factor * self.calc_disc_loss(real_logits, fake_logits)
+ # else:
+ # weight_discriminator_loss = 0
+
+ # breakpoint()
+
+ # log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),
+ # "{}/logits_real".format(split): real_logits.detach().mean(),
+ # "{}/logits_fake".format(split): fake_logits.detach().mean()
+ # }
+
+ # return weight_discriminator_loss, recon_video, log
+
+ def disc_forward(
+ self,
+ video,
+ recon_video,
+ global_step,
+ split = "train",
+ ):
+ if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
+ # if video_contains_first_frame:
+ # Since we don't have enough T frames, pad anyways
+ real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)
+ fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)
+ real_logits = self.discriminator(real_video.contiguous.detach())
+ fake_logits = self.discriminator(fake_video.contiguous.detach())
disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)
- weighted_gan_loss = d_weight * disc_factor * gan_loss
- breakpoint()
- total_loss += weighted_gan_loss
-
- log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),
- "{}/nll_loss".format(split): nll_loss.detach().mean(),
- "{}/recon_loss".format(split): recon_loss.detach().mean(),
- "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),
- "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),
- "{}/weighted_gan_loss".format(split): weighted_gan_loss.detach().mean(),
- "{}gan_adaptive_weight".format(split): d_weight.detach(),
- "{}/disc_factor".format(split): torch.tensor(disc_factor),
- }
- return total_loss, recon_video, log
-
-
- if optimizer_idx == 1: # second pass for discriminator update
- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
- # if video_contains_first_frame:
- # Since we don't have enough T frames, pad anyways
- real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)
- fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)
- real_logits = self.discriminator(real_video.contiguous.detach())
- fake_logits = self.discriminator(fake_video.contiguous.detach())
- disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)
- weight_discriminator_loss = disc_factor * self.calc_disc_loss(real_logits, fake_logits)
- else:
- weight_discriminator_loss = 0
-
- breakpoint()
-
- log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),
- "{}/logits_real".format(split): real_logits.detach().mean(),
- "{}/logits_fake".format(split): fake_logits.detach().mean()
- }
+ weight_discriminator_loss = disc_factor * self.calc_disc_loss(real_logits, fake_logits)
+ else:
+ weight_discriminator_loss = 0
- return weight_discriminator_loss, recon_video, log
+ breakpoint()
+ log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),
+ "{}/logits_real".format(split): real_logits.detach().mean(),
+ "{}/logits_fake".format(split): fake_logits.detach().mean()
+ }
+ return weight_discriminator_loss, log
@MODELS.register_module("VAE_MAGVIT_V2")
diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.py
index 922886e..853bf0d 100644
--- a/scripts/train-vae-v2.py
+++ b/scripts/train-vae-v2.py
@@ -14,6 +14,7 @@ from colossalai.nn.optimizer import HybridAdam
from colossalai.utils import get_current_device
from tqdm import tqdm
import os
+from einops import rearrange
from opensora.acceleration.checkpoint import set_grad_checkpoint
from opensora.acceleration.parallel_states import (
@@ -141,11 +142,19 @@ def main():
# 4.3. move to device
vae = vae.to(device, dtype)
+ breakpoint() # TODO: check vae parameters
+
# 4.5. setup optimizer
+ # vae optimizer
optimizer = HybridAdam(
filter(lambda p: p.requires_grad, vae.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
)
lr_scheduler = None
+ # disc optimizer
+ disc_optimizer = HybridAdam(
+ filter(lambda p: p.requires_grad, vae.disc_parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
+ )
+ disc_lr_scheduler = None
# 4.6. prepare for training
if cfg.grad_checkpoint:
@@ -160,6 +169,12 @@ def main():
vae, optimizer, _, dataloader, lr_scheduler = booster.boost(
model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader
)
+ # TODO: do I need to pass the other parameters?
+ # TODO: can we use the same booster.boost?
+ vae.discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(
+ model=vae.discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler
+ )
+
torch.set_default_dtype(torch.float)
num_steps_per_epoch = len(dataloader)
logger.info("Boost vae for distributed training")
@@ -169,6 +184,7 @@ def main():
# =======================================================
start_epoch = start_step = log_step = sampler_start_idx = 0
running_loss = 0.0
+ running_disc_loss = 0.0
# 6.1. resume training
@@ -176,8 +192,12 @@ def main():
logger.info("Loading checkpoint")
booster.load_model(vae, os.path.join(cfg.load, "model"))
booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))
+ booster.load_optimizer(disc_optimizer, os.path.join(cfg.load, "disc_optimizer"))
if lr_scheduler is not None:
booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))
+ if disc_lr_scheduler is not None:
+ booster.load_lr_scheduler(disc_lr_scheduler, os.path.join(cfg.load, "disc_lr_scheduler"))
+
running_states = load_json(os.path.join(cfg.load, "running_states.json"))
dist.barrier()
start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]
@@ -189,9 +209,6 @@ def main():
# # define loss function
# loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)
- # TODO: Use separate optimizers, 0 for main model and 1 for gan model
- optimizer_idx = 0
-
# 6.2. training loop
for epoch in range(start_epoch, cfg.epochs):
@@ -207,34 +224,60 @@ def main():
initial=start_step,
) as pbar:
for step in pbar:
+
+ # SCH: calc global step at the start
+ global_step = epoch * num_steps_per_epoch + step
+
batch = next(dataloader_iter)
x = batch["video"].to(device, dtype) # [B, C, T, H, W]
+ # supprt for image or video inputs
+ assert x.ndim in {4, 5}, f"received input of {x.ndim} dimensions" # either image or video
+ assert x.shape[-2:] == cfg.image_size, f"received input size {x.shape[-2:]}, but config image size is {cfg.image_size}"
+ is_image = x.ndim == 4
+ if is_image:
+ video = rearrange(x, 'b c ... -> b c 1 ...')
+ video_contains_first_frame = True
+ else:
+ video = x
+
# loss = vae.get_loss(x)
- loss, reconstructions, log = vae(
- x,
- optimizer_idx, # TODO: add
+ loss, recon_video, log = vae(
+ video,
global_step,
- video_contains_first_frame = cfg.video_contains_first_frame,
+ video_contains_first_frame = video_contains_first_frame,
split = "train",
)
- # loss = loss_function(x, reconstructions, posterior)
# Backward & update
booster.backward(loss=loss, optimizer=optimizer)
optimizer.step()
optimizer.zero_grad()
+ disc_loss, disc_log = vae.disc_forward(
+ video,
+ recon_video,
+ global_step,
+ split = "train",
+ )
+
+ booster.backward(loss=disc_loss, optimizer=disc_optimizer)
+ disc_optimizer.step()
+ disc_optimizer.zero_grad()
+
# Log loss values:
all_reduce_mean(loss)
running_loss += loss.item()
- global_step = epoch * num_steps_per_epoch + step
+ all_reduce_mean(disc_loss)
+ running_disc_loss += disc_loss.item()
log_step += 1
+
# Log to tensorboard
if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:
avg_loss = running_loss / log_step
- pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step, "opt_idx": optimizer_idx})
+ avg_disc_loss = running_disc_loss / log_step
+ pbar.set_postfix({"loss": avg_loss, "disc_loss": avg_disc_loss, "step": step, "global_step": global_step})
running_loss = 0
log_step = 0
writer.add_scalar("loss", loss.item(), global_step)
@@ -242,7 +285,6 @@ def main():
wandb.log(
{
"iter": global_step,
- "optimizer_idx": optimizer_idx,
"num_samples": global_step * total_batch_size,
"epoch": epoch,
"loss": loss.item(),
@@ -258,8 +300,13 @@ def main():
# TODO: save in model?
booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)
booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)
+ booster.save_optimizer(disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096)
+
if lr_scheduler is not None:
booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))
+ if disc_lr_scheduler is not None:
+ booster.save_lr_scheduler(disc_lr_scheduler, os.path.join(save_dir, "disc_lr_scheduler"))
+
running_states = {
"epoch": epoch,
"step": step+1,