Open-Sora: Democratizing Efficient Video Production for All
Python
gan optimizer
Shen-Chenhuie004d43f6987
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.pyindex 048fd77..23e07a7 100644--- a/configs/vae_magvit_v2/train/17x128x128.py+++ b/configs/vae_magvit_v2/train/17x128x128.py@@ -1,5 +1,4 @@num_frames = 17-video_contains_first_frame = Trueframe_interval = 3image_size = (128, 128)diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex 46ccef8..a5499ea 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -9,7 +9,6 @@ from opensora.models.vae import model_utilsfrom opensora.registry import MODELSfrom opensora.utils.ckpt_utils import load_checkpointfrom einops import rearrange, repeat, pack, unpack-from einops.layers.torch import Rearrangeimport torch.nn.functional as Fimport torchvisionfrom torchvision.models import VGG16_Weights@@ -18,7 +17,6 @@ from taming.modules.losses.lpips import LPIPS # need to pip install https://githfrom torch import nn, einsum, Tensorfrom kornia.filters import filter3dimport math-from torch.autograd import grad as torch_grad"""Encoder and Decoder stuctures with 3D CNNs."""@@ -799,26 +797,31 @@ class VAE_3D_V2(nn.Module):def get_last_layer(self):return self.conv_out.weight+ def parameters(self):+ return [+ *self.conv_in.parameters(),+ *self.conv_in_first_frame.parameters(),+ *self.encoder.paramters(),+ *self.quant_conv.parameters(),+ *self.post_quant_conv.parameters(),+ *self.decoder.parameters(),+ *self.conv_out_first_frame.parametrs(),+ *self.conv_out.parameters()+ ]++ def disc_parameters(self):+ return self.discriminator.parameters()+def forward(self,- input,- optimizer_idx,+ video,+ # optimizer_idx,global_step,sample_posterior=True,video_contains_first_frame = True,split = "train",):- assert input.ndim in {4, 5}, f"received input of {input.ndim} dimensions" # either image or video- assert input.shape[-2:] == self.image_size, f"received input size {input.shape[-2:]}, but config image size is {self.image_size}"-- is_image = input.ndim == 4-- if is_image:- video = rearrange(input, 'b c ... -> b c 1 ...')- video_contains_first_frame = True- else:- video = inputbatch, channels, frames = video.shape[:3]assert divisible_by(frames - int(video_contains_first_frame), self.time_downsample_factor), f'number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}'@@ -877,63 +880,88 @@ class VAE_3D_V2(nn.Module):# GAN- if optimizer_idx == 0: # generator update- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:- try:- d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, self.get_last_layer())- except RuntimeError:- assert not self.training- d_weight = torch.tensor(0.0)- # if video_contains_first_frame:- # Since we don't have enough T frames, pad anyways- fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)- # real_logits = self.discriminator(real_video)- fake_logits = self.discriminator(fake_video.contiguous())- gan_loss = -torch.mean(fake_logits)- else:+ # if optimizer_idx == 0: # generator update+ if self.discriminator_factor is not None and self.discriminator_factor > 0.0:+ try:+ d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, self.get_last_layer())+ except RuntimeError:+ assert not self.trainingd_weight = torch.tensor(0.0)- gan_loss = torch.tensor(0.0)+ # if video_contains_first_frame:+ # Since we don't have enough T frames, pad anyways+ fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)+ fake_logits = self.discriminator(fake_video.contiguous()) # DETACH?+ gan_loss = -torch.mean(fake_logits)+ else:+ d_weight = torch.tensor(0.0)+ gan_loss = torch.tensor(0.0)+ disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)+ weighted_gan_loss = d_weight * disc_factor * gan_loss+ breakpoint()+ total_loss += weighted_gan_loss++ log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),+ "{}/nll_loss".format(split): nll_loss.detach().mean(),+ "{}/recon_loss".format(split): recon_loss.detach().mean(),+ "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),+ "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),+ "{}/weighted_gan_loss".format(split): weighted_gan_loss.detach().mean(),+ "{}gan_adaptive_weight".format(split): d_weight.detach(),+ "{}/disc_factor".format(split): torch.tensor(disc_factor),+ }+ return total_loss, recon_video, log++ ## SCH: move to a different function call+ # if optimizer_idx == 1: # second pass for discriminator update+ # if self.discriminator_factor is not None and self.discriminator_factor > 0.0:+ # # if video_contains_first_frame:+ # # Since we don't have enough T frames, pad anyways+ # real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)+ # fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)+ # real_logits = self.discriminator(real_video.contiguous.detach())+ # fake_logits = self.discriminator(fake_video.contiguous.detach())+ # disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)+ # weight_discriminator_loss = disc_factor * self.calc_disc_loss(real_logits, fake_logits)+ # else:+ # weight_discriminator_loss = 0++ # breakpoint()++ # log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),+ # "{}/logits_real".format(split): real_logits.detach().mean(),+ # "{}/logits_fake".format(split): fake_logits.detach().mean()+ # }++ # return weight_discriminator_loss, recon_video, log++ def disc_forward(+ self,+ video,+ recon_video,+ global_step,+ split = "train",+ ):+ if self.discriminator_factor is not None and self.discriminator_factor > 0.0:+ # if video_contains_first_frame:+ # Since we don't have enough T frames, pad anyways+ real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)+ fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)+ real_logits = self.discriminator(real_video.contiguous.detach())+ fake_logits = self.discriminator(fake_video.contiguous.detach())disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)- weighted_gan_loss = d_weight * disc_factor * gan_loss- breakpoint()- total_loss += weighted_gan_loss-- log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),- "{}/nll_loss".format(split): nll_loss.detach().mean(),- "{}/recon_loss".format(split): recon_loss.detach().mean(),- "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),- "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),- "{}/weighted_gan_loss".format(split): weighted_gan_loss.detach().mean(),- "{}gan_adaptive_weight".format(split): d_weight.detach(),- "{}/disc_factor".format(split): torch.tensor(disc_factor),- }- return total_loss, recon_video, log--- if optimizer_idx == 1: # second pass for discriminator update- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:- # if video_contains_first_frame:- # Since we don't have enough T frames, pad anyways- real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)- fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)- real_logits = self.discriminator(real_video.contiguous.detach())- fake_logits = self.discriminator(fake_video.contiguous.detach())- disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)- weight_discriminator_loss = disc_factor * self.calc_disc_loss(real_logits, fake_logits)- else:- weight_discriminator_loss = 0-- breakpoint()-- log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),- "{}/logits_real".format(split): real_logits.detach().mean(),- "{}/logits_fake".format(split): fake_logits.detach().mean()- }+ weight_discriminator_loss = disc_factor * self.calc_disc_loss(real_logits, fake_logits)+ else:+ weight_discriminator_loss = 0- return weight_discriminator_loss, recon_video, log+ breakpoint()+ log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),+ "{}/logits_real".format(split): real_logits.detach().mean(),+ "{}/logits_fake".format(split): fake_logits.detach().mean()+ }+ return weight_discriminator_loss, log@MODELS.register_module("VAE_MAGVIT_V2")diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pyindex 922886e..853bf0d 100644--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@ -14,6 +14,7 @@ from colossalai.nn.optimizer import HybridAdamfrom colossalai.utils import get_current_devicefrom tqdm import tqdmimport os+from einops import rearrangefrom opensora.acceleration.checkpoint import set_grad_checkpointfrom opensora.acceleration.parallel_states import (@@ -141,11 +142,19 @@ def main():# 4.3. move to devicevae = vae.to(device, dtype)+ breakpoint() # TODO: check vae parameters+# 4.5. setup optimizer+ # vae optimizeroptimizer = HybridAdam(filter(lambda p: p.requires_grad, vae.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True)lr_scheduler = None+ # disc optimizer+ disc_optimizer = HybridAdam(+ filter(lambda p: p.requires_grad, vae.disc_parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True+ )+ disc_lr_scheduler = None# 4.6. prepare for trainingif cfg.grad_checkpoint:@@ -160,6 +169,12 @@ def main():vae, optimizer, _, dataloader, lr_scheduler = booster.boost(model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader)+ # TODO: do I need to pass the other parameters?+ # TODO: can we use the same booster.boost?+ vae.discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(+ model=vae.discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler+ )+torch.set_default_dtype(torch.float)num_steps_per_epoch = len(dataloader)logger.info("Boost vae for distributed training")@@ -169,6 +184,7 @@ def main():# =======================================================start_epoch = start_step = log_step = sampler_start_idx = 0running_loss = 0.0+ running_disc_loss = 0.0# 6.1. resume training@@ -176,8 +192,12 @@ def main():logger.info("Loading checkpoint")booster.load_model(vae, os.path.join(cfg.load, "model"))booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))+ booster.load_optimizer(disc_optimizer, os.path.join(cfg.load, "disc_optimizer"))if lr_scheduler is not None:booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))+ if disc_lr_scheduler is not None:+ booster.load_lr_scheduler(disc_lr_scheduler, os.path.join(cfg.load, "disc_lr_scheduler"))+running_states = load_json(os.path.join(cfg.load, "running_states.json"))dist.barrier()start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]@@ -189,9 +209,6 @@ def main():# # define loss function# loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)- # TODO: Use separate optimizers, 0 for main model and 1 for gan model- optimizer_idx = 0-# 6.2. training loopfor epoch in range(start_epoch, cfg.epochs):@@ -207,34 +224,60 @@ def main():initial=start_step,) as pbar:for step in pbar:++ # SCH: calc global step at the start+ global_step = epoch * num_steps_per_epoch + step+batch = next(dataloader_iter)x = batch["video"].to(device, dtype) # [B, C, T, H, W]+ # supprt for image or video inputs+ assert x.ndim in {4, 5}, f"received input of {x.ndim} dimensions" # either image or video+ assert x.shape[-2:] == cfg.image_size, f"received input size {x.shape[-2:]}, but config image size is {cfg.image_size}"+ is_image = x.ndim == 4+ if is_image:+ video = rearrange(x, 'b c ... -> b c 1 ...')+ video_contains_first_frame = True+ else:+ video = x+# loss = vae.get_loss(x)- loss, reconstructions, log = vae(- x,- optimizer_idx, # TODO: add+ loss, recon_video, log = vae(+ video,global_step,- video_contains_first_frame = cfg.video_contains_first_frame,+ video_contains_first_frame = video_contains_first_frame,split = "train",)- # loss = loss_function(x, reconstructions, posterior)# Backward & updatebooster.backward(loss=loss, optimizer=optimizer)optimizer.step()optimizer.zero_grad()+ disc_loss, disc_log = vae.disc_forward(+ video,+ recon_video,+ global_step,+ split = "train",+ )++ booster.backward(loss=disc_loss, optimizer=disc_optimizer)+ disc_optimizer.step()+ disc_optimizer.zero_grad()+# Log loss values:all_reduce_mean(loss)running_loss += loss.item()- global_step = epoch * num_steps_per_epoch + step+ all_reduce_mean(disc_loss)+ running_disc_loss += disc_loss.item()log_step += 1+# Log to tensorboardif coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:avg_loss = running_loss / log_step- pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step, "opt_idx": optimizer_idx})+ avg_disc_loss = running_disc_loss / log_step+ pbar.set_postfix({"loss": avg_loss, "disc_loss": avg_disc_loss, "step": step, "global_step": global_step})running_loss = 0log_step = 0writer.add_scalar("loss", loss.item(), global_step)@@ -242,7 +285,6 @@ def main():wandb.log({"iter": global_step,- "optimizer_idx": optimizer_idx,"num_samples": global_step * total_batch_size,"epoch": epoch,"loss": loss.item(),@@ -258,8 +300,13 @@ def main():# TODO: save in model?booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)+ booster.save_optimizer(disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096)+if lr_scheduler is not None:booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))+ if disc_lr_scheduler is not None:+ booster.save_lr_scheduler(disc_lr_scheduler, os.path.join(save_dir, "disc_lr_scheduler"))+running_states = {"epoch": epoch,"step": step+1,