Open-Sora: Democratizing Efficient Video Production for All
Python
gan training
Shen-Chenhuide9fffacfd49
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.pyindex f27ac04..9e69791 100644--- a/configs/vae_magvit_v2/train/17x128x128.py+++ b/configs/vae_magvit_v2/train/17x128x128.py@@ -32,16 +32,33 @@ model = dict(activation_fn = 'swish',image_size = image_size,separate_first_frame_encoding = False,- kl_loss_weight = 0.000001,- perceptual_loss_weight = 0.1, # use vgg is not None and more than 0- discriminator_factor = 1.0,+ # kl_loss_weight = 0.000001,+ # perceptual_loss_weight = 0.1, # use vgg is not None and more than 0+ # discriminator_factor = 1.0,+ # discriminator_in_channels = 3,+ # discriminator_filters = 128,+ # discriminator_channel_multipliers = (2,4,4,4,4),+ # discriminator_loss="hinge",+ # discriminator_start = 50001,+)+++discriminator = dict(+ type="DISCRIMINATOR_3D",discriminator_in_channels = 3,discriminator_filters = 128,discriminator_channel_multipliers = (2,4,4,4,4),- discriminator_loss="hinge",discriminator_start = 50001,)++# loss weights+kl_loss_weight = 0.000001,+perceptual_loss_weight = 0.1, # use vgg is not None and more than 0+discriminator_factor = 1.0,+discriminator_loss="hinge",+discriminator_start = 50001,+# Othersseed = 42outputs = "outputs"diff --git a/opensora/models/__init__.py b/opensora/models/__init__.pyindex 6025349..108f3c2 100644--- a/opensora/models/__init__.py+++ b/opensora/models/__init__.py@@ -4,3 +4,6 @@ from .pixart import *from .stdit import *from .text_encoder import *from .vae import *+from .vae_3d import *+from .vae_3d_v2 import *+diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex a6dea12..5a9a542 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -594,19 +594,19 @@ class VAE_3D_V2(nn.Module):num_res_blocks = 2,image_size = (128, 128),separate_first_frame_encoding = False,- kl_loss_weight = 0.000001,- perceptual_loss_weight = 0.1,- vgg = None,- vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,+ # kl_loss_weight = 0.000001,+ # perceptual_loss_weight = 0.1,+ # vgg = None,+ # vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,channel_multipliers = (1, 2, 2, 4),temporal_downsample = (True, True, False),num_frames = 17,- discriminator_factor = 1.0,- discriminator_in_channels = 3,- discriminator_filters = 128,+ # discriminator_factor = 1.0,+ # discriminator_in_channels = 3,+ # discriminator_filters = 128,discriminator_channel_multipliers = (2,4,4,4,4),- discriminator_loss="hinge",- discriminator_start=50001,+ # discriminator_loss="hinge",+ # discriminator_start=50001,num_groups = 32, # for nn.GroupNorm# conv_downsample = False,# upsample = "nearest+conv", # options: "deconv", "nearest+conv"@@ -630,11 +630,11 @@ class VAE_3D_V2(nn.Module):# ==== Model Params ====- self.image_size = cast_tuple(image_size, 2)+ # self.image_size = cast_tuple(image_size, 2)self.time_downsample_factor = 2**sum(temporal_downsample)self.time_padding = self.time_downsample_factor - 1- self.discr_time_downsample_factor = 2**len(discriminator_channel_multipliers)- self.discr_time_padding = self.discr_time_downsample_factor - num_frames % self.discr_time_downsample_factor+ # self.discr_time_downsample_factor = 2**len(discriminator_channel_multipliers)+ # self.discr_time_padding = self.discr_time_downsample_factor - num_frames % self.discr_time_downsample_factorself.separate_first_frame_encoding = separate_first_frame_encodingimage_down = 2 ** len(temporal_downsample)@@ -685,43 +685,43 @@ class VAE_3D_V2(nn.Module):self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1, device=device, dtype=dtype)self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)- # KL Loss- self.kl_loss_weight = kl_loss_weight-- # Perceptual Loss- self.vgg = None- self.perceptual_loss_weight = perceptual_loss_weight- if perceptual_loss_weight is not None and perceptual_loss_weight > 0:- # self.lpips = LPIPS().eval()- if not exists(vgg):- vgg = torchvision.models.vgg16(- weights = vgg_weights- )- vgg.classifier = Sequential(*vgg.classifier[:-2])- self.vgg = vgg+ # # KL Loss+ # self.kl_loss_weight = kl_loss_weight++ # # Perceptual Loss+ # self.vgg = None+ # self.perceptual_loss_weight = perceptual_loss_weight+ # if perceptual_loss_weight is not None and perceptual_loss_weight > 0:+ # # self.lpips = LPIPS().eval()+ # if not exists(vgg):+ # vgg = torchvision.models.vgg16(+ # weights = vgg_weights+ # )+ # vgg.classifier = Sequential(*vgg.classifier[:-2])+ # self.vgg = vgg- # Adversarial Loss- self.discriminator_factor = discriminator_factor- self.discriminator_start = discriminator_start- self.discriminator = None- if discriminator_factor is not None and discriminator_factor > 0:- self.discriminator = StyleGANDiscriminator(- image_size = image_size,- num_frames = num_frames,- discriminator_in_channels = discriminator_in_channels,- discriminator_filters = discriminator_filters,- discriminator_channel_multipliers = discriminator_channel_multipliers,- num_groups = num_groups,- dtype = dtype,- device = device,- )-- if discriminator_loss == "hinge":- self.disc_loss_fn = hinge_d_loss- elif discriminator_loss == "vanilla":- self.disc_loss_fn = vanilla_d_loss- else:- raise ValueError(f"Unknown GAN loss '{discriminator_loss}'.")+ # # Adversarial Loss+ # self.discriminator_factor = discriminator_factor+ # self.discriminator_start = discriminator_start+ # self.discriminator = None+ # if discriminator_factor is not None and discriminator_factor > 0:+ # self.discriminator = StyleGANDiscriminator(+ # image_size = image_size,+ # num_frames = num_frames,+ # discriminator_in_channels = discriminator_in_channels,+ # discriminator_filters = discriminator_filters,+ # discriminator_channel_multipliers = discriminator_channel_multipliers,+ # num_groups = num_groups,+ # dtype = dtype,+ # device = device,+ # )++ # if discriminator_loss == "hinge":+ # self.disc_loss_fn = hinge_d_loss+ # elif discriminator_loss == "vanilla":+ # self.disc_loss_fn = vanilla_d_loss+ # else:+ # raise ValueError(f"Unknown GAN loss '{discriminator_loss}'.")def get_latent_size(self, input_size):@@ -786,44 +786,31 @@ class VAE_3D_V2(nn.Module):def get_last_layer(self):return self.conv_out.weight- def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer=None):- breakpoint() # TODO: scrutinize- if last_layer is not None:- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]- else:- nll_grads = torch.autograd.grad(nll_loss, self.get_last_layer(), retain_graph=True)[0]- g_grads = torch.autograd.grad(g_loss, self.get_last_layer(), retain_graph=True)[0]-- d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)- d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()- d_weight = d_weight * self.discriminator_weight-- return d_weight- def parameters(self):- return [- *self.conv_in.parameters(),- *self.conv_in_first_frame.parameters(),- *self.encoder.parameters(),- *self.quant_conv.parameters(),- *self.post_quant_conv.parameters(),- *self.decoder.parameters(),- *self.conv_out_first_frame.parameters(),- *self.conv_out.parameters()- ]-- def disc_parameters(self):- return self.discriminator.parameters()+ # def parameters(self):+ # return [+ # *self.conv_in.parameters(),+ # *self.conv_in_first_frame.parameters(),+ # *self.encoder.parameters(),+ # *self.quant_conv.parameters(),+ # *self.post_quant_conv.parameters(),+ # *self.decoder.parameters(),+ # *self.conv_out_first_frame.parameters(),+ # *self.conv_out.parameters()+ # ]++ # def disc_parameters(self):+ # return self.discriminator.parameters()def forward(self,video,# optimizer_idx,- global_step,+ # global_step,+ # discriminator, # TODOsample_posterior=True,video_contains_first_frame = True,- split = "train",+ # split = "train",):@@ -846,10 +833,124 @@ class VAE_3D_V2(nn.Module):video_contains_first_frame = video_contains_first_frame)+ return recon_video, posterior++ # recon_loss = F.mse_loss(video, recon_video)++ # breakpoint()+ # total_loss = recon_loss++ # # KL Loss+ # weighted_kl_loss = 0+ # if self.kl_loss_weight is not None and self.kl_loss_weight > 0:+ # kl_loss = posterior.kl()+ # # NOTE: since we use MSE, here use mean as well, else use sum+ # kl_loss = torch.mean(kl_loss) / kl_loss.shape[0]+ # weighted_kl_loss = kl_loss * self.kl_loss_weight+ # total_loss += weighted_kl_loss++ # # Perceptual Loss+ # # SCH: NOTE: if mse can pick single frame, if use sum of errors, need to calc for all frames!+ # weighted_perceptual_loss = 0+ # if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0:+ # frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices+ # input_vgg_input = pick_video_frame(video, frame_indices)+ # recon_vgg_input = pick_video_frame(recon_video, frame_indices)+ # if channels == 1:+ # input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)+ # recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)+ # elif channels == 4: # SCH: take the first 3 for perceptual loss calc+ # input_vgg_input = input_vgg_input[:, :3]+ # recon_vgg_input = recon_vgg_input[:, :3]+ # input_vgg_feats = self.vgg(input_vgg_input)+ # recon_vgg_feats = self.vgg(recon_vgg_input)+ # perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)+ # # perceptual_loss = self.lpips(input_vgg_input.contiguous(), recon_vgg_input.contiguous())+ # weighted_perceptual_loss = perceptual_loss * self.perceptual_loss_weight+ # total_loss += weighted_perceptual_loss++ # nll_loss = recon_loss + weighted_kl_loss + weighted_perceptual_loss+++ # # GAN+ # # if optimizer_idx == 0: # generator update+ # if self.discriminator_factor is not None and self.discriminator_factor > 0.0:+ # try:+ # d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, self.get_last_layer())+ # except RuntimeError:+ # assert not self.training+ # d_weight = torch.tensor(0.0)+ # # if video_contains_first_frame:+ # # Since we don't have enough T frames, pad anyways++ # fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)+ # fake_logits = discriminator(fake_video.contiguous()) # SCH:DETACH?++ # gan_loss = -torch.mean(fake_logits)+ # else:+ # d_weight = torch.tensor(0.0)+ # gan_loss = torch.tensor(0.0)++ # disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)+ # weighted_gan_loss = d_weight * disc_factor * gan_loss+ # breakpoint()+ # total_loss += weighted_gan_loss++ # log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),+ # "{}/nll_loss".format(split): nll_loss.detach().mean(),+ # "{}/recon_loss".format(split): recon_loss.detach().mean(),+ # "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),+ # "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),+ # "{}/weighted_gan_loss".format(split): weighted_gan_loss.detach().mean(),+ # "{}gan_adaptive_weight".format(split): d_weight.detach(),+ # "{}/disc_factor".format(split): torch.tensor(disc_factor),+ # }+ # return total_loss, recon_video, log+++class VEALoss(nn.Module):+ def __init__(+ self,+ perceptual_loss_weight=0.1,+ kl_loss_weight = 0.000001,+ vgg=None,+ vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,+ ):+ super().__init__()++ # KL Loss+ self.kl_loss_weight = kl_loss_weight+ # Perceptual Loss+ self.perceptual_loss_weight = perceptual_loss_weight+ self.vgg = None+ self.perceptual_loss_weight = perceptual_loss_weight+ if perceptual_loss_weight is not None and perceptual_loss_weight > 0:+ if not exists(vgg):+ vgg = torchvision.models.vgg16(+ weights = vgg_weights+ )+ vgg.classifier = Sequential(*vgg.classifier[:-2])+ self.vgg = vgg++ def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):+ breakpoint() # TODO: scrutinize+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]+ d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)+ d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()+ d_weight = d_weight * self.discriminator_weight+ return d_weight++ def forward(+ self,+ video,+ recon_video,+ posterior,+ split = "train",+ ):+ # reconstruction lossrecon_loss = F.mse_loss(video, recon_video)- # TODO: check if ths is tensor- breakpoint()- total_loss = recon_loss+ nll_loss = recon_loss# KL Lossweighted_kl_loss = 0@@ -860,10 +961,11 @@ class VAE_3D_V2(nn.Module):weighted_kl_loss = kl_loss * self.kl_loss_weighttotal_loss += weighted_kl_loss- # Perceptual Loss- # SCH: NOTE: if mse can pick single frame, if use sum of errors, need to calc for all frames!+ # perceptual lossweighted_perceptual_loss = 0if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0:+ assert exists(self.vgg)+ batch, channels, frames = video.shape[:3]frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indicesinput_vgg_input = pick_video_frame(video, frame_indices)recon_vgg_input = pick_video_frame(recon_video, frame_indices)@@ -876,25 +978,63 @@ class VAE_3D_V2(nn.Module):input_vgg_feats = self.vgg(input_vgg_input)recon_vgg_feats = self.vgg(recon_vgg_input)perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)- # perceptual_loss = self.lpips(input_vgg_input.contiguous(), recon_vgg_input.contiguous())weighted_perceptual_loss = perceptual_loss * self.perceptual_loss_weight- total_loss += weighted_perceptual_loss-- nll_loss = recon_loss + weighted_kl_loss + weighted_perceptual_loss+ nll_loss += weighted_perceptual_loss++ # # GAN Loss+ # if self.discriminator_factor is not None and self.discriminator_factor > 0.0:+ # try:+ # d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, last_layer)+ # except RuntimeError:+ # assert not self.training+ # d_weight = torch.tensor(0.0)+ # # if video_contains_first_frame:+ # # Since we don't have enough T frames, pad anyways+ # fake_video = pad_at_dim(recon_video, (discriminator_time_padding, 0), value = 0., dim = 2)+ # fake_logits = discriminator(fake_video.contiguous())+ # gan_loss = -torch.mean(fake_logits)+ # else:+ # d_weight = torch.tensor(0.0)+ # gan_loss = torch.tensor(0.0)++ # disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)+ # weighted_gan_loss = d_weight * disc_factor * gan_loss+ # breakpoint()+ # total_loss = nll_loss + weighted_gan_loss++ log = {+ "{}/total_loss".format(split): nll_loss.clone().detach().mean(),+ "{}/recon_loss".format(split): recon_loss.detach().mean(),+ "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),+ "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),+ }+ return nll_loss, log++class AdversarialLoss(nn.Module):+ def __init__(+ self,+ discriminator_factor = 1.0,+ discriminator_start = 50001,+ ):+ super().__init__()+ self.discriminator_factor = discriminator_factor+ self.discriminator_start = discriminator_start- # GAN- # if optimizer_idx == 0: # generator update+ def forward(+ self,+ fake_logits,+ nll_loss,+ last_layer,+ global_step,+ is_training = True,+ ):if self.discriminator_factor is not None and self.discriminator_factor > 0.0:try:- d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, self.get_last_layer())+ d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, last_layer)except RuntimeError:- assert not self.training+ assert not is_trainingd_weight = torch.tensor(0.0)- # if video_contains_first_frame:- # Since we don't have enough T frames, pad anyways- fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)- fake_logits = self.discriminator(fake_video.contiguous()) # DETACH?gan_loss = -torch.mean(fake_logits)else:d_weight = torch.tensor(0.0)@@ -902,75 +1042,55 @@ class VAE_3D_V2(nn.Module):disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)weighted_gan_loss = d_weight * disc_factor * gan_loss- breakpoint()- total_loss += weighted_gan_loss- log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),- "{}/nll_loss".format(split): nll_loss.detach().mean(),- "{}/recon_loss".format(split): recon_loss.detach().mean(),- "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),- "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),- "{}/weighted_gan_loss".format(split): weighted_gan_loss.detach().mean(),- "{}gan_adaptive_weight".format(split): d_weight.detach(),- "{}/disc_factor".format(split): torch.tensor(disc_factor),- }- return total_loss, recon_video, log-- ## SCH: move to a different function call- # if optimizer_idx == 1: # second pass for discriminator update- # if self.discriminator_factor is not None and self.discriminator_factor > 0.0:- # # if video_contains_first_frame:- # # Since we don't have enough T frames, pad anyways- # real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)- # fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)- # real_logits = self.discriminator(real_video.contiguous.detach())- # fake_logits = self.discriminator(fake_video.contiguous.detach())- # disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)- # weight_discriminator_loss = disc_factor * self.disc_loss_fn(real_logits, fake_logits)- # else:- # weight_discriminator_loss = 0-- # breakpoint()-- # log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),- # "{}/logits_real".format(split): real_logits.detach().mean(),- # "{}/logits_fake".format(split): fake_logits.detach().mean()- # }-- # return weight_discriminator_loss, recon_video, log-- def disc_forward(+ return weighted_gan_loss++class DiscriminatorLoss(nn.Module):+ def __init__(self,- video,- recon_video,+ discriminator_factor = 1.0,+ discriminator_start = 50001,+ discriminator_loss="hinge",+ ):+ super().__init__()++ assert discriminator_loss in ["hinge", "vanilla"]+ self.discriminator_factor = discriminator_factor+ self.discriminator_start = discriminator_start++ if discriminator_loss == "hinge":+ self.disc_loss_fn = hinge_d_loss+ elif discriminator_loss == "vanilla":+ self.disc_loss_fn = vanilla_d_loss+ else:+ raise ValueError(f"Unknown GAN loss '{discriminator_loss}'.")++ def forward(+ self,+ real_logits,+ fake_logits,global_step,- split = "train",):if self.discriminator_factor is not None and self.discriminator_factor > 0.0:- # if video_contains_first_frame:- # Since we don't have enough T frames, pad anyways- real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)- fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)- real_logits = self.discriminator(real_video.contiguous.detach())- fake_logits = self.discriminator(fake_video.contiguous.detach())disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)weight_discriminator_loss = disc_factor * self.disc_loss_fn(real_logits, fake_logits)else:weight_discriminator_loss = 0- breakpoint()-- log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),- "{}/logits_real".format(split): real_logits.detach().mean(),- "{}/logits_fake".format(split): fake_logits.detach().mean()- }-- return weight_discriminator_loss, log+ breakpoint()+ return weight_discriminator_loss@MODELS.register_module("VAE_MAGVIT_V2")def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):model = VAE_3D_V2(**kwargs)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return model+[email protected]_module("DISCRIMINATOR_3D")+def DISCRIMINATOR_3D(from_pretrained=None, **kwargs):+ model = StyleGANDiscriminator(**kwargs)if from_pretrained is not None:load_checkpoint(model, from_pretrained)return model\ No newline at end of filediff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pyindex 853bf0d..0f2a354 100644--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@ -34,7 +34,7 @@ from opensora.utils.config_utils import ()from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, requires_grad, to_torch_dtypefrom opensora.utils.train_utils import update_ema, MaskGenerator-from opensora.models.vae.model_utils import VEA3DLoss+from opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, pad_at_dimdef main():@@ -137,10 +137,16 @@ def main():logger.info(f"Trainable vae params: {format_numel_str(vae_numel_trainable)}, Total model params: {format_numel_str(vae_numel)}")-++ discriminator = build_module(cfg.discriminator, MODELS, device=device)+ discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)+ logger.info(+ f"Trainable discriminator params: {format_numel_str(discriminator_numel_trainable)}, Total model params: {format_numel_str(discriminator_numel)}"+ )# 4.3. move to devicevae = vae.to(device, dtype)+ discriminator = discriminator.to(device, dtype)breakpoint() # TODO: check vae parameters@@ -152,14 +158,16 @@ def main():lr_scheduler = None# disc optimizerdisc_optimizer = HybridAdam(- filter(lambda p: p.requires_grad, vae.disc_parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True+ filter(lambda p: p.requires_grad, discriminator.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True)disc_lr_scheduler = None# 4.6. prepare for trainingif cfg.grad_checkpoint:set_grad_checkpoint(vae)+ set_grad_checkpoint(discriminator)vae.train()+ discriminator.train()# =======================================================@@ -169,16 +177,17 @@ def main():vae, optimizer, _, dataloader, lr_scheduler = booster.boost(model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader)- # TODO: do I need to pass the other parameters?- # TODO: can we use the same booster.boost?- vae.discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(- model=vae.discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler- )-torch.set_default_dtype(torch.float)num_steps_per_epoch = len(dataloader)logger.info("Boost vae for distributed training")++ discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(+ model=discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler+ )+ logger.info("Boost discriminator for distributed training")++# =======================================================# 6. training loop# =======================================================@@ -191,6 +200,7 @@ def main():if cfg.load is not None:logger.info("Loading checkpoint")booster.load_model(vae, os.path.join(cfg.load, "model"))+ booster.load_model(discriminator, os.path.join(cfg.load, "discriminator"))booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))booster.load_optimizer(disc_optimizer, os.path.join(cfg.load, "disc_optimizer"))if lr_scheduler is not None:@@ -206,11 +216,29 @@ def main():dataloader.sampler.set_start_index(sampler_start_idx)- # # define loss function- # loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)+ # 6.2 Define loss functions+ nll_loss_fn = VEALoss(+ perceptual_loss_weight = cfg.perceptual_loss_weight,+ kl_loss_weight = cfg.kl_loss_weight,+ )++ adversarial_loss_fn = AdversarialLoss(+ discriminator_factor = cfg.discriminator_factor,+ discriminator_start = cfg.discriminator_start,+ )++ disc_loss_fn = DiscriminatorLoss(+ discriminator_factor = cfg.discriminator_factor,+ discriminator_start = cfg.discriminator_start,+ discriminator_loss = cfg.discriminator_loss,+ )++ # 6.3. training loop+ # calculate discriminator_time_padding+ disc_time_downsample_factor = 2 ** cfg.discriminator.discriminator_channel_multipliers+ disc_time_padding = disc_time_downsample_factor - cfg.num_frames % disc_time_downsample_factor- # 6.2. training loopfor epoch in range(start_epoch, cfg.epochs):dataloader.sampler.set_epoch(epoch)dataloader_iter = iter(dataloader)@@ -241,38 +269,59 @@ def main():else:video = x- # loss = vae.get_loss(x)- loss, recon_video, log = vae(+ # padded videos for GAN+ if global_step > cfg.discriminator_start:+ real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)++ # ====== VAE ======+ optimizer.zero_grad()+ recon_video, posterior = vae(video,- global_step,video_contains_first_frame = video_contains_first_frame,- split = "train",)-- # Backward & update- booster.backward(loss=loss, optimizer=optimizer)- optimizer.step()- optimizer.zero_grad()-- disc_loss, disc_log = vae.disc_forward(+ # simple nll loss+ nll_loss, nll_loss_log = nll_loss_fn(video,recon_video,- global_step,- split = "train",+ posterior,+ split = "train")-+ vae_loss = nll_loss+ # adversarial loss+ if global_step > cfg.discriminator_start:+ fake_logits = discriminator(fake_video.contiguous)+ adversarial_loss = adversarial_loss_fn(+ fake_logits,+ nll_loss,+ vae.get_last_layer(),+ global_step,+ is_training = vae.training,+ )+ vae_loss += adversarial_loss+ # Backward & update+ booster.backward(loss=vae_loss, optimizer=optimizer)+ optimizer.step()+ # Log loss values:+ all_reduce_mean(vae_loss)+ running_loss += vae_loss.item()++ # ====== Discriminator ======+ disc_optimizer.zero_grad()+ # if video_contains_first_frame:+ # Since we don't have enough T frames, pad anyways+ real_logits = discriminator(real_video.contiguous.detach())+ fake_logits = discriminator(fake_video.contiguous.detach())+ disc_loss = disc_loss_fn(real_logits, fake_logits, global_step)+ # Backward & updatebooster.backward(loss=disc_loss, optimizer=disc_optimizer)disc_optimizer.step()- disc_optimizer.zero_grad()-# Log loss values:- all_reduce_mean(loss)- running_loss += loss.item()all_reduce_mean(disc_loss)running_disc_loss += disc_loss.item()+log_step += 1-# Log to tensorboardif coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:avg_loss = running_loss / log_step@@ -280,14 +329,15 @@ def main():pbar.set_postfix({"loss": avg_loss, "disc_loss": avg_disc_loss, "step": step, "global_step": global_step})running_loss = 0log_step = 0- writer.add_scalar("loss", loss.item(), global_step)+ writer.add_scalar("loss", vae_loss.item(), global_step)if cfg.wandb:wandb.log({"iter": global_step,"num_samples": global_step * total_batch_size,"epoch": epoch,- "loss": loss.item(),+ "loss": vae_loss.item(),+ "disc_loss": disc_loss.item(),"avg_loss": avg_loss,},step=global_step,@@ -297,8 +347,8 @@ def main():if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)- # TODO: save in model?booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)+ booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)booster.save_optimizer(disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096)