Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

gan training

Shen-Chenhuide9fffacfd49
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.py
index f27ac04..9e69791 100644
--- a/configs/vae_magvit_v2/train/17x128x128.py
+++ b/configs/vae_magvit_v2/train/17x128x128.py
@@ -32,16 +32,33 @@ model = dict(
activation_fn = 'swish',
image_size = image_size,
separate_first_frame_encoding = False,
- kl_loss_weight = 0.000001,
- perceptual_loss_weight = 0.1, # use vgg is not None and more than 0
- discriminator_factor = 1.0,
+ # kl_loss_weight = 0.000001,
+ # perceptual_loss_weight = 0.1, # use vgg is not None and more than 0
+ # discriminator_factor = 1.0,
+ # discriminator_in_channels = 3,
+ # discriminator_filters = 128,
+ # discriminator_channel_multipliers = (2,4,4,4,4),
+ # discriminator_loss="hinge",
+ # discriminator_start = 50001,
+)
+
+
+discriminator = dict(
+ type="DISCRIMINATOR_3D",
discriminator_in_channels = 3,
discriminator_filters = 128,
discriminator_channel_multipliers = (2,4,4,4,4),
- discriminator_loss="hinge",
discriminator_start = 50001,
)
+
+# loss weights
+kl_loss_weight = 0.000001,
+perceptual_loss_weight = 0.1, # use vgg is not None and more than 0
+discriminator_factor = 1.0,
+discriminator_loss="hinge",
+discriminator_start = 50001,
+
# Others
seed = 42
outputs = "outputs"
diff --git a/opensora/models/__init__.py b/opensora/models/__init__.py
index 6025349..108f3c2 100644
--- a/opensora/models/__init__.py
+++ b/opensora/models/__init__.py
@@ -4,3 +4,6 @@ from .pixart import *
from .stdit import *
from .text_encoder import *
from .vae import *
+from .vae_3d import *
+from .vae_3d_v2 import *
+
diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.py
index a6dea12..5a9a542 100644
--- a/opensora/models/vae/vae_3d_v2.py
+++ b/opensora/models/vae/vae_3d_v2.py
@@ -594,19 +594,19 @@ class VAE_3D_V2(nn.Module):
num_res_blocks = 2,
image_size = (128, 128),
separate_first_frame_encoding = False,
- kl_loss_weight = 0.000001,
- perceptual_loss_weight = 0.1,
- vgg = None,
- vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
+ # kl_loss_weight = 0.000001,
+ # perceptual_loss_weight = 0.1,
+ # vgg = None,
+ # vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
channel_multipliers = (1, 2, 2, 4),
temporal_downsample = (True, True, False),
num_frames = 17,
- discriminator_factor = 1.0,
- discriminator_in_channels = 3,
- discriminator_filters = 128,
+ # discriminator_factor = 1.0,
+ # discriminator_in_channels = 3,
+ # discriminator_filters = 128,
discriminator_channel_multipliers = (2,4,4,4,4),
- discriminator_loss="hinge",
- discriminator_start=50001,
+ # discriminator_loss="hinge",
+ # discriminator_start=50001,
num_groups = 32, # for nn.GroupNorm
# conv_downsample = False,
# upsample = "nearest+conv", # options: "deconv", "nearest+conv"
@@ -630,11 +630,11 @@ class VAE_3D_V2(nn.Module):
# ==== Model Params ====
- self.image_size = cast_tuple(image_size, 2)
+ # self.image_size = cast_tuple(image_size, 2)
self.time_downsample_factor = 2**sum(temporal_downsample)
self.time_padding = self.time_downsample_factor - 1
- self.discr_time_downsample_factor = 2**len(discriminator_channel_multipliers)
- self.discr_time_padding = self.discr_time_downsample_factor - num_frames % self.discr_time_downsample_factor
+ # self.discr_time_downsample_factor = 2**len(discriminator_channel_multipliers)
+ # self.discr_time_padding = self.discr_time_downsample_factor - num_frames % self.discr_time_downsample_factor
self.separate_first_frame_encoding = separate_first_frame_encoding
image_down = 2 ** len(temporal_downsample)
@@ -685,43 +685,43 @@ class VAE_3D_V2(nn.Module):
self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1, device=device, dtype=dtype)
self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)
- # KL Loss
- self.kl_loss_weight = kl_loss_weight
-
- # Perceptual Loss
- self.vgg = None
- self.perceptual_loss_weight = perceptual_loss_weight
- if perceptual_loss_weight is not None and perceptual_loss_weight > 0:
- # self.lpips = LPIPS().eval()
- if not exists(vgg):
- vgg = torchvision.models.vgg16(
- weights = vgg_weights
- )
- vgg.classifier = Sequential(*vgg.classifier[:-2])
- self.vgg = vgg
+ # # KL Loss
+ # self.kl_loss_weight = kl_loss_weight
+
+ # # Perceptual Loss
+ # self.vgg = None
+ # self.perceptual_loss_weight = perceptual_loss_weight
+ # if perceptual_loss_weight is not None and perceptual_loss_weight > 0:
+ # # self.lpips = LPIPS().eval()
+ # if not exists(vgg):
+ # vgg = torchvision.models.vgg16(
+ # weights = vgg_weights
+ # )
+ # vgg.classifier = Sequential(*vgg.classifier[:-2])
+ # self.vgg = vgg
- # Adversarial Loss
- self.discriminator_factor = discriminator_factor
- self.discriminator_start = discriminator_start
- self.discriminator = None
- if discriminator_factor is not None and discriminator_factor > 0:
- self.discriminator = StyleGANDiscriminator(
- image_size = image_size,
- num_frames = num_frames,
- discriminator_in_channels = discriminator_in_channels,
- discriminator_filters = discriminator_filters,
- discriminator_channel_multipliers = discriminator_channel_multipliers,
- num_groups = num_groups,
- dtype = dtype,
- device = device,
- )
-
- if discriminator_loss == "hinge":
- self.disc_loss_fn = hinge_d_loss
- elif discriminator_loss == "vanilla":
- self.disc_loss_fn = vanilla_d_loss
- else:
- raise ValueError(f"Unknown GAN loss '{discriminator_loss}'.")
+ # # Adversarial Loss
+ # self.discriminator_factor = discriminator_factor
+ # self.discriminator_start = discriminator_start
+ # self.discriminator = None
+ # if discriminator_factor is not None and discriminator_factor > 0:
+ # self.discriminator = StyleGANDiscriminator(
+ # image_size = image_size,
+ # num_frames = num_frames,
+ # discriminator_in_channels = discriminator_in_channels,
+ # discriminator_filters = discriminator_filters,
+ # discriminator_channel_multipliers = discriminator_channel_multipliers,
+ # num_groups = num_groups,
+ # dtype = dtype,
+ # device = device,
+ # )
+
+ # if discriminator_loss == "hinge":
+ # self.disc_loss_fn = hinge_d_loss
+ # elif discriminator_loss == "vanilla":
+ # self.disc_loss_fn = vanilla_d_loss
+ # else:
+ # raise ValueError(f"Unknown GAN loss '{discriminator_loss}'.")
def get_latent_size(self, input_size):
@@ -786,44 +786,31 @@ class VAE_3D_V2(nn.Module):
def get_last_layer(self):
return self.conv_out.weight
- def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer=None):
- breakpoint() # TODO: scrutinize
- if last_layer is not None:
- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]
- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]
- else:
- nll_grads = torch.autograd.grad(nll_loss, self.get_last_layer(), retain_graph=True)[0]
- g_grads = torch.autograd.grad(g_loss, self.get_last_layer(), retain_graph=True)[0]
-
- d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
- d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
- d_weight = d_weight * self.discriminator_weight
-
- return d_weight
- def parameters(self):
- return [
- *self.conv_in.parameters(),
- *self.conv_in_first_frame.parameters(),
- *self.encoder.parameters(),
- *self.quant_conv.parameters(),
- *self.post_quant_conv.parameters(),
- *self.decoder.parameters(),
- *self.conv_out_first_frame.parameters(),
- *self.conv_out.parameters()
- ]
-
- def disc_parameters(self):
- return self.discriminator.parameters()
+ # def parameters(self):
+ # return [
+ # *self.conv_in.parameters(),
+ # *self.conv_in_first_frame.parameters(),
+ # *self.encoder.parameters(),
+ # *self.quant_conv.parameters(),
+ # *self.post_quant_conv.parameters(),
+ # *self.decoder.parameters(),
+ # *self.conv_out_first_frame.parameters(),
+ # *self.conv_out.parameters()
+ # ]
+
+ # def disc_parameters(self):
+ # return self.discriminator.parameters()
def forward(
self,
video,
# optimizer_idx,
- global_step,
+ # global_step,
+ # discriminator, # TODO
sample_posterior=True,
video_contains_first_frame = True,
- split = "train",
+ # split = "train",
):
@@ -846,10 +833,124 @@ class VAE_3D_V2(nn.Module):
video_contains_first_frame = video_contains_first_frame
)
+ return recon_video, posterior
+
+ # recon_loss = F.mse_loss(video, recon_video)
+
+ # breakpoint()
+ # total_loss = recon_loss
+
+ # # KL Loss
+ # weighted_kl_loss = 0
+ # if self.kl_loss_weight is not None and self.kl_loss_weight > 0:
+ # kl_loss = posterior.kl()
+ # # NOTE: since we use MSE, here use mean as well, else use sum
+ # kl_loss = torch.mean(kl_loss) / kl_loss.shape[0]
+ # weighted_kl_loss = kl_loss * self.kl_loss_weight
+ # total_loss += weighted_kl_loss
+
+ # # Perceptual Loss
+ # # SCH: NOTE: if mse can pick single frame, if use sum of errors, need to calc for all frames!
+ # weighted_perceptual_loss = 0
+ # if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0:
+ # frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
+ # input_vgg_input = pick_video_frame(video, frame_indices)
+ # recon_vgg_input = pick_video_frame(recon_video, frame_indices)
+ # if channels == 1:
+ # input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)
+ # recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)
+ # elif channels == 4: # SCH: take the first 3 for perceptual loss calc
+ # input_vgg_input = input_vgg_input[:, :3]
+ # recon_vgg_input = recon_vgg_input[:, :3]
+ # input_vgg_feats = self.vgg(input_vgg_input)
+ # recon_vgg_feats = self.vgg(recon_vgg_input)
+ # perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)
+ # # perceptual_loss = self.lpips(input_vgg_input.contiguous(), recon_vgg_input.contiguous())
+ # weighted_perceptual_loss = perceptual_loss * self.perceptual_loss_weight
+ # total_loss += weighted_perceptual_loss
+
+ # nll_loss = recon_loss + weighted_kl_loss + weighted_perceptual_loss
+
+
+ # # GAN
+ # # if optimizer_idx == 0: # generator update
+ # if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
+ # try:
+ # d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, self.get_last_layer())
+ # except RuntimeError:
+ # assert not self.training
+ # d_weight = torch.tensor(0.0)
+ # # if video_contains_first_frame:
+ # # Since we don't have enough T frames, pad anyways
+
+ # fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)
+ # fake_logits = discriminator(fake_video.contiguous()) # SCH:DETACH?
+
+ # gan_loss = -torch.mean(fake_logits)
+ # else:
+ # d_weight = torch.tensor(0.0)
+ # gan_loss = torch.tensor(0.0)
+
+ # disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
+ # weighted_gan_loss = d_weight * disc_factor * gan_loss
+ # breakpoint()
+ # total_loss += weighted_gan_loss
+
+ # log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),
+ # "{}/nll_loss".format(split): nll_loss.detach().mean(),
+ # "{}/recon_loss".format(split): recon_loss.detach().mean(),
+ # "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),
+ # "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),
+ # "{}/weighted_gan_loss".format(split): weighted_gan_loss.detach().mean(),
+ # "{}gan_adaptive_weight".format(split): d_weight.detach(),
+ # "{}/disc_factor".format(split): torch.tensor(disc_factor),
+ # }
+ # return total_loss, recon_video, log
+
+
+class VEALoss(nn.Module):
+ def __init__(
+ self,
+ perceptual_loss_weight=0.1,
+ kl_loss_weight = 0.000001,
+ vgg=None,
+ vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
+ ):
+ super().__init__()
+
+ # KL Loss
+ self.kl_loss_weight = kl_loss_weight
+ # Perceptual Loss
+ self.perceptual_loss_weight = perceptual_loss_weight
+ self.vgg = None
+ self.perceptual_loss_weight = perceptual_loss_weight
+ if perceptual_loss_weight is not None and perceptual_loss_weight > 0:
+ if not exists(vgg):
+ vgg = torchvision.models.vgg16(
+ weights = vgg_weights
+ )
+ vgg.classifier = Sequential(*vgg.classifier[:-2])
+ self.vgg = vgg
+
+ def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):
+ breakpoint() # TODO: scrutinize
+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]
+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]
+ d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
+ d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
+ d_weight = d_weight * self.discriminator_weight
+ return d_weight
+
+ def forward(
+ self,
+ video,
+ recon_video,
+ posterior,
+ split = "train",
+ ):
+ # reconstruction loss
recon_loss = F.mse_loss(video, recon_video)
- # TODO: check if ths is tensor
- breakpoint()
- total_loss = recon_loss
+ nll_loss = recon_loss
# KL Loss
weighted_kl_loss = 0
@@ -860,10 +961,11 @@ class VAE_3D_V2(nn.Module):
weighted_kl_loss = kl_loss * self.kl_loss_weight
total_loss += weighted_kl_loss
- # Perceptual Loss
- # SCH: NOTE: if mse can pick single frame, if use sum of errors, need to calc for all frames!
+ # perceptual loss
weighted_perceptual_loss = 0
if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0:
+ assert exists(self.vgg)
+ batch, channels, frames = video.shape[:3]
frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
input_vgg_input = pick_video_frame(video, frame_indices)
recon_vgg_input = pick_video_frame(recon_video, frame_indices)
@@ -876,25 +978,63 @@ class VAE_3D_V2(nn.Module):
input_vgg_feats = self.vgg(input_vgg_input)
recon_vgg_feats = self.vgg(recon_vgg_input)
perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)
- # perceptual_loss = self.lpips(input_vgg_input.contiguous(), recon_vgg_input.contiguous())
weighted_perceptual_loss = perceptual_loss * self.perceptual_loss_weight
- total_loss += weighted_perceptual_loss
-
- nll_loss = recon_loss + weighted_kl_loss + weighted_perceptual_loss
+ nll_loss += weighted_perceptual_loss
+
+ # # GAN Loss
+ # if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
+ # try:
+ # d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, last_layer)
+ # except RuntimeError:
+ # assert not self.training
+ # d_weight = torch.tensor(0.0)
+ # # if video_contains_first_frame:
+ # # Since we don't have enough T frames, pad anyways
+ # fake_video = pad_at_dim(recon_video, (discriminator_time_padding, 0), value = 0., dim = 2)
+ # fake_logits = discriminator(fake_video.contiguous())
+ # gan_loss = -torch.mean(fake_logits)
+ # else:
+ # d_weight = torch.tensor(0.0)
+ # gan_loss = torch.tensor(0.0)
+
+ # disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
+ # weighted_gan_loss = d_weight * disc_factor * gan_loss
+ # breakpoint()
+ # total_loss = nll_loss + weighted_gan_loss
+
+ log = {
+ "{}/total_loss".format(split): nll_loss.clone().detach().mean(),
+ "{}/recon_loss".format(split): recon_loss.detach().mean(),
+ "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),
+ "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),
+ }
+ return nll_loss, log
+
+class AdversarialLoss(nn.Module):
+ def __init__(
+ self,
+ discriminator_factor = 1.0,
+ discriminator_start = 50001,
+ ):
+ super().__init__()
+ self.discriminator_factor = discriminator_factor
+ self.discriminator_start = discriminator_start
- # GAN
- # if optimizer_idx == 0: # generator update
+ def forward(
+ self,
+ fake_logits,
+ nll_loss,
+ last_layer,
+ global_step,
+ is_training = True,
+ ):
if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
try:
- d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, self.get_last_layer())
+ d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, last_layer)
except RuntimeError:
- assert not self.training
+ assert not is_training
d_weight = torch.tensor(0.0)
- # if video_contains_first_frame:
- # Since we don't have enough T frames, pad anyways
- fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)
- fake_logits = self.discriminator(fake_video.contiguous()) # DETACH?
gan_loss = -torch.mean(fake_logits)
else:
d_weight = torch.tensor(0.0)
@@ -902,75 +1042,55 @@ class VAE_3D_V2(nn.Module):
disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
weighted_gan_loss = d_weight * disc_factor * gan_loss
- breakpoint()
- total_loss += weighted_gan_loss
- log = {"{}/total_loss".format(split): total_loss.clone().detach().mean(),
- "{}/nll_loss".format(split): nll_loss.detach().mean(),
- "{}/recon_loss".format(split): recon_loss.detach().mean(),
- "{}/weighted_perceptual_loss".format(split): weighted_perceptual_loss.detach().mean(),
- "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),
- "{}/weighted_gan_loss".format(split): weighted_gan_loss.detach().mean(),
- "{}gan_adaptive_weight".format(split): d_weight.detach(),
- "{}/disc_factor".format(split): torch.tensor(disc_factor),
- }
- return total_loss, recon_video, log
-
- ## SCH: move to a different function call
- # if optimizer_idx == 1: # second pass for discriminator update
- # if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
- # # if video_contains_first_frame:
- # # Since we don't have enough T frames, pad anyways
- # real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)
- # fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)
- # real_logits = self.discriminator(real_video.contiguous.detach())
- # fake_logits = self.discriminator(fake_video.contiguous.detach())
- # disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_iter_start)
- # weight_discriminator_loss = disc_factor * self.disc_loss_fn(real_logits, fake_logits)
- # else:
- # weight_discriminator_loss = 0
-
- # breakpoint()
-
- # log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),
- # "{}/logits_real".format(split): real_logits.detach().mean(),
- # "{}/logits_fake".format(split): fake_logits.detach().mean()
- # }
-
- # return weight_discriminator_loss, recon_video, log
-
- def disc_forward(
+ return weighted_gan_loss
+
+class DiscriminatorLoss(nn.Module):
+ def __init__(
self,
- video,
- recon_video,
+ discriminator_factor = 1.0,
+ discriminator_start = 50001,
+ discriminator_loss="hinge",
+ ):
+ super().__init__()
+
+ assert discriminator_loss in ["hinge", "vanilla"]
+ self.discriminator_factor = discriminator_factor
+ self.discriminator_start = discriminator_start
+
+ if discriminator_loss == "hinge":
+ self.disc_loss_fn = hinge_d_loss
+ elif discriminator_loss == "vanilla":
+ self.disc_loss_fn = vanilla_d_loss
+ else:
+ raise ValueError(f"Unknown GAN loss '{discriminator_loss}'.")
+
+ def forward(
+ self,
+ real_logits,
+ fake_logits,
global_step,
- split = "train",
):
if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
- # if video_contains_first_frame:
- # Since we don't have enough T frames, pad anyways
- real_video = pad_at_dim(video, (self.discr_time_padding, 0), value = 0., dim = 2)
- fake_video = pad_at_dim(recon_video, (self.discr_time_padding, 0), value = 0., dim = 2)
- real_logits = self.discriminator(real_video.contiguous.detach())
- fake_logits = self.discriminator(fake_video.contiguous.detach())
disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
weight_discriminator_loss = disc_factor * self.disc_loss_fn(real_logits, fake_logits)
else:
weight_discriminator_loss = 0
- breakpoint()
-
- log = {"{}/weighted_disc_loss".format(split): weight_discriminator_loss.clone().detach().mean(),
- "{}/logits_real".format(split): real_logits.detach().mean(),
- "{}/logits_fake".format(split): fake_logits.detach().mean()
- }
-
- return weight_discriminator_loss, log
+ breakpoint()
+ return weight_discriminator_loss
@MODELS.register_module("VAE_MAGVIT_V2")
def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):
model = VAE_3D_V2(**kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
+
[email protected]_module("DISCRIMINATOR_3D")
+def DISCRIMINATOR_3D(from_pretrained=None, **kwargs):
+ model = StyleGANDiscriminator(**kwargs)
if from_pretrained is not None:
load_checkpoint(model, from_pretrained)
return model
\ No newline at end of file
diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.py
index 853bf0d..0f2a354 100644
--- a/scripts/train-vae-v2.py
+++ b/scripts/train-vae-v2.py
@@ -34,7 +34,7 @@ from opensora.utils.config_utils import (
)
from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, requires_grad, to_torch_dtype
from opensora.utils.train_utils import update_ema, MaskGenerator
-from opensora.models.vae.model_utils import VEA3DLoss
+from opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, pad_at_dim
def main():
@@ -137,10 +137,16 @@ def main():
logger.info(
f"Trainable vae params: {format_numel_str(vae_numel_trainable)}, Total model params: {format_numel_str(vae_numel)}"
)
-
+
+ discriminator = build_module(cfg.discriminator, MODELS, device=device)
+ discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)
+ logger.info(
+ f"Trainable discriminator params: {format_numel_str(discriminator_numel_trainable)}, Total model params: {format_numel_str(discriminator_numel)}"
+ )
# 4.3. move to device
vae = vae.to(device, dtype)
+ discriminator = discriminator.to(device, dtype)
breakpoint() # TODO: check vae parameters
@@ -152,14 +158,16 @@ def main():
lr_scheduler = None
# disc optimizer
disc_optimizer = HybridAdam(
- filter(lambda p: p.requires_grad, vae.disc_parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
+ filter(lambda p: p.requires_grad, discriminator.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
)
disc_lr_scheduler = None
# 4.6. prepare for training
if cfg.grad_checkpoint:
set_grad_checkpoint(vae)
+ set_grad_checkpoint(discriminator)
vae.train()
+ discriminator.train()
# =======================================================
@@ -169,16 +177,17 @@ def main():
vae, optimizer, _, dataloader, lr_scheduler = booster.boost(
model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader
)
- # TODO: do I need to pass the other parameters?
- # TODO: can we use the same booster.boost?
- vae.discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(
- model=vae.discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler
- )
-
torch.set_default_dtype(torch.float)
num_steps_per_epoch = len(dataloader)
logger.info("Boost vae for distributed training")
+
+ discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(
+ model=discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler
+ )
+ logger.info("Boost discriminator for distributed training")
+
+
# =======================================================
# 6. training loop
# =======================================================
@@ -191,6 +200,7 @@ def main():
if cfg.load is not None:
logger.info("Loading checkpoint")
booster.load_model(vae, os.path.join(cfg.load, "model"))
+ booster.load_model(discriminator, os.path.join(cfg.load, "discriminator"))
booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))
booster.load_optimizer(disc_optimizer, os.path.join(cfg.load, "disc_optimizer"))
if lr_scheduler is not None:
@@ -206,11 +216,29 @@ def main():
dataloader.sampler.set_start_index(sampler_start_idx)
- # # define loss function
- # loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)
+ # 6.2 Define loss functions
+ nll_loss_fn = VEALoss(
+ perceptual_loss_weight = cfg.perceptual_loss_weight,
+ kl_loss_weight = cfg.kl_loss_weight,
+ )
+
+ adversarial_loss_fn = AdversarialLoss(
+ discriminator_factor = cfg.discriminator_factor,
+ discriminator_start = cfg.discriminator_start,
+ )
+
+ disc_loss_fn = DiscriminatorLoss(
+ discriminator_factor = cfg.discriminator_factor,
+ discriminator_start = cfg.discriminator_start,
+ discriminator_loss = cfg.discriminator_loss,
+ )
+
+ # 6.3. training loop
+ # calculate discriminator_time_padding
+ disc_time_downsample_factor = 2 ** cfg.discriminator.discriminator_channel_multipliers
+ disc_time_padding = disc_time_downsample_factor - cfg.num_frames % disc_time_downsample_factor
- # 6.2. training loop
for epoch in range(start_epoch, cfg.epochs):
dataloader.sampler.set_epoch(epoch)
dataloader_iter = iter(dataloader)
@@ -241,38 +269,59 @@ def main():
else:
video = x
- # loss = vae.get_loss(x)
- loss, recon_video, log = vae(
+ # padded videos for GAN
+ if global_step > cfg.discriminator_start:
+ real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)
+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)
+
+ # ====== VAE ======
+ optimizer.zero_grad()
+ recon_video, posterior = vae(
video,
- global_step,
video_contains_first_frame = video_contains_first_frame,
- split = "train",
)
-
- # Backward & update
- booster.backward(loss=loss, optimizer=optimizer)
- optimizer.step()
- optimizer.zero_grad()
-
- disc_loss, disc_log = vae.disc_forward(
+ # simple nll loss
+ nll_loss, nll_loss_log = nll_loss_fn(
video,
recon_video,
- global_step,
- split = "train",
+ posterior,
+ split = "train"
)
-
+ vae_loss = nll_loss
+ # adversarial loss
+ if global_step > cfg.discriminator_start:
+ fake_logits = discriminator(fake_video.contiguous)
+ adversarial_loss = adversarial_loss_fn(
+ fake_logits,
+ nll_loss,
+ vae.get_last_layer(),
+ global_step,
+ is_training = vae.training,
+ )
+ vae_loss += adversarial_loss
+ # Backward & update
+ booster.backward(loss=vae_loss, optimizer=optimizer)
+ optimizer.step()
+ # Log loss values:
+ all_reduce_mean(vae_loss)
+ running_loss += vae_loss.item()
+
+ # ====== Discriminator ======
+ disc_optimizer.zero_grad()
+ # if video_contains_first_frame:
+ # Since we don't have enough T frames, pad anyways
+ real_logits = discriminator(real_video.contiguous.detach())
+ fake_logits = discriminator(fake_video.contiguous.detach())
+ disc_loss = disc_loss_fn(real_logits, fake_logits, global_step)
+ # Backward & update
booster.backward(loss=disc_loss, optimizer=disc_optimizer)
disc_optimizer.step()
- disc_optimizer.zero_grad()
-
# Log loss values:
- all_reduce_mean(loss)
- running_loss += loss.item()
all_reduce_mean(disc_loss)
running_disc_loss += disc_loss.item()
+
log_step += 1
-
# Log to tensorboard
if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:
avg_loss = running_loss / log_step
@@ -280,14 +329,15 @@ def main():
pbar.set_postfix({"loss": avg_loss, "disc_loss": avg_disc_loss, "step": step, "global_step": global_step})
running_loss = 0
log_step = 0
- writer.add_scalar("loss", loss.item(), global_step)
+ writer.add_scalar("loss", vae_loss.item(), global_step)
if cfg.wandb:
wandb.log(
{
"iter": global_step,
"num_samples": global_step * total_batch_size,
"epoch": epoch,
- "loss": loss.item(),
+ "loss": vae_loss.item(),
+ "disc_loss": disc_loss.item(),
"avg_loss": avg_loss,
},
step=global_step,
@@ -297,8 +347,8 @@ def main():
if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:
save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")
os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)
- # TODO: save in model?
booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)
+ booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)
booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)
booster.save_optimizer(disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096)