Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

add logvar

Shen-Chenhui235d195daca5
diff --git a/configs/vae_magvit_v2/train/16x128x128.py b/configs/vae_magvit_v2/train/16x128x128.py
new file mode 100644
index 0000000..706b4ce
--- /dev/null
+++ b/configs/vae_magvit_v2/train/16x128x128.py
@@ -0,0 +1,81 @@
+num_frames = 16
+frame_interval = 3
+image_size = (128, 128)
+
+# Define dataset
+root = None
+data_path = "CSV_PATH"
+use_image_transform = False
+num_workers = 4
+video_contains_first_frame = False
+
+# Define acceleration
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+
+# Define model
+
+model = dict(
+ type="VAE_MAGVIT_V2",
+ in_out_channels = 3,
+ latent_embed_dim = 256,
+ filters = 128,
+ num_res_blocks = 4,
+ channel_multipliers = (1, 2, 2, 4),
+ temporal_downsample = (False, True, True),
+ num_groups = 32, # for nn.GroupNorm
+ kl_embed_dim = 64,
+ activation_fn = 'swish',
+ separate_first_frame_encoding = False,
+ custom_conv_padding = None
+)
+
+
+discriminator = dict(
+ type="DISCRIMINATOR_3D",
+ image_size = image_size,
+ num_frames = num_frames,
+ in_channels = 3,
+ filters = 128,
+ channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
+)
+
+
+# loss weights
+logvar_init=0.0
+kl_loss_weight = 0.000001
+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
+discriminator_factor = 1.0 # for discriminator adversarial loss
+# discriminator_loss_weight = 0.5 # for generator adversarial loss
+generator_factor = 0.5 # for generator adversarial loss
+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
+discriminator_loss_type="non-saturating"
+generator_loss_type="non-saturating"
+discriminator_start = 50000 # 50000 NOTE: change to correct val, debug use -1 for now
+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
+ema_decay = 0.999 # ema decay factor for generator
+
+
+# Others
+seed = 42
+outputs = "outputs"
+wandb = False
+
+# Training
+''' NOTE:
+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
+3-6 epochs for pexel, from pexel observation its correct
+'''
+
+epochs = 10
+log_every = 1
+ckpt_every = 1000
+load = None
+
+batch_size = 4
+lr = 1e-4
+grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.py
index 8684ba2..d357911 100644
--- a/configs/vae_magvit_v2/train/17x128x128.py
+++ b/configs/vae_magvit_v2/train/17x128x128.py
@@ -31,16 +31,6 @@ model = dict(
activation_fn = 'swish',
separate_first_frame_encoding = False,
custom_conv_padding = None
- # num_frames = num_frames,
- # image_size = image_size,
- # kl_loss_weight = 0.000001,
- # perceptual_loss_weight = 0.1, # use vgg is not None and more than 0
- # discriminator_factor = 1.0,
- # discriminator_in_channels = 3,
- # discriminator_filters = 128,
- # discriminator_channel_multipliers = (2,4,4,4,4),
- # discriminator_loss="hinge",
- # discriminator_start = 50001,
)
@@ -57,12 +47,14 @@ discriminator = dict(
# loss weights
kl_loss_weight = 0.000001
perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0
-discriminator_loss_weight = 0.5
+discriminator_factor = 1.0 # for discriminator adversarial loss
+# discriminator_loss_weight = 0.5 # for generator adversarial loss
+generator_factor = 0.5 # for generator adversarial loss
lecam_loss_weight = None # TODO: not clear in MAGVIT what is the weight
-discriminator_loss="non-saturating"
+discriminator_loss_type="non-saturating"
+generator_loss_type="non-saturating"
discriminator_start = 50000 # 50000 TODO: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: following MAGVIT config.vqgan.grad_penalty_cost, 10
+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
ema_decay = 0.999 # ema decay factor for generator
diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.py
index 6a8ec42..5fdd923 100644
--- a/opensora/models/vae/vae_3d_v2.py
+++ b/opensora/models/vae/vae_3d_v2.py
@@ -12,12 +12,13 @@ from einops import rearrange, repeat, pack, unpack
import torch.nn.functional as F
import torchvision
from torchvision.models import VGG16_Weights
-from collections import namedtuple
from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers
-from torch import nn, einsum, Tensor
-from kornia.filters import filter3d
+from torch import nn
import math
+# from diffusers.models.modeling_utils import ModelMixin
+
+
"""Encoder and Decoder stuctures with 3D CNNs."""
"""
@@ -28,8 +29,6 @@ NOTE:
!!! opensora read video into [B,C,T,H,W] format output
-TODO:
- check data dimensions format
"""
def cast_tuple(t, length = 1):
return t if isinstance(t, tuple) else ((t,) * length)
@@ -83,8 +82,8 @@ def gradient_penalty_fn(images, output):
# ============== Discriminator Adversarial Loss Functions ==============
def hinge_d_loss(logits_real, logits_fake):
- loss_real = torch.mean(F.relu(1. - logits_real))
- loss_fake = torch.mean(F.relu(1. + logits_fake))
+ loss_real = torch.mean(F.relu(1.0 - logits_real))
+ loss_fake = torch.mean(F.relu(1.0 + logits_fake))
d_loss = 0.5 * (loss_real + loss_fake)
return d_loss
@@ -261,7 +260,7 @@ class BlurPool3D(nn.Module):
filt_2d = a[:,None]*a[None,:]
filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)
- filt = filt_3d/torch.sum(filt_3d) # TODO: modify to make it 3D
+ filt = filt_3d/torch.sum(filt_3d) # SCH: modified to it 3D
self.register_buffer('filt', filt[None,None,:,:,:].repeat((self.channels,1,1,1,1)))
self.pad = get_pad_layer(pad_type)(self.pad_sizes)
@@ -657,7 +656,7 @@ class Decoder(nn.Module):
@MODELS.register_module()
-class VAE_3D_V2(nn.Module):
+class VAE_3D_V2(nn.Module): # , ModelMixin
"""The 3D VAE """
def __init__(
self,
@@ -870,10 +869,11 @@ class VAE_3D_V2(nn.Module):
class VEALoss(nn.Module):
def __init__(
self,
+ logvar_init=0.0,
perceptual_loss_weight = 0.1,
kl_loss_weight = 0.000001,
- vgg=None,
- vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
+ # vgg=None,
+ # vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
device = "cpu",
dtype = "bf16"
):
@@ -890,15 +890,18 @@ class VEALoss(nn.Module):
# KL Loss
self.kl_loss_weight = kl_loss_weight
# Perceptual Loss
- self.vgg = None
+ self.perceptual_loss_fn = LPIPS().eval()
self.perceptual_loss_weight = perceptual_loss_weight
- if perceptual_loss_weight is not None and perceptual_loss_weight > 0.0:
- if not exists(vgg):
- vgg = torchvision.models.vgg16(
- weights = vgg_weights
- )
- vgg.classifier = Sequential(*vgg.classifier[:-2])
- self.vgg = vgg.to(device, dtype).eval() # SCH: added eval
+ self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
+
+ # self.vgg = None
+ # if perceptual_loss_weight is not None and perceptual_loss_weight > 0.0:
+ # if not exists(vgg):
+ # vgg = torchvision.models.vgg16(
+ # weights = vgg_weights
+ # )
+ # vgg.classifier = Sequential(*vgg.classifier[:-2])
+ # self.vgg = vgg.to(device, dtype).eval() # SCH: added eval
def forward(
@@ -906,40 +909,69 @@ class VEALoss(nn.Module):
video,
recon_video,
posterior,
+ nll_weights=None,
split = "train",
- ):
- # reconstruction loss
- recon_loss = F.mse_loss(video, recon_video)
- nll_loss = recon_loss
+ ):
+ video = rearrange(video, "b c t h w -> (b t) c h w").contiguous()
+ recon_video = rearrange(recon_video, "b c t h w -> (b t) c h w").contiguous()
- # KL Loss
- weighted_kl_loss = 0
- if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:
- kl_loss = posterior.kl()
- # NOTE: since we use MSE, here use mean as well, else use sum
- kl_loss = torch.mean(kl_loss) / kl_loss.shape[0]
- weighted_kl_loss = kl_loss * self.kl_loss_weight
- nll_loss += weighted_kl_loss
+ # reconstruction loss
+ recon_loss = torch.abs(video - recon_video)
# perceptual loss
- weighted_perceptual_loss = 0
if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:
- assert exists(self.vgg)
- batch, channels, frames = video.shape[:3]
- frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
- input_vgg_input = pick_video_frame(video, frame_indices)
- recon_vgg_input = pick_video_frame(recon_video, frame_indices)
+ # handle channels
+ channels = video.shape[1]
if channels == 1:
input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)
recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)
elif channels == 4: # SCH: take the first 3 for perceptual loss calc
input_vgg_input = input_vgg_input[:, :3]
recon_vgg_input = recon_vgg_input[:, :3]
- input_vgg_feats = self.vgg(input_vgg_input)
- recon_vgg_feats = self.vgg(recon_vgg_input)
- perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)
- weighted_perceptual_loss = perceptual_loss * self.perceptual_loss_weight
- nll_loss += weighted_perceptual_loss
+ perceptual_loss = self.perceptual_loss_fn(video, recon_video)
+ recon_loss = recon_loss + self.perceptual_loss_weight * perceptual_loss
+
+ nll_loss = recon_loss / torch.exp(self.logvar) + self.logvar
+ weighted_nll_loss = nll_loss
+ if nll_weights is not None:
+ weighted_nll_loss = nll_weights * nll_loss
+ weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
+ nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
+
+ # recon_loss = F.mse_loss(video, recon_video)
+ # nll_loss = recon_loss
+
+ # KL Loss
+ weighted_kl_loss = 0
+ if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:
+ kl_loss = posterior.kl()
+ # # NOTE: since we use MSE, here use mean as well, else use sum
+ # kl_loss = torch.mean(kl_loss) / kl_loss.shape[0]
+ kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
+ weighted_kl_loss = kl_loss * self.kl_loss_weight
+
+ return nll_loss, weighted_nll_loss, weighted_kl_loss
+
+ # # perceptual loss
+ # # TODO: use all frames and take average instead of sampling
+ # weighted_perceptual_loss = 0
+ # if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:
+ # assert exists(self.vgg)
+ # batch, channels, frames = video.shape[:3]
+ # frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
+ # input_vgg_input = pick_video_frame(video, frame_indices)
+ # recon_vgg_input = pick_video_frame(recon_video, frame_indices)
+ # if channels == 1:
+ # input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)
+ # recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)
+ # elif channels == 4: # SCH: take the first 3 for perceptual loss calc
+ # input_vgg_input = input_vgg_input[:, :3]
+ # recon_vgg_input = recon_vgg_input[:, :3]
+ # input_vgg_feats = self.vgg(input_vgg_input)
+ # recon_vgg_feats = self.vgg(recon_vgg_input)
+ # perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)
+ # weighted_perceptual_loss = perceptual_loss * self.perceptual_loss_weight
+ # nll_loss += weighted_perceptual_loss
# log = {
# "{}/total_loss".format(split): nll_loss.clone().detach().mean(),
@@ -948,7 +980,7 @@ class VEALoss(nn.Module):
# "{}/weighted_kl_loss".format(split): weighted_kl_loss.detach().mean(),
# }
- return nll_loss
+ # return nll_loss, weighted_kl_loss
class AdversarialLoss(nn.Module):
@@ -956,19 +988,21 @@ class AdversarialLoss(nn.Module):
self,
discriminator_factor = 1.0,
discriminator_start = 50001,
- discriminator_loss_weight = 0.5,
+ generator_factor = 0.5,
+ generator_loss_type="non-saturating",
):
super().__init__()
self.discriminator_factor = discriminator_factor
self.discriminator_start = discriminator_start
- self.discriminator_loss_weight = discriminator_loss_weight
+ self.generator_factor = generator_factor
+ self.generator_loss_type = generator_loss_type
def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):
nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat
g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added create_graph=True
d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
- d_weight = d_weight * self.discriminator_loss_weight
+ d_weight = d_weight * self.generator_factor
return d_weight
def forward(
@@ -979,12 +1013,22 @@ class AdversarialLoss(nn.Module):
global_step,
is_training = True,
):
- gan_loss = -torch.mean(fake_logits)
+ # NOTE: following MAGVIT to allow non_saturating
+ assert self.generator_loss_type in ["hinge", "vanilla", "non-saturating"]
+ if self.generator_loss_type == "hinge":
+ gen_loss = -torch.mean(fake_logits)
+ elif self.generator_loss_type == "non-saturating":
+ gen_loss = torch.mean(
+ sigmoid_cross_entropy_with_logits(
+ labels=torch.ones_like(fake_logits), logits=fake_logits)
+ )
+ else:
+ raise ValueError("Generator loss {} not supported".format(self.generator_loss_type))
+
+
if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
- # DEBUG: NOTE: something about this d_weigth calculation is causing issue
- # d_weight = 1
try:
- d_weight = self.calculate_adaptive_weight(nll_loss, gan_loss, last_layer)
+ d_weight = self.calculate_adaptive_weight(nll_loss, gen_loss, last_layer)
except RuntimeError:
assert not is_training
d_weight = torch.tensor(0.0)
@@ -992,9 +1036,9 @@ class AdversarialLoss(nn.Module):
d_weight = torch.tensor(0.0)
disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
- weighted_gan_loss = d_weight * disc_factor * gan_loss
+ weighted_gen_loss = d_weight * disc_factor * gen_loss
- return weighted_gan_loss
+ return weighted_gen_loss
@@ -1003,18 +1047,18 @@ class DiscriminatorLoss(nn.Module):
self,
discriminator_factor = 1.0,
discriminator_start = 50001,
- discriminator_loss="non-saturating",
- lecam_loss_weight=0,
- gradient_penalty_loss_weight=10, # SCH: following MAGVIT config.vqgan.grad_penalty_cost
+ discriminator_loss_type="non-saturating",
+ lecam_loss_weight=None,
+ gradient_penalty_loss_weight=None, # SCH: following MAGVIT config.vqgan.grad_penalty_cost
):
super().__init__()
- assert discriminator_loss in ["hinge", "vanilla", "non-saturating"]
+ assert discriminator_loss_type in ["hinge", "vanilla", "non-saturating"]
self.discriminator_factor = discriminator_factor
self.discriminator_start = discriminator_start
self.lecam_loss_weight = lecam_loss_weight
self.gradient_penalty_loss_weight = gradient_penalty_loss_weight
- self.discriminator_loss_type = discriminator_loss
+ self.discriminator_loss_type = discriminator_loss_type
def forward(
diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.py
index 88a11f8..a210193 100644
--- a/scripts/train-vae-v2.py
+++ b/scripts/train-vae-v2.py
@@ -37,6 +37,8 @@ from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_num
from opensora.utils.train_utils import update_ema, MaskGenerator
from opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, pad_at_dim
+
+
# efficiency
# from torch.profiler import profile, record_function, ProfilerActivity
@@ -218,7 +220,8 @@ def main():
dataloader.sampler.set_start_index(sampler_start_idx)
# 6.2 Define loss functions
- nll_loss_fn = VEALoss(
+ vae_loss_fn = VEALoss(
+ logvar_init=cfg.logvar_init,
perceptual_loss_weight = cfg.perceptual_loss_weight,
kl_loss_weight = cfg.kl_loss_weight,
device=device,
@@ -228,12 +231,14 @@ def main():
adversarial_loss_fn = AdversarialLoss(
discriminator_factor = cfg.discriminator_factor,
discriminator_start = cfg.discriminator_start,
+ generator_factor = cfg.generator_factor,
+ generator_loss_type = cfg.generator_loss_type,
)
disc_loss_fn = DiscriminatorLoss(
discriminator_factor = cfg.discriminator_factor,
discriminator_start = cfg.discriminator_start,
- discriminator_loss = cfg.discriminator_loss,
+ discriminator_loss_type = cfg.discriminator_loss_type,
lecam_loss_weight = cfg.lecam_loss_weight,
gradient_penalty_loss_weight = cfg.gradient_penalty_loss_weight,
)
@@ -303,7 +308,6 @@ def main():
video = x
# ====== VAE ======
- optimizer.zero_grad()
recon_video, posterior = vae(
video,
video_contains_first_frame = video_contains_first_frame,
@@ -311,19 +315,18 @@ def main():
# ====== Generator Loss ======
# simple nll loss
- nll_loss = nll_loss_fn(
+ nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(
video,
recon_video,
posterior,
split = "train"
)
- vae_loss = nll_loss
-
+
# adversarial loss
if global_step > cfg.discriminator_start:
# padded videos for GAN
fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)
- fake_logits = discriminator(fake_video) # TODO: take out contiguous?
+ fake_logits = discriminator(fake_video.contiguous()) # TODO: take out contiguous?
adversarial_loss = adversarial_loss_fn(
fake_logits,
nll_loss,
@@ -331,12 +334,16 @@ def main():
global_step,
is_training = vae.training,
)
- vae_loss += adversarial_loss
- # Backward & update
+
+ vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss
+
+ optimizer.zero_grad()
+ # Backward & update
booster.backward(loss=vae_loss, optimizer=optimizer)
# NOTE: clip gradients? this is done in Open-Sora-Plan
torch.nn.utils.clip_grad_norm_(vae.parameters(), 1)
optimizer.step()
+
# Log loss values:
all_reduce_mean(vae_loss)
running_loss += vae_loss.item()
@@ -345,7 +352,7 @@ def main():
# ====== Discriminator Loss ======
if global_step > cfg.discriminator_start:
- disc_optimizer.zero_grad()
+
# if video_contains_first_frame:
# Since we don't have enough T frames, pad anyways
@@ -354,11 +361,9 @@ def main():
if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:
real_video = real_video.requires_grad_()
-
- real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculation
-
- if cfg.gradient_penalty_loss_weight is None:
- real_logits = real_logits.detach()
+ real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculation
+ else:
+ real_logits = discriminator(real_video.contiguous().detach())
fake_logits = discriminator(fake_video.contiguous().detach())
disc_loss = disc_loss_fn(
@@ -375,11 +380,11 @@ def main():
lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(real_logits.clone().detach())
lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(fake_logits.clone().detach())
+ disc_optimizer.zero_grad()
# Backward & update
booster.backward(loss=disc_loss, optimizer=disc_optimizer)
# NOTE: TODO: clip gradients? this is done in Open-Sora-Plan
torch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1)
-
disc_optimizer.step()
# Log loss values: