Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

merged

zhengzangw9632b5bcf875
diff --cc opensora/models/vae/vae_3d.py
index 0cf288a,29491b4..9c61c97
--- a/opensora/models/vae/vae_3d.py
+++ b/opensora/models/vae/vae_3d.py
@@@ -6,12 -6,12 +6,11 @@@ import numpy as n
import torch
import torch.nn as nn
import torch.nn.functional as F
- from einops import pack, rearrange, repeat
-from einops import pack, rearrange, unpack
++from einops import pack, rearrange
from opensora.registry import MODELS
from opensora.utils.ckpt_utils import find_model, load_checkpoint
--from .lpips import LPIPS
from .utils import DiagonalGaussianDistribution
@@@ -180,12 -71,9 +70,6 @@@ def SameConv2d(dim_in, dim_out, kernel_
return nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, padding=padding)
- def adopt_weight(weight, global_step, threshold=0, value=0.0):
- if global_step < threshold:
- weight = value
- return weight
-
--
--
class CausalConv3d(nn.Module):
def __init__(
self,
@@@ -1042,238 -927,229 +926,6 @@@ class VAE_3D_V2(nn.Module): # , ModelM
return recon_video, posterior
- class VEALoss(nn.Module):
- def __init__(
- self,
- logvar_init=0.0,
- perceptual_loss_weight=0.1,
- kl_loss_weight=0.000001,
- # vgg=None,
- # vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
- device="cpu",
- dtype="bf16",
- ):
- super().__init__()
--
-<<<<<<< HEAD
-=======
-- if type(dtype) == str:
-- if dtype == "bf16":
-- dtype = torch.bfloat16
-- elif dtype == "fp16":
-- dtype = torch.float16
-- else:
-- raise NotImplementedError(f"dtype: {dtype}")
--
-- # KL Loss
-- self.kl_loss_weight = kl_loss_weight
-- # Perceptual Loss
-- self.perceptual_loss_fn = LPIPS().eval().to(device, dtype)
-- self.perceptual_loss_weight = perceptual_loss_weight
-- self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
--
-- def forward(
-- self,
-- video,
-- recon_video,
-- posterior,
-- nll_weights=None,
-- split="train",
-- ):
-- video = rearrange(video, "b c t h w -> (b t) c h w").contiguous()
-- recon_video = rearrange(recon_video, "b c t h w -> (b t) c h w").contiguous()
--
-- # reconstruction loss
-- recon_loss = torch.abs(video - recon_video)
--
-- # perceptual loss
-- if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:
-- # handle channels
-- channels = video.shape[1]
-- assert channels in {1, 3, 4}
-- if channels == 1:
-- input_vgg_input = repeat(video, "b 1 h w -> b c h w", c=3)
-- recon_vgg_input = repeat(recon_video, "b 1 h w -> b c h w", c=3)
-- elif channels == 4: # SCH: take the first 3 for perceptual loss calc
-- input_vgg_input = video[:, :3]
-- recon_vgg_input = recon_video[:, :3]
-- else:
-- input_vgg_input = video
-- recon_vgg_input = recon_video
--
-- perceptual_loss = self.perceptual_loss_fn(input_vgg_input, recon_vgg_input)
-- recon_loss = recon_loss + self.perceptual_loss_weight * perceptual_loss
--
-- nll_loss = recon_loss / torch.exp(self.logvar) + self.logvar
-- weighted_nll_loss = nll_loss
-- if nll_weights is not None:
-- weighted_nll_loss = nll_weights * nll_loss
-- weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
-- nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
--
-- # KL Loss
-- weighted_kl_loss = 0
-- if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:
-- kl_loss = posterior.kl()
-- kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
-- weighted_kl_loss = kl_loss * self.kl_loss_weight
--
-- return nll_loss, weighted_nll_loss, weighted_kl_loss
--
--
--class AdversarialLoss(nn.Module):
-- def __init__(
-- self,
-- discriminator_factor=1.0,
-- discriminator_start=50001,
-- generator_factor=0.5,
-- generator_loss_type="non-saturating",
-- ):
-- super().__init__()
-- self.discriminator_factor = discriminator_factor
-- self.discriminator_start = discriminator_start
-- self.generator_factor = generator_factor
-- self.generator_loss_type = generator_loss_type
--
-- def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):
-- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat
-- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[
-- 0
-- ] # SCH: TODO: debug added create_graph=True
-- d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
-- d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
-- d_weight = d_weight * self.generator_factor
-- return d_weight
--
-- def forward(
-- self,
-- fake_logits,
-- nll_loss,
-- last_layer,
-- global_step,
-- is_training=True,
-- ):
-- # NOTE: following MAGVIT to allow non_saturating
-- assert self.generator_loss_type in ["hinge", "vanilla", "non-saturating"]
--
-- if self.generator_loss_type == "hinge":
-- gen_loss = -torch.mean(fake_logits)
-- elif self.generator_loss_type == "non-saturating":
-- gen_loss = torch.mean(
-- sigmoid_cross_entropy_with_logits(labels=torch.ones_like(fake_logits), logits=fake_logits)
-- )
-- else:
-- raise ValueError("Generator loss {} not supported".format(self.generator_loss_type))
--
-- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
-- try:
-- d_weight = self.calculate_adaptive_weight(nll_loss, gen_loss, last_layer)
-- except RuntimeError:
-- assert not is_training
-- d_weight = torch.tensor(0.0)
-- else:
-- d_weight = torch.tensor(0.0)
--
-- disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
-- weighted_gen_loss = d_weight * disc_factor * gen_loss
--
-- return weighted_gen_loss
--
--
--class LeCamEMA:
-- def __init__(self, ema_real=0.0, ema_fake=0.0, decay=0.999, dtype=torch.bfloat16, device="cpu"):
-- self.decay = decay
-- self.ema_real = torch.tensor(ema_real).to(device, dtype)
-- self.ema_fake = torch.tensor(ema_fake).to(device, dtype)
--
-- def update(self, ema_real, ema_fake):
-- self.ema_real = self.ema_real * self.decay + ema_real * (1 - self.decay)
-- self.ema_fake = self.ema_fake * self.decay + ema_fake * (1 - self.decay)
--
-- def get(self):
-- return self.ema_real, self.ema_fake
--
--
--class DiscriminatorLoss(nn.Module):
-- def __init__(
-- self,
-- discriminator_factor=1.0,
-- discriminator_start=50001,
-- discriminator_loss_type="non-saturating",
-- lecam_loss_weight=None,
-- gradient_penalty_loss_weight=None, # SCH: following MAGVIT config.vqgan.grad_penalty_cost
-- ):
-- super().__init__()
--
-- assert discriminator_loss_type in ["hinge", "vanilla", "non-saturating"]
-- self.discriminator_factor = discriminator_factor
-- self.discriminator_start = discriminator_start
-- self.lecam_loss_weight = lecam_loss_weight
-- self.gradient_penalty_loss_weight = gradient_penalty_loss_weight
-- self.discriminator_loss_type = discriminator_loss_type
--
-- def forward(
-- self,
-- real_logits,
-- fake_logits,
-- global_step,
-- lecam_ema_real=None,
-- lecam_ema_fake=None,
-- real_video=None,
-- split="train",
-- ):
-- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
-- disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
--
-- if self.discriminator_loss_type == "hinge":
-- disc_loss = hinge_d_loss(real_logits, fake_logits)
-- elif self.discriminator_loss_type == "non-saturating":
-- if real_logits is not None:
-- real_loss = sigmoid_cross_entropy_with_logits(
-- labels=torch.ones_like(real_logits), logits=real_logits
-- )
-- else:
-- real_loss = 0.0
-- if fake_logits is not None:
-- fake_loss = sigmoid_cross_entropy_with_logits(
-- labels=torch.zeros_like(fake_logits), logits=fake_logits
-- )
-- else:
-- fake_loss = 0.0
-- disc_loss = 0.5 * (torch.mean(real_loss) + torch.mean(fake_loss))
-- elif self.discriminator_loss_type == "vanilla":
-- disc_loss = vanilla_d_loss(real_logits, fake_logits)
-- else:
-- raise ValueError(f"Unknown GAN loss '{self.discriminator_loss_type}'.")
--
-- weighted_d_adversarial_loss = disc_factor * disc_loss
--
-- else:
-- weighted_d_adversarial_loss = 0
--
-- lecam_loss = torch.tensor(0.0)
-- if self.lecam_loss_weight is not None and self.lecam_loss_weight > 0.0:
-- real_pred = torch.mean(real_logits)
-- fake_pred = torch.mean(fake_logits)
-- lecam_loss = lecam_reg(real_pred, fake_pred, lecam_ema_real, lecam_ema_fake)
-- lecam_loss = lecam_loss * self.lecam_loss_weight
--
-- gradient_penalty = torch.tensor(0.0)
-- if self.gradient_penalty_loss_weight is not None and self.gradient_penalty_loss_weight > 0.0:
-- assert real_video is not None
-- gradient_penalty = gradient_penalty_fn(real_video, real_logits)
-- # gradient_penalty = r1_penalty(real_video, real_logits) # MAGVIT uses r1 penalty
-- gradient_penalty *= self.gradient_penalty_loss_weight
--
-- # discriminator_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty
--
-- # log = {
-- # "{}/d_adversarial_loss".format(split): weighted_d_adversarial_loss.detach().mean(),
-- # "{}/lecam_loss".format(split): lecam_loss.detach().mean(),
-- # "{}/gradient_penalty".format(split): gradient_penalty.detach().mean(),
-- # }
--
-- return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)
->>>>>>> d7afd4d9dbe00ba5933aa5ea3431351b97d2a7de
--
--
@MODELS.register_module("VAE_MAGVIT_V2")
def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):
model = VAE_3D_V2(**kwargs)
diff --cc scripts/inference-vae.py
index b9731d2,e566a24..865b50c
--- a/scripts/inference-vae.py
+++ b/scripts/inference-vae.py
@@@ -9,7 -10,10 +9,8 @@@ from tqdm import tqd
from opensora.acceleration.parallel_states import get_data_parallel_group
from opensora.datasets import prepare_dataloader, save_sample
- from opensora.models.vae.vae_3d import AdversarialLoss, DiscriminatorLoss, LeCamEMA, VEALoss, pad_at_dim
-
+ from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, VEALoss
+ from opensora.models.vae.vae_3d import LeCamEMA, pad_at_dim
-
from opensora.registry import DATASETS, MODELS, build_module
from opensora.utils.config_utils import parse_configs
from opensora.utils.misc import to_torch_dtype