Open-Sora: Democratizing Efficient Video Production for All
Python
debug
Shen-Chenhui52e869079c54
diff --git a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py b/configs/vae_magvit_v2/inference/17x128x128_pixabay.pyindex 86b85d8..49ac37c 100644--- a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py+++ b/configs/vae_magvit_v2/inference/17x128x128_pixabay.py@@ -88,12 +88,8 @@ magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 1283-6 epochs for pexel, from pexel observation its correct'''-epochs = 10-log_every = 1-ckpt_every = 500-load = None-batch_size = 4+batch_size = 1lr = 1e-4grad_clip = 1.0diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae-v2.pyindex c666e98..58d223c 100644--- a/scripts/inference-vae-v2.py+++ b/scripts/inference-vae-v2.py@@ -9,7 +9,7 @@ from mmengine.runner import set_random_seedfrom opensora.acceleration.parallel_states import set_sequence_parallel_groupfrom opensora.datasets import save_samplefrom opensora.registry import MODELS, SCHEDULERS, build_module-from opensora.utils.config_utils import parse_configs+from opensora.utils.config_utils import parse_configs, load_jsonfrom opensora.utils.misc import to_torch_dtypefrom opensora.datasets import prepare_dataloader, prepare_variable_dataloaderfrom opensora.registry import DATASETS, MODELS, build_module@@ -19,7 +19,7 @@ from opensora.acceleration.parallel_states import (set_sequence_parallel_group,)from tqdm import tqdm-from opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, pad_at_dim+from opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, LeCamEMA, pad_at_dimfrom einops import rearrangefrom colossalai.utils import get_current_device@@ -129,9 +129,24 @@ def main():lecam_loss_weight = cfg.lecam_loss_weight,gradient_penalty_loss_weight = cfg.gradient_penalty_loss_weight,)++ # LeCam EMA for discriminator+ lecam_path = os.path.join(cfg.load, "lecam_states.json")+ if cfg.lecam_loss_weight is not None and os.path.exists(lecam_path):+ lecam_state = load_json(lecam_path)+ lecam_ema_real, lecam_ema_fake = lecam_state["lecam_ema_real"], lecam_state["lecam_ema_fake"]+ lecam_ema = LeCamEMA(+ decay=cfg.ema_decay, ema_real=lecam_ema_real, ema_fake=lecam_ema_fake, dtype=dtype, device=device+ )+ else:+ lecam_ema = LeCamEMA(+ decay=cfg.ema_decay, dtype=dtype, device=device+ )+running_loss = 0.0+ running_nll = 0.0running_disc_loss = 0.0loss_steps = 0@@ -142,9 +157,6 @@ def main():disc_time_padding = 0video_contains_first_frame = cfg.video_contains_first_frame- lecam_ema_real = torch.tensor(0.0)- lecam_ema_fake = torch.tensor(0.0)-total_steps = len(dataloader)if cfg.max_test_samples > 0:total_steps = min(int(cfg.max_test_samples//cfg.batch_size), total_steps)@@ -195,7 +207,7 @@ def main():)fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)- fake_logits = discriminator(fake_video.contiguous()) # TODO: take out contiguous?+ fake_logits = discriminator(fake_video.contiguous())adversarial_loss = adversarial_loss_fn(fake_logits,nll_loss,@@ -217,6 +229,8 @@ def main():real_logits = discriminator(real_video.contiguous().detach())fake_logits = discriminator(fake_video.contiguous().detach())++ lecam_ema_real, lecam_ema_fake = lecam_ema.get()weighted_d_adversarial_loss, lecam_loss, gradient_penalty_loss = disc_loss_fn(real_logits,fake_logits,@@ -228,15 +242,10 @@ def main():disc_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty_loss- if cfg.ema_decay is not None:- # SCH: TODO: is this written properly like this for moving average? e.g. distributed training etc.- lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(real_logits.clone().detach())- lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(fake_logits.clone().detach())--loss_steps += 1- running_disc_loss = disc_loss.item()/loss_steps + disc_loss * ((loss_steps - 1) / loss_steps)+ running_disc_loss = disc_loss.item()/loss_steps + running_disc_loss * ((loss_steps - 1) / loss_steps)running_loss = vae_loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)+ running_nll = nll_loss.item() / loss_steps + running_nll * ((loss_steps - 1) / loss_steps)# ===== Spatial VAE =====@@ -266,6 +275,7 @@ def main():if cfg.calc_loss:print("test vae loss:", running_loss)+ print("test nll loss:", running_nll)print("test disc loss:", running_disc_loss)