Open-Sora: Democratizing Efficient Video Production for All
Python
update config
zhengzangw1171e5b6f9ed
diff --git a/.gitignore b/.gitignoreindex b9f8121..4d5dc97 100644--- a/.gitignore+++ b/.gitignore@@ -181,3 +181,4 @@ cache/hostfilegradio_cached_examples/wandb/+taming/diff --git a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py b/configs/vae_magvit_v2/inference/17x128x128_pixabay.pyindex 9f6cd50..b264c2e 100644--- a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py+++ b/configs/vae_magvit_v2/inference/17x128x128_pixabay.py@@ -36,44 +36,44 @@ vae_2d = dict(model = dict(type="VAE_MAGVIT_V2",- in_out_channels = 4,- latent_embed_dim = 64,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 4,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = True,- custom_conv_padding = None,- encoder_double_z = True,+ in_out_channels=4,+ latent_embed_dim=64,+ filters=128,+ num_res_blocks=4,+ channel_multipliers=(1, 2, 2, 4),+ temporal_downsample=(False, True, True),+ num_groups=32, # for nn.GroupNorm+ kl_embed_dim=4,+ activation_fn="swish",+ separate_first_frame_encoding=False,+ disable_space=True,+ custom_conv_padding=None,+ encoder_double_z=True,)discriminator = dict(type="DISCRIMINATOR_3D",- image_size = (128, 128),- num_frames = num_frames,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4),+ image_size=(128, 128),+ num_frames=num_frames,+ in_channels=3,+ filters=128,+ channel_multipliers=(2, 4, 4, 4, 4),# channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution)-# loss weights-logvar_init=0.0+# loss weights+logvar_init = 0.0kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0+discriminator_factor = 1.0 # for discriminator adversarial loss# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # for generator adversarial loss-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"-discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use+generator_factor = 0.1 # for generator adversarial loss+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight+discriminator_loss_type = "non-saturating"+generator_loss_type = "non-saturating"+discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't useema_decay = 0.999 # ema decay factor for generator@@ -83,15 +83,15 @@ save_dir = "outputs/samples_pixabay_17"wandb = False# Training-''' NOTE:+""" NOTE:magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],3-6 epochs for pexel, from pexel observation its correct-'''+"""batch_size = 1lr = 1e-4grad_clip = 1.0-calc_loss = True\ No newline at end of file+calc_loss = Truediff --git a/configs/vae_magvit_v2/inference/1x256x256_pixabay.py b/configs/vae_magvit_v2/inference/1x256x256_pixabay.pyindex 31a1327..98c162e 100644--- a/configs/vae_magvit_v2/inference/1x256x256_pixabay.py+++ b/configs/vae_magvit_v2/inference/1x256x256_pixabay.py@@ -1,23 +1,18 @@num_frames = 1-image_size = (256, 256)-dataset = dict(type="VideoTextDataset",data_path=None,num_frames=num_frames,- frame_interval=3,+ frame_interval=1,image_size=image_size,get_text=False,)fps = 24 // 3is_vae = True--# Define datasetmax_test_samples = -1-# Define accelerationnum_workers = 4dtype = "bf16"@@ -33,67 +28,70 @@ video_contains_first_frame = Truevae_2d = dict(type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+ micro_batch_size=4,+ local_files_only=True,)model = dict(type="VAE_MAGVIT_V2",- in_out_channels = 4,- latent_embed_dim = 64,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 4,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = True,- custom_conv_padding = None,- encoder_double_z = True,+ in_out_channels=4,+ latent_embed_dim=64,+ filters=128,+ num_res_blocks=4,+ channel_multipliers=(1, 2, 2, 4),+ temporal_downsample=(False, True, True),+ num_groups=32, # for nn.GroupNorm+ kl_embed_dim=4,+ activation_fn="swish",+ separate_first_frame_encoding=False,+ disable_space=True,+ custom_conv_padding=None,+ encoder_double_z=True,)discriminator = dict(type="DISCRIMINATOR_3D",- image_size = image_size,- num_frames = num_frames,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4),+ image_size=image_size,+ num_frames=num_frames,+ in_channels=3,+ filters=128,+ channel_multipliers=(2, 4, 4, 4, 4),# channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution)-# loss weights-logvar_init=0.0+# loss weights+logvar_init = 0.0kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0+discriminator_factor = 1.0 # for discriminator adversarial loss# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # for generator adversarial loss-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"-discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use+generator_factor = 0.1 # for generator adversarial loss+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight+discriminator_loss_type = "non-saturating"+generator_loss_type = "non-saturating"+discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't useema_decay = 0.999 # ema decay factor for generator# Othersseed = 42-save_dir = "outputs/samples_pixabay_17"+save_dir = "samples/samples_pixabay_17"wandb = False# Training-''' NOTE:+""" NOTE:magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],3-6 epochs for pexel, from pexel observation its correct-'''+"""batch_size = 1lr = 1e-4grad_clip = 1.0-calc_loss = True\ No newline at end of file+calc_loss = Truediff --git a/configs/vae_magvit_v2/train/1x256x256_pixabay.py b/configs/vae_magvit_v2/train/1x256x256_pixabay.pyindex 2c34afa..91d99a9 100644--- a/configs/vae_magvit_v2/train/1x256x256_pixabay.py+++ b/configs/vae_magvit_v2/train/1x256x256_pixabay.py@@ -48,8 +48,8 @@ model = dict(activation_fn="swish",separate_first_frame_encoding=False,disable_space=True,- encoder_double_z=True,custom_conv_padding=None,+ encoder_double_z=True,)@@ -60,8 +60,8 @@ discriminator = dict(in_channels=3,filters=128,use_pretrained=True, # NOTE: set to False only if we want to disable load- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z+ channel_multipliers=(2, 4, 4, 4, 4), # (2,4,4,4) for 64x64 resolution+ # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z)@@ -76,7 +76,7 @@ discriminator_loss_type = "non-saturating"generator_loss_type = "non-saturating"# discriminator_loss_type="hinge"# generator_loss_type="hinge"-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch+discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epochgradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't useema_decay = 0.999 # ema decay factor for generatordiff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.pyindex 2b0655b..804de27 100644--- a/opensora/utils/ckpt_utils.py+++ b/opensora/utils/ckpt_utils.py@@ -256,7 +256,7 @@ def create_logger(logging_dir):return logger-def load_checkpoint(model, ckpt_path, save_as_pt=True, model_name="model"):+def load_checkpoint(model, ckpt_path, save_as_pt=False, model_name="model"):if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):state_dict = find_model(ckpt_path, model=model)missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae-v2.pyindex ab43079..2fd64fe 100644--- a/scripts/inference-vae-v2.py+++ b/scripts/inference-vae-v2.py@@ -4,25 +4,16 @@ import colossalaiimport torchimport torch.distributed as distfrom colossalai.cluster import DistCoordinator-from mmengine.runner import set_random_seed+from colossalai.utils import get_current_device+from einops import rearrange+from tqdm import tqdm-from opensora.acceleration.parallel_states import set_sequence_parallel_group-from opensora.datasets import save_sample-from opensora.registry import MODELS, SCHEDULERS, build_module+from opensora.acceleration.parallel_states import get_data_parallel_group+from opensora.datasets import prepare_dataloader, save_sample+from opensora.models.vae.vae_3d_v2 import AdversarialLoss, DiscriminatorLoss, LeCamEMA, VEALoss, pad_at_dim+from opensora.registry import DATASETS, MODELS, build_modulefrom opensora.utils.config_utils import parse_configsfrom opensora.utils.misc import to_torch_dtype-from opensora.datasets import prepare_dataloader, prepare_variable_dataloader-from opensora.registry import DATASETS, MODELS, build_module-from opensora.acceleration.parallel_states import (- get_data_parallel_group,- set_data_parallel_group,- set_sequence_parallel_group,-)-from tqdm import tqdm-from opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, LeCamEMA, pad_at_dim--from einops import rearrange-from colossalai.utils import get_current_devicedef main():@@ -49,9 +40,6 @@ def main():device = get_current_device()dtype = to_torch_dtype(cfg.dtype)---# ======================================================# 3. build dataset and dataloader# ======================================================@@ -101,43 +89,37 @@ def main():save_dir = cfg.save_diros.makedirs(save_dir, exist_ok=True)--# 4.1. batch generation-+# define loss functionif cfg.calc_loss:vae_loss_fn = VEALoss(logvar_init=cfg.logvar_init,- perceptual_loss_weight = cfg.perceptual_loss_weight,- kl_loss_weight = cfg.kl_loss_weight,+ perceptual_loss_weight=cfg.perceptual_loss_weight,+ kl_loss_weight=cfg.kl_loss_weight,device=device,dtype=dtype,)adversarial_loss_fn = AdversarialLoss(- discriminator_factor = cfg.discriminator_factor,- discriminator_start = cfg.discriminator_start,- generator_factor = cfg.generator_factor,- generator_loss_type = cfg.generator_loss_type,+ discriminator_factor=cfg.discriminator_factor,+ discriminator_start=cfg.discriminator_start,+ generator_factor=cfg.generator_factor,+ generator_loss_type=cfg.generator_loss_type,)disc_loss_fn = DiscriminatorLoss(- discriminator_factor = cfg.discriminator_factor,- discriminator_start = cfg.discriminator_start,- discriminator_loss_type = cfg.discriminator_loss_type,- lecam_loss_weight = cfg.lecam_loss_weight,- gradient_penalty_loss_weight = cfg.gradient_penalty_loss_weight,- )+ discriminator_factor=cfg.discriminator_factor,+ discriminator_start=cfg.discriminator_start,+ discriminator_loss_type=cfg.discriminator_loss_type,+ lecam_loss_weight=cfg.lecam_loss_weight,+ gradient_penalty_loss_weight=cfg.gradient_penalty_loss_weight,+ )- # LeCam EMA for discriminator+ # LeCam EMA for discriminator- lecam_ema = LeCamEMA(- decay=cfg.ema_decay, dtype=dtype, device=device- )+ lecam_ema = LeCamEMA(decay=cfg.ema_decay, dtype=dtype, device=device)--running_loss = 0.0running_nll = 0.0running_disc_loss = 0.0@@ -152,7 +134,7 @@ def main():total_steps = len(dataloader)if cfg.max_test_samples > 0:- total_steps = min(int(cfg.max_test_samples//cfg.batch_size), total_steps)+ total_steps = min(int(cfg.max_test_samples // cfg.batch_size), total_steps)print(f"limiting test dataset to {int(cfg.max_test_samples//cfg.batch_size) * cfg.batch_size}")dataloader_iter = iter(dataloader)@@ -169,7 +151,7 @@ def main():is_image = x.ndim == 4if is_image:- video = rearrange(x, 'b c ... -> b c 1 ...')+ video = rearrange(x, "b c ... -> b c 1 ...")video_contains_first_frame = Trueelse:video = x@@ -180,98 +162,88 @@ def main():video_enc_spatial = vae_2d.encode(video)recon_dec_spatial, posterior = vae(- video_enc_spatial,- video_contains_first_frame = video_contains_first_frame+ video_enc_spatial, video_contains_first_frame=video_contains_first_frame)recon_video = vae_2d.decode(recon_dec_spatial)recon_2d = vae_2d.decode(video_enc_spatial)else:- recon_video, posterior = vae(- video,- video_contains_first_frame = video_contains_first_frame- )+ recon_video, posterior = vae(video, video_contains_first_frame=video_contains_first_frame)if cfg.calc_loss:# ====== Calc Loss ======# simple nll loss- nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(- video,- recon_video,- posterior,- split = "eval"- )+ nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(video, recon_video, posterior, split="eval")- fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)fake_logits = discriminator(fake_video.contiguous())adversarial_loss = adversarial_loss_fn(fake_logits,- nll_loss,+ nll_loss,vae.get_last_layer(),- cfg.discriminator_start+1, # Hack to use discriminator- is_training = vae.training,+ cfg.discriminator_start + 1, # Hack to use discriminator+ is_training=vae.training,)-+vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss-+# ====== Discriminator Loss ======- real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)- fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)+ real_video = pad_at_dim(video, (disc_time_padding, 0), value=0.0, dim=2)+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:real_video = real_video.requires_grad_()- real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculation+ real_logits = discriminator(+ real_video.contiguous()+ ) # SCH: not detached for now for gradient_penalty calculationelse:- real_logits = discriminator(real_video.contiguous().detach())+ real_logits = discriminator(real_video.contiguous().detach())fake_logits = discriminator(fake_video.contiguous().detach())lecam_ema_real, lecam_ema_fake = lecam_ema.get()weighted_d_adversarial_loss, lecam_loss, gradient_penalty_loss = disc_loss_fn(- real_logits,- fake_logits,- cfg.discriminator_start+1, # Hack to use discriminator- lecam_ema_real = lecam_ema_real,- lecam_ema_fake = lecam_ema_fake,- real_video = real_video if cfg.gradient_penalty_loss_weight is not None else None,+ real_logits,+ fake_logits,+ cfg.discriminator_start + 1, # Hack to use discriminator+ lecam_ema_real=lecam_ema_real,+ lecam_ema_fake=lecam_ema_fake,+ real_video=real_video if cfg.gradient_penalty_loss_weight is not None else None,)disc_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty_lossloss_steps += 1- running_disc_loss = disc_loss.item()/loss_steps + running_disc_loss * ((loss_steps - 1) / loss_steps)- running_loss = vae_loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)+ running_disc_loss = disc_loss.item() / loss_steps + running_disc_loss * ((loss_steps - 1) / loss_steps)+ running_loss = vae_loss.item() / loss_steps + running_loss * ((loss_steps - 1) / loss_steps)running_nll = nll_loss.item() / loss_steps + running_nll * ((loss_steps - 1) / loss_steps)-# ===== Spatial VAE =====-if coordinator.is_master():if cfg.get("use_pipeline") == True:- for idx, (sample_original, sample_pipeline, sample_2d) in enumerate(zip(video, recon_video, recon_2d)):+ for idx, (sample_original, sample_pipeline, sample_2d) in enumerate(+ zip(video, recon_video, recon_2d)+ ):pos = step * cfg.batch_size + idxsave_path = os.path.join(save_dir, f"sample_{pos}")- save_sample(sample_original, fps=cfg.fps, save_path=save_path+"_original")- save_sample(sample_2d, fps=cfg.fps, save_path=save_path+"_2d")- save_sample(sample_pipeline, fps=cfg.fps, save_path=save_path+"_pipeline")+ save_sample(sample_original, fps=cfg.fps, save_path=save_path + "_original")+ save_sample(sample_2d, fps=cfg.fps, save_path=save_path + "_2d")+ save_sample(sample_pipeline, fps=cfg.fps, save_path=save_path + "_pipeline")else:for idx, (original, recon) in enumerate(zip(video, recon_video)):pos = step * cfg.batch_size + idxsave_path = os.path.join(save_dir, f"sample_{pos}")- save_sample(original, fps=cfg.fps, save_path=save_path+"_original")- save_sample(recon, fps=cfg.fps, save_path=save_path+"_recon")--+ save_sample(original, fps=cfg.fps, save_path=save_path + "_original")+ save_sample(recon, fps=cfg.fps, save_path=save_path + "_recon")if cfg.calc_loss:print("test vae loss:", running_loss)print("test nll loss:", running_nll)print("test disc loss:", running_disc_loss)-if __name__ == "__main__":main()