Open-Sora: Democratizing Efficient Video Production for All
Python
pipeline config for trial data
Shen-Chenhui6ecc85835020
diff --git a/configs/vae_magvit_v2/inference/16x128x128.py b/configs/vae_magvit_v2/inference/16x128x128.pyindex 4fdfc4e..5d25983 100644--- a/configs/vae_magvit_v2/inference/16x128x128.py+++ b/configs/vae_magvit_v2/inference/16x128x128.py@@ -52,7 +52,7 @@ kl_loss_weight = 0.000001perceptual_loss_weight = 0.1 # use vgg is not None and more than 0discriminator_factor = 1.0 # for discriminator adversarial loss# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.5 # for generator adversarial loss+generator_factor = 0.1 # for generator adversarial losslecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weightdiscriminator_loss_type="non-saturating"generator_loss_type="non-saturating"diff --git a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py b/configs/vae_magvit_v2/inference/pipeline_16x128x128.pynew file mode 100644index 0000000..064da4a--- /dev/null+++ b/configs/vae_magvit_v2/inference/pipeline_16x128x128.py@@ -0,0 +1,94 @@+num_frames = 16+frame_interval = 3+image_size = (128, 128)+fps = 24 // 3+is_vae = True+use_pipeline = True++# Define dataset+root = None+data_path = "CSV_PATH"+use_image_transform = False+num_workers = 4+video_contains_first_frame = False++++# Define acceleration+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1+++# Define model++vae_2d = dict(+ type="VideoAutoencoderKL",+ from_pretrained="stabilityai/sd-vae-ft-ema",+)++model = dict(+ type="VAE_MAGVIT_V2",+ in_out_channels = 4,+ latent_embed_dim = 256,+ filters = 128,+ num_res_blocks = 4,+ channel_multipliers = (1, 2, 2, 4),+ temporal_downsample = (False, True, True),+ num_groups = 32, # for nn.GroupNorm+ kl_embed_dim = 64,+ activation_fn = 'swish',+ separate_first_frame_encoding = False,+ disable_space = True,+ custom_conv_padding = None+)+++discriminator = dict(+ type="DISCRIMINATOR_3D",+ image_size = image_size,+ num_frames = num_frames,+ in_channels = 4,+ filters = 128,+ channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution+)+++# loss weights+logvar_init=0.0+kl_loss_weight = 0.000001+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0+discriminator_factor = 1.0 # for discriminator adversarial loss+# discriminator_loss_weight = 0.5 # for generator adversarial loss+generator_factor = 0.1 # for generator adversarial loss+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight+discriminator_loss_type="non-saturating"+generator_loss_type="non-saturating"+discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use+ema_decay = 0.999 # ema decay factor for generator+++# Others+seed = 42+save_dir = "outputs/pipeline_samples_v2"+wandb = False++# Training+''' NOTE:+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],+3-6 epochs for pexel, from pexel observation its correct+'''++epochs = 10+log_every = 1+ckpt_every = 500+load = None++batch_size = 4+lr = 1e-4+grad_clip = 1.0++calc_loss = True\ No newline at end of filediff --git a/configs/vae_magvit_v2/train/pipeline_16x128x128.py b/configs/vae_magvit_v2/train/pipeline_16x128x128.pyindex 2751951..a2e7bf4 100644--- a/configs/vae_magvit_v2/train/pipeline_16x128x128.py+++ b/configs/vae_magvit_v2/train/pipeline_16x128x128.py@@ -1,6 +1,7 @@num_frames = 16frame_interval = 3image_size = (128, 128)+use_pipeline = True# Define datasetroot = None@@ -8,7 +9,6 @@ data_path = "CSV_PATH"use_image_transform = Falsenum_workers = 4video_contains_first_frame = False-use_pipeline = True# Define accelerationdtype = "bf16"diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae-v2.pyindex f44945a..4ccab91 100644--- a/scripts/inference-vae-v2.py+++ b/scripts/inference-vae-v2.py@@ -84,10 +84,15 @@ def main():# 4. build model & load weights# ======================================================# 3.1. build model+ if cfg.get("use_pipeline") == True:+ # use 2D VAE, then temporal VAE+ vae_2d = build_module(cfg.vae_2d, MODELS)vae = build_module(cfg.model, MODELS, device=device)discriminator = build_module(cfg.discriminator, MODELS, device=device)# 3.2. move to device & eval+ if cfg.get("use_pipeline") == True:+ vae_2d.to(device, dtype).eval()vae = vae.to(device, dtype).eval()discriminator = discriminator.to(device, dtype).eval()@@ -167,6 +172,11 @@ def main():else:video = x+ # ===== Spatial VAE =====+ if cfg.get("use_pipeline") == True:+ with torch.no_grad():+ video = vae_2d.encode(video)+recon_video, posterior = vae(video,video_contains_first_frame = video_contains_first_frame@@ -224,11 +234,34 @@ def main():running_disc_loss = disc_loss.item()/loss_steps + running_loss * ((loss_steps - 1) / loss_steps)running_loss = vae_loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)++ # ===== Spatial VAE =====+ if cfg.get("use_pipeline") == True:+ with torch.no_grad():+ recon_video_decode_spatial = vae_2d.decode(recon_video)+if coordinator.is_master():for idx, sample in enumerate(recon_video):pos = step * cfg.batch_size + idx- save_path = os.path.join(save_dir, f"sample_{pos}")+ if cfg.get("use_pipeline") == True:+ save_path = os.path.join(save_dir, f"sample_{pos}_time_decode")+ else:+ save_path = os.path.join(save_dir, f"sample_{pos}")save_sample(sample, fps=cfg.fps, save_path=save_path)++ if cfg.get("use_pipeline") == True:+ # store intermediate encoded video (spatial)+ for idx, sample in enumerate(video):+ pos = step * cfg.batch_size + idx+ save_path = os.path.join(save_dir, f"sample_{pos}_space_encode")+ save_sample(sample, fps=cfg.fps, save_path=save_path)+ # store final decoded video (decompressed spatially)+ for idx, sample in enumerate(recon_video_decode_spatial):+ pos = step * cfg.batch_size + idx+ save_path = os.path.join(save_dir, f"sample_{pos}_final")+ save_sample(sample, fps=cfg.fps, save_path=save_path)++if cfg.calc_loss:print("test vae loss:", running_loss)