Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

pipeline config for trial data

Shen-Chenhui6ecc85835020
diff --git a/configs/vae_magvit_v2/inference/16x128x128.py b/configs/vae_magvit_v2/inference/16x128x128.py
index 4fdfc4e..5d25983 100644
--- a/configs/vae_magvit_v2/inference/16x128x128.py
+++ b/configs/vae_magvit_v2/inference/16x128x128.py
@@ -52,7 +52,7 @@ kl_loss_weight = 0.000001
perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
discriminator_factor = 1.0 # for discriminator adversarial loss
# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.5 # for generator adversarial loss
+generator_factor = 0.1 # for generator adversarial loss
lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
discriminator_loss_type="non-saturating"
generator_loss_type="non-saturating"
diff --git a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py b/configs/vae_magvit_v2/inference/pipeline_16x128x128.py
new file mode 100644
index 0000000..064da4a
--- /dev/null
+++ b/configs/vae_magvit_v2/inference/pipeline_16x128x128.py
@@ -0,0 +1,94 @@
+num_frames = 16
+frame_interval = 3
+image_size = (128, 128)
+fps = 24 // 3
+is_vae = True
+use_pipeline = True
+
+# Define dataset
+root = None
+data_path = "CSV_PATH"
+use_image_transform = False
+num_workers = 4
+video_contains_first_frame = False
+
+
+
+# Define acceleration
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+
+# Define model
+
+vae_2d = dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="stabilityai/sd-vae-ft-ema",
+)
+
+model = dict(
+ type="VAE_MAGVIT_V2",
+ in_out_channels = 4,
+ latent_embed_dim = 256,
+ filters = 128,
+ num_res_blocks = 4,
+ channel_multipliers = (1, 2, 2, 4),
+ temporal_downsample = (False, True, True),
+ num_groups = 32, # for nn.GroupNorm
+ kl_embed_dim = 64,
+ activation_fn = 'swish',
+ separate_first_frame_encoding = False,
+ disable_space = True,
+ custom_conv_padding = None
+)
+
+
+discriminator = dict(
+ type="DISCRIMINATOR_3D",
+ image_size = image_size,
+ num_frames = num_frames,
+ in_channels = 4,
+ filters = 128,
+ channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
+)
+
+
+# loss weights
+logvar_init=0.0
+kl_loss_weight = 0.000001
+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
+discriminator_factor = 1.0 # for discriminator adversarial loss
+# discriminator_loss_weight = 0.5 # for generator adversarial loss
+generator_factor = 0.1 # for generator adversarial loss
+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
+discriminator_loss_type="non-saturating"
+generator_loss_type="non-saturating"
+discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now
+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
+ema_decay = 0.999 # ema decay factor for generator
+
+
+# Others
+seed = 42
+save_dir = "outputs/pipeline_samples_v2"
+wandb = False
+
+# Training
+''' NOTE:
+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
+3-6 epochs for pexel, from pexel observation its correct
+'''
+
+epochs = 10
+log_every = 1
+ckpt_every = 500
+load = None
+
+batch_size = 4
+lr = 1e-4
+grad_clip = 1.0
+
+calc_loss = True
\ No newline at end of file
diff --git a/configs/vae_magvit_v2/train/pipeline_16x128x128.py b/configs/vae_magvit_v2/train/pipeline_16x128x128.py
index 2751951..a2e7bf4 100644
--- a/configs/vae_magvit_v2/train/pipeline_16x128x128.py
+++ b/configs/vae_magvit_v2/train/pipeline_16x128x128.py
@@ -1,6 +1,7 @@
num_frames = 16
frame_interval = 3
image_size = (128, 128)
+use_pipeline = True
# Define dataset
root = None
@@ -8,7 +9,6 @@ data_path = "CSV_PATH"
use_image_transform = False
num_workers = 4
video_contains_first_frame = False
-use_pipeline = True
# Define acceleration
dtype = "bf16"
diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae-v2.py
index f44945a..4ccab91 100644
--- a/scripts/inference-vae-v2.py
+++ b/scripts/inference-vae-v2.py
@@ -84,10 +84,15 @@ def main():
# 4. build model & load weights
# ======================================================
# 3.1. build model
+ if cfg.get("use_pipeline") == True:
+ # use 2D VAE, then temporal VAE
+ vae_2d = build_module(cfg.vae_2d, MODELS)
vae = build_module(cfg.model, MODELS, device=device)
discriminator = build_module(cfg.discriminator, MODELS, device=device)
# 3.2. move to device & eval
+ if cfg.get("use_pipeline") == True:
+ vae_2d.to(device, dtype).eval()
vae = vae.to(device, dtype).eval()
discriminator = discriminator.to(device, dtype).eval()
@@ -167,6 +172,11 @@ def main():
else:
video = x
+ # ===== Spatial VAE =====
+ if cfg.get("use_pipeline") == True:
+ with torch.no_grad():
+ video = vae_2d.encode(video)
+
recon_video, posterior = vae(
video,
video_contains_first_frame = video_contains_first_frame
@@ -224,11 +234,34 @@ def main():
running_disc_loss = disc_loss.item()/loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
running_loss = vae_loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
+
+ # ===== Spatial VAE =====
+ if cfg.get("use_pipeline") == True:
+ with torch.no_grad():
+ recon_video_decode_spatial = vae_2d.decode(recon_video)
+
if coordinator.is_master():
for idx, sample in enumerate(recon_video):
pos = step * cfg.batch_size + idx
- save_path = os.path.join(save_dir, f"sample_{pos}")
+ if cfg.get("use_pipeline") == True:
+ save_path = os.path.join(save_dir, f"sample_{pos}_time_decode")
+ else:
+ save_path = os.path.join(save_dir, f"sample_{pos}")
save_sample(sample, fps=cfg.fps, save_path=save_path)
+
+ if cfg.get("use_pipeline") == True:
+ # store intermediate encoded video (spatial)
+ for idx, sample in enumerate(video):
+ pos = step * cfg.batch_size + idx
+ save_path = os.path.join(save_dir, f"sample_{pos}_space_encode")
+ save_sample(sample, fps=cfg.fps, save_path=save_path)
+ # store final decoded video (decompressed spatially)
+ for idx, sample in enumerate(recon_video_decode_spatial):
+ pos = step * cfg.batch_size + idx
+ save_path = os.path.join(save_dir, f"sample_{pos}_final")
+ save_sample(sample, fps=cfg.fps, save_path=save_path)
+
+
if cfg.calc_loss:
print("test vae loss:", running_loss)