Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

latest inference script

Shen-Chenhuiffc306c4fb84
diff --git a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py b/configs/vae_magvit_v2/inference/17x128x128_pixabay.py
index 49ac37c..9f6cd50 100644
--- a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py
+++ b/configs/vae_magvit_v2/inference/17x128x128_pixabay.py
@@ -48,16 +48,17 @@ model = dict(
separate_first_frame_encoding = False,
disable_space = True,
custom_conv_padding = None,
- encoder_double_z = False,
+ encoder_double_z = True,
)
discriminator = dict(
type="DISCRIMINATOR_3D",
- image_size = 16,
+ image_size = (128, 128),
num_frames = num_frames,
- in_channels = 4,
+ in_channels = 3,
filters = 128,
- channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
+ channel_multipliers = (2,4,4,4,4),
+ # channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
)
diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae-v2.py
index 3ffc3e4..ab43079 100644
--- a/scripts/inference-vae-v2.py
+++ b/scripts/inference-vae-v2.py
@@ -179,10 +179,14 @@ def main():
with torch.no_grad():
video_enc_spatial = vae_2d.encode(video)
- recon_video, posterior = vae(
+ recon_dec_spatial, posterior = vae(
video_enc_spatial,
video_contains_first_frame = video_contains_first_frame
)
+
+ recon_video = vae_2d.decode(recon_dec_spatial)
+ recon_2d = vae_2d.decode(video_enc_spatial)
+
else:
recon_video, posterior = vae(
video,
@@ -193,7 +197,7 @@ def main():
# ====== Calc Loss ======
# simple nll loss
nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(
- video_enc_spatial,
+ video,
recon_video,
posterior,
split = "eval"
@@ -212,7 +216,7 @@ def main():
vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss
# ====== Discriminator Loss ======
- real_video = pad_at_dim(video_enc_spatial, (disc_time_padding, 0), value = 0., dim = 2)
+ real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)
fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)
if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:
@@ -246,11 +250,7 @@ def main():
if coordinator.is_master():
if cfg.get("use_pipeline") == True:
- with torch.no_grad(): # 2nd stage decoding
- recon_pipeline = vae_2d.decode(recon_video)
- recon_2d = vae_2d.decode(video_enc_spatial)
-
- for idx, (sample_original, sample_pipeline, sample_2d) in enumerate(zip(video, recon_pipeline, recon_2d)):
+ for idx, (sample_original, sample_pipeline, sample_2d) in enumerate(zip(video, recon_video, recon_2d)):
pos = step * cfg.batch_size + idx
save_path = os.path.join(save_dir, f"sample_{pos}")
save_sample(sample_original, fps=cfg.fps, save_path=save_path+"_original")