Open-Sora: Democratizing Efficient Video Production for All
Python
inference config
Shen-Chenhui41a600b61b49
diff --git a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py b/configs/vae_magvit_v2/inference/pipeline_16x128x128.pyindex 064da4a..a67ea55 100644--- a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py+++ b/configs/vae_magvit_v2/inference/pipeline_16x128x128.py@@ -47,11 +47,11 @@ model = dict(discriminator = dict(type="DISCRIMINATOR_3D",- image_size = image_size,+ image_size = 16,num_frames = num_frames,in_channels = 4,filters = 128,- channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution+ channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution)diff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.mdindex 0238dc5..73099ac 100644--- a/opensora/models/vae/README.md+++ b/opensora/models/vae/README.md@@ -19,6 +19,15 @@ CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripCUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --load /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50```+version 2 pipeline+```yaml+# NOTE: first VAE is pretrained 2D, 16x128x128 --> 16x16x16+# then we train our own temporal VAE, 16x16x16 --> 4x16x16+# we use a 3 layer discriminator on the intermediate of 16x16x16+WANDB_API_KEY=<wandb_api_key> CUDA_VISIBLE_DEVICES=7 torchrun --master_port=29580 --nnodes=1 --nproc_per_node=1 scripts/train-vae-v2.py configs/vae_magvit_v2/train/pipeline_16x128x128.py --data-path /home/shenchenhui/data/trial_data/train_short.csv --wandb True+```++## 2. MAGVIT-v2### 2.1 dependenciesdiff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pyindex 8ff7045..d9900ef 100644--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@ -416,6 +416,7 @@ def main():pbar.set_postfix({"loss": avg_loss, "disc_loss": avg_disc_loss, "step": step, "global_step": global_step})running_loss = 0log_step = 0+ running_disc_loss = 0writer.add_scalar("loss", vae_loss.item(), global_step)if cfg.wandb:wandb.log(