Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

inference config

Shen-Chenhui41a600b61b49
diff --git a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py b/configs/vae_magvit_v2/inference/pipeline_16x128x128.py
index 064da4a..a67ea55 100644
--- a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py
+++ b/configs/vae_magvit_v2/inference/pipeline_16x128x128.py
@@ -47,11 +47,11 @@ model = dict(
discriminator = dict(
type="DISCRIMINATOR_3D",
- image_size = image_size,
+ image_size = 16,
num_frames = num_frames,
in_channels = 4,
filters = 128,
- channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
+ channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
)
diff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.md
index 0238dc5..73099ac 100644
--- a/opensora/models/vae/README.md
+++ b/opensora/models/vae/README.md
@@ -19,6 +19,15 @@ CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scrip
CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --load /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50
```
+version 2 pipeline
+```yaml
+# NOTE: first VAE is pretrained 2D, 16x128x128 --> 16x16x16
+# then we train our own temporal VAE, 16x16x16 --> 4x16x16
+# we use a 3 layer discriminator on the intermediate of 16x16x16
+WANDB_API_KEY=<wandb_api_key> CUDA_VISIBLE_DEVICES=7 torchrun --master_port=29580 --nnodes=1 --nproc_per_node=1 scripts/train-vae-v2.py configs/vae_magvit_v2/train/pipeline_16x128x128.py --data-path /home/shenchenhui/data/trial_data/train_short.csv --wandb True
+```
+
+
## 2. MAGVIT-v2
### 2.1 dependencies
diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.py
index 8ff7045..d9900ef 100644
--- a/scripts/train-vae-v2.py
+++ b/scripts/train-vae-v2.py
@@ -416,6 +416,7 @@ def main():
pbar.set_postfix({"loss": avg_loss, "disc_loss": avg_disc_loss, "step": step, "global_step": global_step})
running_loss = 0
log_step = 0
+ running_disc_loss = 0
writer.add_scalar("loss", vae_loss.item(), global_step)
if cfg.wandb:
wandb.log(