Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

debug

Shen-Chenhui4b27448a49ea
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.py
index 1728661..5571e9f 100644
--- a/configs/vae_3d/inference/16x256x256.py
+++ b/configs/vae_3d/inference/16x256x256.py
@@ -18,7 +18,7 @@ sp_size = 1
# Define Loss
kl_weight = 0.000001
-perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGVIT paper
+perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper
# Define model
diff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.md
index 362dab4..6cdbe14 100644
--- a/opensora/models/vae/README.md
+++ b/opensora/models/vae/README.md
@@ -11,7 +11,19 @@ WANDB_API_KEY=<wandb_api_key> CUDA_VISIBLE_DEVICES=<n> torchrun --master_port=<p
### 2. Inference
```yaml
-CUDA_VISIBLE_DEVICES=7 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/028-F16S3-VAE_3D/epoch3-global_step20000/ --data-path /home/shenchenhui/data/pexels/test.csv
+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/train_pexel_028/epoch3-global_step20000/ --data-path /home/shenchenhui/data/pexels/debug.csv
+
+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/004-F16S3-VAE_3D_B/epoch0-global_step1000 --data-path /home/shenchenhui/data/pexels/debug.csv
+
+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/004-F16S3-VAE_3D_B/epoch0-global_step2000 --data-path /home/shenchenhui/data/pexels/debug.csv
+
+
+# debug on the same 8 samples
+WANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --wandb True
+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --data-path /home/shenchenhui/data/pexels/debug.csv
+
+# resume training debug
+WANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --wandb True --load /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50
```
```yaml
diff --git a/scripts/inference-vae.py b/scripts/inference-vae.py
index b4c873e..365c4a2 100644
--- a/scripts/inference-vae.py
+++ b/scripts/inference-vae.py
@@ -69,7 +69,7 @@ def main():
dataset,
batch_size=cfg.batch_size,
num_workers=cfg.num_workers,
- shuffle=True,
+ shuffle=False,
drop_last=True,
pin_memory=True,
process_group=get_data_parallel_group(),
@@ -120,23 +120,24 @@ def main():
dataloader_iter = iter(dataloader)
with tqdm(
- range(total_steps),
- # desc=f"Avg Loss: {running_loss}",
- disable=not coordinator.is_master(),
- total=total_steps,
- initial=0,
- ) as pbar:
- for step in pbar:
- batch = next(dataloader_iter)
- x = batch["video"].to(device, dtype) # [B, C, T, H, W]
- reconstructions, posterior = vae(x)
- loss = loss_function(x, reconstructions, posterior)
- loss_steps += 1
- running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
+ range(total_steps),
+ # desc=f"Avg Loss: {running_loss}",
+ disable=not coordinator.is_master(),
+ total=total_steps,
+ initial=0,
+ ) as pbar:
+ for step in pbar:
+ batch = next(dataloader_iter)
+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]
+ reconstructions, posterior = vae(x)
+ loss = loss_function(x, reconstructions, posterior)
+ loss_steps += 1
+ running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
if coordinator.is_master():
for idx, sample in enumerate(reconstructions):
- save_path = os.path.join(save_dir, f"sample_{idx}")
+ pos = step * cfg.batch_size + idx
+ save_path = os.path.join(save_dir, f"sample_{pos}")
save_sample(sample, fps=cfg.fps, save_path=save_path)
print("test loss:", running_loss)