Open-Sora: Democratizing Efficient Video Production for All
Python
debug
Shen-Chenhui4b27448a49ea
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.pyindex 1728661..5571e9f 100644--- a/configs/vae_3d/inference/16x256x256.py+++ b/configs/vae_3d/inference/16x256x256.py@@ -18,7 +18,7 @@ sp_size = 1# Define Losskl_weight = 0.000001-perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGVIT paper+perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper# Define modeldiff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.mdindex 362dab4..6cdbe14 100644--- a/opensora/models/vae/README.md+++ b/opensora/models/vae/README.md@@ -11,7 +11,19 @@ WANDB_API_KEY=<wandb_api_key> CUDA_VISIBLE_DEVICES=<n> torchrun --master_port=<p### 2. Inference```yaml-CUDA_VISIBLE_DEVICES=7 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/028-F16S3-VAE_3D/epoch3-global_step20000/ --data-path /home/shenchenhui/data/pexels/test.csv+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/train_pexel_028/epoch3-global_step20000/ --data-path /home/shenchenhui/data/pexels/debug.csv++CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/004-F16S3-VAE_3D_B/epoch0-global_step1000 --data-path /home/shenchenhui/data/pexels/debug.csv++CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/004-F16S3-VAE_3D_B/epoch0-global_step2000 --data-path /home/shenchenhui/data/pexels/debug.csv+++# debug on the same 8 samples+WANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --wandb True+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --data-path /home/shenchenhui/data/pexels/debug.csv++# resume training debug+WANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --wandb True --load /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50``````yamldiff --git a/scripts/inference-vae.py b/scripts/inference-vae.pyindex b4c873e..365c4a2 100644--- a/scripts/inference-vae.py+++ b/scripts/inference-vae.py@@ -69,7 +69,7 @@ def main():dataset,batch_size=cfg.batch_size,num_workers=cfg.num_workers,- shuffle=True,+ shuffle=False,drop_last=True,pin_memory=True,process_group=get_data_parallel_group(),@@ -120,23 +120,24 @@ def main():dataloader_iter = iter(dataloader)with tqdm(- range(total_steps),- # desc=f"Avg Loss: {running_loss}",- disable=not coordinator.is_master(),- total=total_steps,- initial=0,- ) as pbar:- for step in pbar:- batch = next(dataloader_iter)- x = batch["video"].to(device, dtype) # [B, C, T, H, W]- reconstructions, posterior = vae(x)- loss = loss_function(x, reconstructions, posterior)- loss_steps += 1- running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)+ range(total_steps),+ # desc=f"Avg Loss: {running_loss}",+ disable=not coordinator.is_master(),+ total=total_steps,+ initial=0,+ ) as pbar:+ for step in pbar:+ batch = next(dataloader_iter)+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]+ reconstructions, posterior = vae(x)+ loss = loss_function(x, reconstructions, posterior)+ loss_steps += 1+ running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)if coordinator.is_master():for idx, sample in enumerate(reconstructions):- save_path = os.path.join(save_dir, f"sample_{idx}")+ pos = step * cfg.batch_size + idx+ save_path = os.path.join(save_dir, f"sample_{pos}")save_sample(sample, fps=cfg.fps, save_path=save_path)print("test loss:", running_loss)