Open-Sora: Democratizing Efficient Video Production for All
Python
debug
Shen-Chenhuic6181ebdfdb3
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.pyindex d89e446..dd404de 100644--- a/configs/vae_3d/inference/16x256x256.py+++ b/configs/vae_3d/inference/16x256x256.py@@ -1,57 +1,8 @@-# num_frames = 16-# frame_interval = 3-# fps = 24 // 3-# image_size = (256, 256)-# is_vae = True--# # Define dataset-# root = None-# data_path = "CSV_PATH"-# use_image_transform = False-# num_workers = 4--# # Define acceleration-# dtype = "bf16"-# grad_checkpoint = True-# plugin = "zero2"-# sp_size = 1--# # Define Loss-# kl_weight = 0.000001-# perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper--# # Define model--# model = dict(-# type="VAE_3D_B",-# from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained model-# in_out_channels = 3,-# latent_embed_dim = 256,-# filters = 64,-# num_res_blocks = 2,-# channel_multipliers = (1, 2, 2, 4),-# temporal_downsample = (True, True, False),-# num_groups = 32, # for nn.GroupNorm-# conv_downsample = False,-# upsample = "nearest+conv", # options: "deconv", "nearest+conv"-# kl_embed_dim = 64,-# custom_conv_padding = None,-# activation_fn = 'swish',-# )---# # Others-# seed = 42-# save_dir = "outputs/samples"-# batch_size = 8--# grad_clip = 1.0--num_frames = 16frame_interval = 3-image_size = (256, 256)fps = 24 // 3+image_size = (256, 256)+is_vae = True# Define datasetroot = None@@ -73,6 +24,7 @@ perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVmodel = dict(type="VAE_3D_B",+ from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained modelin_out_channels = 3,latent_embed_dim = 256,filters = 64,@@ -87,23 +39,11 @@ model = dict(activation_fn = 'swish',)+# Othersseed = 42-outputs = "outputs"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 5-log_every = 1-ckpt_every = 1000-load = None-+save_dir = "outputs/samples"batch_size = 8-lr = 1e-4+grad_clip = 1.0+diff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.mdindex 2f251a0..05ecfb4 100644--- a/opensora/models/vae/README.md+++ b/opensora/models/vae/README.md@@ -18,8 +18,9 @@ CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripCUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/004-F16S3-VAE_3D_B/epoch0-global_step2000 --data-path /home/shenchenhui/data/pexels/debug.csv --save-dir outputs/pexel-# debug on the same 8 samples+# debug train and inference on the same few samplesWANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --wandb True+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --data-path /home/shenchenhui/data/pexels/debug.csv# resume training debug@@ -32,4 +33,4 @@ scp -P 31081 [email protected]:/home/shenchenhui/Open-Sora-dev/outputscp -P 31081 [email protected]:/home/shenchenhui/data/pexels/test.csv /Users/shenchenhui/Desktop-```\ No newline at end of file+```\ No newline at end of filediff --git a/scripts/inference-vae.py b/scripts/inference-vae.pyindex bc9f597..3a64b77 100644--- a/scripts/inference-vae.py+++ b/scripts/inference-vae.py@@ -20,10 +20,10 @@ from opensora.acceleration.parallel_states import (from tqdm import tqdmfrom opensora.models.vae.model_utils import VEA3DLoss-# DEBUG-from colossalai.booster import Booster-from colossalai.booster.plugin import LowLevelZeroPlugin-from opensora.acceleration.plugin import ZeroSeqParallelPlugin+# # DEBUG+# from colossalai.booster import Booster+# from colossalai.booster.plugin import LowLevelZeroPlugin+# from opensora.acceleration.plugin import ZeroSeqParallelPlugin@@ -55,29 +55,29 @@ def main():set_random_seed(seed=cfg.seed)- # 2.3 DEBUG: USE BOOSTER- # 2.3. initialize ColossalAI booster- if cfg.plugin == "zero2":- plugin = LowLevelZeroPlugin(- stage=2,- precision=cfg.dtype,- initial_scale=2**16,- max_norm=cfg.grad_clip,- )- set_data_parallel_group(dist.group.WORLD)- elif cfg.plugin == "zero2-seq":- plugin = ZeroSeqParallelPlugin(- sp_size=cfg.sp_size,- stage=2,- precision=cfg.dtype,- initial_scale=2**16,- max_norm=cfg.grad_clip,- )- set_sequence_parallel_group(plugin.sp_group)- set_data_parallel_group(plugin.dp_group)- else:- raise ValueError(f"Unknown plugin {cfg.plugin}")- booster = Booster(plugin=plugin)+ # # 2.3 DEBUG: USE BOOSTER+ # # 2.3. initialize ColossalAI booster+ # if cfg.plugin == "zero2":+ # plugin = LowLevelZeroPlugin(+ # stage=2,+ # precision=cfg.dtype,+ # initial_scale=2**16,+ # max_norm=cfg.grad_clip,+ # )+ # set_data_parallel_group(dist.group.WORLD)+ # elif cfg.plugin == "zero2-seq":+ # plugin = ZeroSeqParallelPlugin(+ # sp_size=cfg.sp_size,+ # stage=2,+ # precision=cfg.dtype,+ # initial_scale=2**16,+ # max_norm=cfg.grad_clip,+ # )+ # set_sequence_parallel_group(plugin.sp_group)+ # set_data_parallel_group(plugin.dp_group)+ # else:+ # raise ValueError(f"Unknown plugin {cfg.plugin}")+ # booster = Booster(plugin=plugin)# ======================================================