Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

debug

Shen-Chenhuic6181ebdfdb3
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.py
index d89e446..dd404de 100644
--- a/configs/vae_3d/inference/16x256x256.py
+++ b/configs/vae_3d/inference/16x256x256.py
@@ -1,57 +1,8 @@
-# num_frames = 16
-# frame_interval = 3
-# fps = 24 // 3
-# image_size = (256, 256)
-# is_vae = True
-
-# # Define dataset
-# root = None
-# data_path = "CSV_PATH"
-# use_image_transform = False
-# num_workers = 4
-
-# # Define acceleration
-# dtype = "bf16"
-# grad_checkpoint = True
-# plugin = "zero2"
-# sp_size = 1
-
-# # Define Loss
-# kl_weight = 0.000001
-# perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper
-
-# # Define model
-
-# model = dict(
-# type="VAE_3D_B",
-# from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained model
-# in_out_channels = 3,
-# latent_embed_dim = 256,
-# filters = 64,
-# num_res_blocks = 2,
-# channel_multipliers = (1, 2, 2, 4),
-# temporal_downsample = (True, True, False),
-# num_groups = 32, # for nn.GroupNorm
-# conv_downsample = False,
-# upsample = "nearest+conv", # options: "deconv", "nearest+conv"
-# kl_embed_dim = 64,
-# custom_conv_padding = None,
-# activation_fn = 'swish',
-# )
-
-
-# # Others
-# seed = 42
-# save_dir = "outputs/samples"
-# batch_size = 8
-
-# grad_clip = 1.0
-
-
num_frames = 16
frame_interval = 3
-image_size = (256, 256)
fps = 24 // 3
+image_size = (256, 256)
+is_vae = True
# Define dataset
root = None
@@ -73,6 +24,7 @@ perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGV
model = dict(
type="VAE_3D_B",
+ from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained model
in_out_channels = 3,
latent_embed_dim = 256,
filters = 64,
@@ -87,23 +39,11 @@ model = dict(
activation_fn = 'swish',
)
+
# Others
seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 5
-log_every = 1
-ckpt_every = 1000
-load = None
-
+save_dir = "outputs/samples"
batch_size = 8
-lr = 1e-4
+
grad_clip = 1.0
+
diff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.md
index 2f251a0..05ecfb4 100644
--- a/opensora/models/vae/README.md
+++ b/opensora/models/vae/README.md
@@ -18,8 +18,9 @@ CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scrip
CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/004-F16S3-VAE_3D_B/epoch0-global_step2000 --data-path /home/shenchenhui/data/pexels/debug.csv --save-dir outputs/pexel
-# debug on the same 8 samples
+# debug train and inference on the same few samples
WANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --wandb True
+
CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --data-path /home/shenchenhui/data/pexels/debug.csv
# resume training debug
@@ -32,4 +33,4 @@ scp -P 31081 [email protected]:/home/shenchenhui/Open-Sora-dev/output
scp -P 31081 [email protected]:/home/shenchenhui/data/pexels/test.csv /Users/shenchenhui/Desktop
-```
\ No newline at end of file
+```
\ No newline at end of file
diff --git a/scripts/inference-vae.py b/scripts/inference-vae.py
index bc9f597..3a64b77 100644
--- a/scripts/inference-vae.py
+++ b/scripts/inference-vae.py
@@ -20,10 +20,10 @@ from opensora.acceleration.parallel_states import (
from tqdm import tqdm
from opensora.models.vae.model_utils import VEA3DLoss
-# DEBUG
-from colossalai.booster import Booster
-from colossalai.booster.plugin import LowLevelZeroPlugin
-from opensora.acceleration.plugin import ZeroSeqParallelPlugin
+# # DEBUG
+# from colossalai.booster import Booster
+# from colossalai.booster.plugin import LowLevelZeroPlugin
+# from opensora.acceleration.plugin import ZeroSeqParallelPlugin
@@ -55,29 +55,29 @@ def main():
set_random_seed(seed=cfg.seed)
- # 2.3 DEBUG: USE BOOSTER
- # 2.3. initialize ColossalAI booster
- if cfg.plugin == "zero2":
- plugin = LowLevelZeroPlugin(
- stage=2,
- precision=cfg.dtype,
- initial_scale=2**16,
- max_norm=cfg.grad_clip,
- )
- set_data_parallel_group(dist.group.WORLD)
- elif cfg.plugin == "zero2-seq":
- plugin = ZeroSeqParallelPlugin(
- sp_size=cfg.sp_size,
- stage=2,
- precision=cfg.dtype,
- initial_scale=2**16,
- max_norm=cfg.grad_clip,
- )
- set_sequence_parallel_group(plugin.sp_group)
- set_data_parallel_group(plugin.dp_group)
- else:
- raise ValueError(f"Unknown plugin {cfg.plugin}")
- booster = Booster(plugin=plugin)
+ # # 2.3 DEBUG: USE BOOSTER
+ # # 2.3. initialize ColossalAI booster
+ # if cfg.plugin == "zero2":
+ # plugin = LowLevelZeroPlugin(
+ # stage=2,
+ # precision=cfg.dtype,
+ # initial_scale=2**16,
+ # max_norm=cfg.grad_clip,
+ # )
+ # set_data_parallel_group(dist.group.WORLD)
+ # elif cfg.plugin == "zero2-seq":
+ # plugin = ZeroSeqParallelPlugin(
+ # sp_size=cfg.sp_size,
+ # stage=2,
+ # precision=cfg.dtype,
+ # initial_scale=2**16,
+ # max_norm=cfg.grad_clip,
+ # )
+ # set_sequence_parallel_group(plugin.sp_group)
+ # set_data_parallel_group(plugin.dp_group)
+ # else:
+ # raise ValueError(f"Unknown plugin {cfg.plugin}")
+ # booster = Booster(plugin=plugin)
# ======================================================