Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

update training config

zhengzangwe58ddb0d9e1d
diff --git a/configs/opensora-v1-1/train/stage3-hq.py b/configs/opensora-v1-1/train/stage3-hq.py
deleted file mode 100644
index f1fe052..0000000
--- a/configs/opensora-v1-1/train/stage3-hq.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# Define dataset
-dataset = dict(
- type="VariableVideoTextDataset",
- data_path=None,
- num_frames=None,
- frame_interval=3,
- image_size=(None, None),
- transform_name="resize_crop",
-)
-bucket_config = { # 18s/it
- "144p": {1: (1.0, 200), 16: (1.0, 36), 32: (1.0, 18), 64: (1.0, 9), 128: (1.0, 4)},
- "256": {1: (0.8, 200), 16: (0.5, 22), 32: (0.5, 11), 64: (0.5, 6), 128: (0.8, 4)},
- "240p": {1: (0.8, 200), 16: (0.5, 22), 32: (0.5, 10), 64: (0.5, 6), 128: (0.5, 3)},
- "360p": {1: (0.5, 120), 16: (0.5, 9), 32: (0.5, 4), 64: (0.5, 2), 128: (0.5, 1)},
- "512": {1: (0.5, 120), 16: (0.5, 9), 32: (0.5, 4), 64: (0.5, 2), 128: (0.8, 1)},
- "480p": {1: (0.4, 80), 16: (0.6, 6), 32: (0.6, 3), 64: (0.6, 1), 128: (0.0, None)},
- "720p": {1: (0.4, 40), 16: (0.6, 3), 32: (0.6, 1), 96: (0.0, None)},
- "1024": {1: (0.3, 40)},
-}
-mask_ratios = {
- "mask_no": 0.75,
- "mask_quarter_random": 0.025,
- "mask_quarter_head": 0.025,
- "mask_quarter_tail": 0.025,
- "mask_quarter_head_tail": 0.05,
- "mask_image_random": 0.025,
- "mask_image_head": 0.025,
- "mask_image_tail": 0.025,
- "mask_image_head_tail": 0.05,
-}
-
-# Define acceleration
-num_workers = 8
-num_bucket_build_workers = 16
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-# Define model
-model = dict(
- type="STDiT2-XL/2",
- from_pretrained=None,
- input_sq_size=512, # pretrained model is trained on 512x512
- qk_norm=True,
- enable_flashattn=True,
- enable_layernorm_kernel=True,
-)
-vae = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
- micro_batch_size=4,
- local_files_only=True,
-)
-text_encoder = dict(
- type="t5",
- from_pretrained="DeepFloyd/t5-v1_1-xxl",
- model_max_length=200,
- shardformer=True,
- local_files_only=True,
-)
-scheduler = dict(
- type="iddpm",
- timestep_respacing="",
-)
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-epochs = 1000
-log_every = 10
-ckpt_every = 500
-load = None
-
-batch_size = None
-lr = 2e-5
-grad_clip = 1.0
diff --git a/configs/opensora-v1-1/train/stage3.py b/configs/opensora-v1-1/train/stage3.py
index d8455f7..8485762 100644
--- a/configs/opensora-v1-1/train/stage3.py
+++ b/configs/opensora-v1-1/train/stage3.py
@@ -7,16 +7,15 @@ dataset = dict(
image_size=(None, None),
transform_name="resize_crop",
)
-bucket_config = { # 18s/it
- "144p": {1: (1.0, 500), 16: (1.0, 36), 32: (1.0, 18), 64: (1.0, 9), 128: (1.0, 4)},
- "256": {1: (0.8, 300), 16: (0.5, 24), 32: (0.5, 12), 64: (0.5, 7), 128: (0.8, 4)},
- "240p": {1: (0.8, 300), 16: (0.5, 24), 32: (0.5, 12), 64: (0.5, 7), 128: (0.5, 4)},
- "360p": {1: (0.5, 150), 16: (0.5, 12), 32: (0.5, 6), 64: (0.5, 3), 128: (0.5, 1)},
- "512": {1: (0.5, 150), 16: (0.5, 12), 32: (0.5, 6), 64: (0.5, 3), 128: (0.8, 1)},
- "480p": {1: (0.4, 100), 16: (1.0, 8), 32: (1.0, 4), 64: (1.0, 2), 128: (1.0, 1)},
- "720p": {1: (0.4, 50), 16: (1.0, 4), 32: (1.0, 2), 64: (1.0, 1), 96: (0.0, None)},
- "1024": {1: (0.3, 50)},
- "1080p": {1: (0.3, 20)},
+bucket_config = { # 13s/it
+ "144p": {1: (1.0, 200), 16: (1.0, 36), 32: (1.0, 18), 64: (1.0, 9), 128: (1.0, 4)},
+ "256": {1: (0.8, 200), 16: (0.5, 22), 32: (0.5, 11), 64: (0.5, 6), 128: (0.8, 4)},
+ "240p": {1: (0.8, 200), 16: (0.5, 22), 32: (0.5, 10), 64: (0.5, 6), 128: (0.5, 3)},
+ "360p": {1: (0.5, 120), 16: (0.5, 9), 32: (0.5, 4), 64: (0.5, 2), 128: (0.5, 1)},
+ "512": {1: (0.5, 120), 16: (0.5, 9), 32: (0.5, 4), 64: (0.5, 2), 128: (0.8, 1)},
+ "480p": {1: (0.4, 80), 16: (0.6, 6), 32: (0.6, 3), 64: (0.6, 1), 128: (0.0, None)},
+ "720p": {1: (0.4, 40), 16: (0.6, 3), 32: (0.6, 1), 96: (0.0, None)},
+ "1024": {1: (0.3, 40)},
}
mask_ratios = {
"mask_no": 0.75,