Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[exp] update config bucket

Zangwei Zhengeefb351333dd
diff --git a/configs/opensora-v1-2/train/stage1-gc.py b/configs/opensora-v1-2/train/stage1-gc.py
index 6e31b0d..f7e7430 100644
--- a/configs/opensora-v1-2/train/stage1-gc.py
+++ b/configs/opensora-v1-2/train/stage1-gc.py
@@ -4,32 +4,32 @@ dataset = dict(
transform_name="resize_crop",
frame_interval=1,
)
-bucket_config = {
+bucket_config = { # 12s/it
"144p": {1: (1.0, 100)},
- # # ---
+ # ---
"256": {1: (0.5, 100)},
"240p": {1: (0.5, 100)},
- # # ---
+ # ---
"360p": {1: (0.5, 60)},
"512": {1: (0.5, 60)},
- # # ---
+ # ---
"480p": {1: (0.5, 40)},
- # # ---
+ # ---
"720p": {1: (0.2, 20)},
"1024": {1: (0.1, 20)},
- # # ---
+ # ---
"1080p": {1: (0.1, 10)},
# ---
"2048": {1: (0.1, 5)},
}
grad_checkpoint = True
+batch_size = None
# Acceleration settings
-num_workers = 4
+num_workers = 8
num_bucket_build_workers = 16
dtype = "bf16"
plugin = "zero2"
-sp_size = 1
# Model settings
model = dict(
@@ -79,7 +79,6 @@ ckpt_every = 500
# optimization settings
load = None
-batch_size = None
grad_clip = 1.0
lr = 1e-4
ema_decay = 0.99
diff --git a/configs/opensora-v1-2/train/stage1.py b/configs/opensora-v1-2/train/stage1.py
index 5d7ad54..e467240 100644
--- a/configs/opensora-v1-2/train/stage1.py
+++ b/configs/opensora-v1-2/train/stage1.py
@@ -4,32 +4,30 @@ dataset = dict(
transform_name="resize_crop",
frame_interval=1,
)
-bucket_config = {
- "144p": {1: (1.0, 64)},
+bucket_config = { # 2s/it
+ "144p": {1: (0.5, 48), 32: (1.0, 2), 48: (1.0, 4), 96: (1.0, 2), 192: (1.0, 1)},
# ---
- "256": {1: (0.5, 48), 48: (1.0, 2)},
- "240p": {1: (0.5, 48), 48: (0.5, 2)},
+ "256": {1: (0.6, 20), 32: (0.5, 2), 48: (0.5, 1), 64: (0.5, 1), 128: (0.0, None)},
+ "240p": {1: (0.6, 20), 32: (0.5, 2), 48: (0.5, 1), 64: (0.5, 1), 128: (0.0, None)},
# ---
- "360p": {1: (0.5, 18)},
- "512": {1: (0.5, 18)},
+ "360p": {1: (0.5, 8), 32: (0.2, 1), 96: (0.0, None)},
+ "512": {1: (0.5, 8), 32: (0.2, 1), 96: (0.0, None)},
# ---
- "480p": {1: (0.5, 8)},
+ "480p": {1: (0.2, 4), 16: (0.3, 1), 64: (0.0, None)},
# ---
- "720p": {1: (0.2, 4)},
- "1024": {1: (0.2, 4)},
+ "720p": {1: (0.1, 2)},
+ "1024": {1: (0.1, 2)},
# ---
- "1080p": {1: (0.3, 2)},
- # ---
- "2048": {1: (0.3, 1)},
+ "1080p": {1: (0.1, 1)},
}
-grad_checkpoint = False
+grad_checkpoint = False # determine batch size
+batch_size = None
# Acceleration settings
-num_workers = 4
+num_workers = 8
num_bucket_build_workers = 16
dtype = "bf16"
plugin = "zero2"
-sp_size = 1
# Model settings
model = dict(
@@ -38,6 +36,7 @@ model = dict(
qk_norm=True,
enable_flashattn=True,
enable_layernorm_kernel=True,
+ only_train_temporal=True,
)
vae = dict(
type="VideoAutoencoderPipeline",
@@ -69,6 +68,20 @@ scheduler = dict(
sample_method="logit-normal",
)
+# Mask settings
+# mask_ratios = {
+# "mask_random": 0.4,
+# "mask_intepolate": 0.01,
+# "mask_quarter_random": 0.01,
+# "mask_quarter_head": 0.01,
+# "mask_quarter_tail": 0.01,
+# "mask_quarter_head_tail": 0.01,
+# "mask_image_random": 0.01,
+# "mask_image_head": 0.01,
+# "mask_image_tail": 0.01,
+# "mask_image_head_tail": 0.01,
+# }
+
# Log settings
seed = 42
outputs = "outputs"
@@ -79,7 +92,6 @@ ckpt_every = 500
# optimization settings
load = None
-batch_size = None
grad_clip = 1.0
lr = 1e-4
ema_decay = 0.99
diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.py
index cd182ab..efafc37 100644
--- a/opensora/models/stdit/stdit3.py
+++ b/opensora/models/stdit/stdit3.py
@@ -166,6 +166,7 @@ class STDiT3(nn.Module):
enable_flashattn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
+ only_train_temporal=False,
):
super().__init__()
self.pred_sigma = pred_sigma
@@ -249,6 +250,12 @@ class STDiT3(nn.Module):
self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)
self.initialize_weights()
+ if only_train_temporal:
+ for param in self.parameters():
+ param.requires_grad = False
+ for block in self.temporal_blocks:
+ for param in block.parameters():
+ param.requires_grad = True
def initialize_weights(self):
# Initialize transformer layers:
diff --git a/scripts/misc/search_bs.py b/scripts/misc/search_bs.py
index 01d4d86..815f3b2 100644
--- a/scripts/misc/search_bs.py
+++ b/scripts/misc/search_bs.py
@@ -124,7 +124,7 @@ def main():
set_data_parallel_group(dist.group.WORLD)
elif cfg.plugin == "zero2-seq":
plugin = ZeroSeqParallelPlugin(
- sp_size=cfg.sp_size,
+ sp_size=cfg.get("sp_size", 1),
stage=2,
precision=cfg.dtype,
initial_scale=2**16,
diff --git a/scripts/train.py b/scripts/train.py
index 26f0964..f28802f 100644
--- a/scripts/train.py
+++ b/scripts/train.py
@@ -79,7 +79,7 @@ def main():
set_data_parallel_group(dist.group.WORLD)
elif cfg.plugin == "zero2-seq":
plugin = ZeroSeqParallelPlugin(
- sp_size=cfg.sp_size,
+ sp_size=cfg.get("sp_size", 1),
stage=2,
precision=cfg.dtype,
initial_scale=2**16,