Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

update config

Zangwei Zhengb522fc89f1b3
diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.py
index d3b60cb..9df37e7 100644
--- a/configs/opensora-v1-1/train/benchmark.py
+++ b/configs/opensora-v1-1/train/benchmark.py
@@ -8,6 +8,7 @@ dataset = dict(
transform_name="resize_crop",
)
bucket_config = {
+ "240p": {16: (1.0, 16)},
# "256": {1: (1.0, 256)}, # 4.5s/it
# "512": {1: (1.0, 96)}, # 4.7s/it
# "512": {1: (1.0, 128)}, # 6.3s/it
@@ -15,7 +16,7 @@ bucket_config = {
# "1024": {1: (1.0, 32)}, # 6.8s/it
# "1024": {1: (1.0, 20)}, # 4.3s/it
# "1080p": {1: (1.0, 16)}, # 8.6s/it
- "1080p": {1: (1.0, 8)}, # 4.4s/it
+ # "1080p": {1: (1.0, 8)}, # 4.4s/it
}
# Define acceleration
@@ -41,7 +42,7 @@ vae = dict(
text_encoder = dict(
type="t5",
from_pretrained="DeepFloyd/t5-v1_1-xxl",
- model_max_length=120,
+ model_max_length=300,
shardformer=True,
)
scheduler = dict(
diff --git a/configs/opensora-v1-1/train/video.py b/configs/opensora-v1-1/train/video.py
new file mode 100644
index 0000000..c79cf35
--- /dev/null
+++ b/configs/opensora-v1-1/train/video.py
@@ -0,0 +1,63 @@
+# Define dataset
+dataset = dict(
+ type="VariableVideoTextDataset",
+ data_path=None,
+ num_frames=None,
+ frame_interval=3,
+ image_size=(None, None),
+ transform_name="resize_crop",
+)
+bucket_config = { # 6s/it
+ "240p": {16: (1.0, 16)},
+ "256": {1: (1.0, 256)},
+ "512": {1: (1.0, 80)},
+ "480p": {1: (1.0, 52), 16: (1.0, 4)},
+ "720p": {16: (1.0, 2)},
+ "1024": {1: (1.0, 20)},
+ "1080p": {1: (1.0, 8)},
+}
+
+# Define acceleration
+num_workers = 4
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+# Define model
+model = dict(
+ type="STDiT2-XL/2",
+ from_pretrained="PixArt-XL-2-1024-MS.pth",
+ input_sq_size=512, # pretrained model is trained on 512x512
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+)
+vae = dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="stabilityai/sd-vae-ft-ema",
+ micro_batch_size=4,
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+ shardformer=True,
+)
+scheduler = dict(
+ type="iddpm-speed",
+ timestep_respacing="",
+)
+
+# Others
+seed = 42
+outputs = "outputs"
+wandb = False
+
+epochs = 1000
+log_every = 10
+ckpt_every = 500
+load = None
+
+batch_size = 10 # only for logging
+lr = 2e-5
+grad_clip = 1.0