Open-Sora: Democratizing Efficient Video Production for All
Python
update config
Zangwei Zhengb522fc89f1b3
diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.pyindex d3b60cb..9df37e7 100644--- a/configs/opensora-v1-1/train/benchmark.py+++ b/configs/opensora-v1-1/train/benchmark.py@@ -8,6 +8,7 @@ dataset = dict(transform_name="resize_crop",)bucket_config = {+ "240p": {16: (1.0, 16)},# "256": {1: (1.0, 256)}, # 4.5s/it# "512": {1: (1.0, 96)}, # 4.7s/it# "512": {1: (1.0, 128)}, # 6.3s/it@@ -15,7 +16,7 @@ bucket_config = {# "1024": {1: (1.0, 32)}, # 6.8s/it# "1024": {1: (1.0, 20)}, # 4.3s/it# "1080p": {1: (1.0, 16)}, # 8.6s/it- "1080p": {1: (1.0, 8)}, # 4.4s/it+ # "1080p": {1: (1.0, 8)}, # 4.4s/it}# Define acceleration@@ -41,7 +42,7 @@ vae = dict(text_encoder = dict(type="t5",from_pretrained="DeepFloyd/t5-v1_1-xxl",- model_max_length=120,+ model_max_length=300,shardformer=True,)scheduler = dict(diff --git a/configs/opensora-v1-1/train/video.py b/configs/opensora-v1-1/train/video.pynew file mode 100644index 0000000..c79cf35--- /dev/null+++ b/configs/opensora-v1-1/train/video.py@@ -0,0 +1,63 @@+# Define dataset+dataset = dict(+ type="VariableVideoTextDataset",+ data_path=None,+ num_frames=None,+ frame_interval=3,+ image_size=(None, None),+ transform_name="resize_crop",+)+bucket_config = { # 6s/it+ "240p": {16: (1.0, 16)},+ "256": {1: (1.0, 256)},+ "512": {1: (1.0, 80)},+ "480p": {1: (1.0, 52), 16: (1.0, 4)},+ "720p": {16: (1.0, 2)},+ "1024": {1: (1.0, 20)},+ "1080p": {1: (1.0, 8)},+}++# Define acceleration+num_workers = 4+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1++# Define model+model = dict(+ type="STDiT2-XL/2",+ from_pretrained="PixArt-XL-2-1024-MS.pth",+ input_sq_size=512, # pretrained model is trained on 512x512+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+vae = dict(+ type="VideoAutoencoderKL",+ from_pretrained="stabilityai/sd-vae-ft-ema",+ micro_batch_size=4,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+ shardformer=True,+)+scheduler = dict(+ type="iddpm-speed",+ timestep_respacing="",+)++# Others+seed = 42+outputs = "outputs"+wandb = False++epochs = 1000+log_every = 10+ckpt_every = 500+load = None++batch_size = 10 # only for logging+lr = 2e-5+grad_clip = 1.0