Open-Sora: Democratizing Efficient Video Production for All
Python
update stage3-hq config
Zangwei Zheng5c485be65a75
diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.pyindex 7de7dab..5310b43 100644--- a/configs/opensora-v1-1/train/benchmark.py+++ b/configs/opensora-v1-1/train/benchmark.py@@ -82,7 +82,7 @@ text_encoder = dict(local_files_only=True,)scheduler = dict(- type="iddpm-speed",+ type="iddpm",timestep_respacing="",)diff --git a/configs/opensora-v1-1/train/image.py b/configs/opensora-v1-1/train/image.pyindex 74ab439..45748b7 100644--- a/configs/opensora-v1-1/train/image.py+++ b/configs/opensora-v1-1/train/image.py@@ -17,6 +17,7 @@ bucket_config = { # 6s/it# Define accelerationnum_workers = 4+num_bucket_build_workers = 16dtype = "bf16"grad_checkpoint = Trueplugin = "zero2"@@ -27,6 +28,7 @@ model = dict(type="STDiT2-XL/2",from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512+ qk_norm=True,enable_flashattn=True,enable_layernorm_kernel=True,)@@ -34,15 +36,17 @@ vae = dict(type="VideoAutoencoderKL",from_pretrained="stabilityai/sd-vae-ft-ema",micro_batch_size=4,+ local_files_only=True,)text_encoder = dict(type="t5",from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=200,shardformer=True,+ local_files_only=True,)scheduler = dict(- type="iddpm-speed",+ type="iddpm",timestep_respacing="",)diff --git a/configs/opensora-v1-1/train/stage3-hq.py b/configs/opensora-v1-1/train/stage3-hq.pynew file mode 100644index 0000000..470b2df--- /dev/null+++ b/configs/opensora-v1-1/train/stage3-hq.py@@ -0,0 +1,80 @@+# Define dataset+dataset = dict(+ type="VariableVideoTextDataset",+ data_path=None,+ num_frames=None,+ frame_interval=3,+ image_size=(None, None),+ transform_name="resize_crop",+)+bucket_config = { # 18s/it+ "144p": {1: (1.0, 500), 16: (1.0, 36), 32: (1.0, 18), 64: (1.0, 9), 128: (1.0, 4)},+ "256": {1: (0.8, 300), 16: (0.5, 24), 32: (0.5, 12), 64: (0.5, 7), 128: (0.8, 4)},+ "240p": {1: (0.8, 300), 16: (0.5, 24), 32: (0.5, 12), 64: (0.5, 7), 128: (0.5, 4)},+ "360p": {1: (0.5, 150), 16: (0.5, 12), 32: (0.5, 6), 64: (0.5, 3), 128: (0.5, 1)},+ "512": {1: (0.5, 150), 16: (0.5, 12), 32: (0.5, 6), 64: (0.5, 3), 128: (0.8, 1)},+ "480p": {1: (0.4, 100), 16: (0.6, 8), 32: (0.6, 4), 64: (0.6, 2), 128: (0.8, 1)},+ "720p": {1: (0.4, 50), 16: (0.6, 4), 32: (0.6, 2), 64: (0.6, 1), 96: (0.0, None)},+ "1024": {1: (0.3, 50)},+ "1080p": {1: (0.3, 20)},+}+mask_ratios = {+ "mask_no": 0.75,+ "mask_quarter_random": 0.025,+ "mask_quarter_head": 0.025,+ "mask_quarter_tail": 0.025,+ "mask_quarter_head_tail": 0.05,+ "mask_image_random": 0.025,+ "mask_image_head": 0.025,+ "mask_image_tail": 0.025,+ "mask_image_head_tail": 0.05,+}++# Define acceleration+num_workers = 8+num_bucket_build_workers = 16+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1++# Define model+model = dict(+ type="STDiT2-XL/2",+ from_pretrained=None,+ input_sq_size=512, # pretrained model is trained on 512x512+ qk_norm=True,+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+vae = dict(+ type="VideoAutoencoderKL",+ from_pretrained="stabilityai/sd-vae-ft-ema",+ micro_batch_size=4,+ local_files_only=True,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=200,+ shardformer=True,+ local_files_only=True,+)+scheduler = dict(+ type="iddpm",+ timestep_respacing="",+)++# Others+seed = 42+outputs = "outputs"+wandb = False++epochs = 1000+log_every = 10+ckpt_every = 500+load = None++batch_size = None+lr = 2e-5+grad_clip = 1.0diff --git a/configs/opensora-v1-1/train/video.py b/configs/opensora-v1-1/train/video.pyindex 20325ae..ef574f2 100644--- a/configs/opensora-v1-1/train/video.py+++ b/configs/opensora-v1-1/train/video.py@@ -16,16 +16,10 @@ bucket_config = { # 6s/it"1024": {1: (0.3, 20)},"1080p": {1: (0.3, 8)},}-# mask_ratios = {x-# "mask_no": 0.9,-# "mask_random": 0.06,-# "mask_head": 0.01,-# "mask_tail": 0.01,-# "mask_head_tail": 0.02,-# }# Define accelerationnum_workers = 4+num_bucket_build_workers = 16dtype = "bf16"grad_checkpoint = Trueplugin = "zero2"@@ -36,6 +30,7 @@ model = dict(type="STDiT2-XL/2",from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512+ qk_norm=True,enable_flashattn=True,enable_layernorm_kernel=True,)@@ -43,15 +38,17 @@ vae = dict(type="VideoAutoencoderKL",from_pretrained="stabilityai/sd-vae-ft-ema",micro_batch_size=4,+ local_files_only=True,)text_encoder = dict(type="t5",from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=200,shardformer=True,+ local_files_only=True,)scheduler = dict(- type="iddpm-speed",+ type="iddpm",timestep_respacing="",)