Open-Sora: Democratizing Efficient Video Production for All
Python
update mask & config
Zangwei Zheng6762476261d0
diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.pyindex d45c2a5..669ce14 100644--- a/configs/opensora-v1-1/train/benchmark.py+++ b/configs/opensora-v1-1/train/benchmark.py@@ -3,12 +3,17 @@ dataset = dict(type="VariableVideoTextDataset",data_path=None,num_frames=None,- frame_interval=3,+ frame_interval=1,image_size=(None, None),transform_name="resize_crop",)bucket_config = {- "240p": {16: (1.0, 16)}, # 4.6s/it+ # "240p": {128: (1.0, 2)}, # 4.28s/it+ # "240p": {64: (1.0, 4)},+ # "240p": {32: (1.0, 8)}, # 4.6s/it+ # "240p": {16: (1.0, 16)}, # 4.6s/it+ # "480p": {16: (1.0, 4)}, # 4.6s/it+ "720p": {16: (1.0, 2)},# "256": {1: (1.0, 256)}, # 4.5s/it# "512": {1: (1.0, 96)}, # 4.7s/it# "512": {1: (1.0, 128)}, # 6.3s/it@@ -18,10 +23,10 @@ bucket_config = {# "1080p": {1: (1.0, 16)}, # 8.6s/it# "1080p": {1: (1.0, 8)}, # 4.4s/it}-mask_ratios = {- "mask_no": 0.0,- "mask_random": 1.0,-}+# mask_ratios = {+# "mask_no": 0.0,+# "mask_random": 1.0,+# }# Define accelerationnum_workers = 4diff --git a/configs/opensora-v1-1/train/video.py b/configs/opensora-v1-1/train/video.pyindex c79cf35..285ae84 100644--- a/configs/opensora-v1-1/train/video.py+++ b/configs/opensora-v1-1/train/video.py@@ -3,19 +3,26 @@ dataset = dict(type="VariableVideoTextDataset",data_path=None,num_frames=None,- frame_interval=3,+ frame_interval=1,image_size=(None, None),transform_name="resize_crop",)bucket_config = { # 6s/it- "240p": {16: (1.0, 16)},+ "240p": {16: (1.0, 16), 32: (1.0, 8), 64: (1.0, 4), 128: (1.0, 2)},"256": {1: (1.0, 256)},"512": {1: (1.0, 80)},- "480p": {1: (1.0, 52), 16: (1.0, 4)},- "720p": {16: (1.0, 2)},+ "480p": {1: (1.0, 52), 16: (0.5, 4), 32: (0.0, None)},+ "720p": {16: (1.0, 2), 32: (0.0, None)}, # No examples now"1024": {1: (1.0, 20)},"1080p": {1: (1.0, 8)},}+mask_ratios = {+ "mask_no": 0.9,+ "mask_random": 0.06,+ "mask_head": 0.01,+ "mask_tail": 0.01,+ "mask_head_tail": 0.02,+}# Define accelerationnum_workers = 4diff --git a/opensora/models/stdit/stdit2.py b/opensora/models/stdit/stdit2.pyindex e494209..506ef1e 100644--- a/opensora/models/stdit/stdit2.py+++ b/opensora/models/stdit/stdit2.py@@ -90,7 +90,7 @@ class STDiT2Block(nn.Module):# x: [B, (T, S), C]# mased_x: [B, (T, S), C]# x_mask: [B, T]- x = rearrange(x, "B (T S) C -> B T S C", T=self.T, S=self.S)+ x = rearrange(x, "B (T S) C -> B T S C", T=T, S=S)masked_x = rearrange(masked_x, "B (T S) C -> B T S C", T=T, S=S)x = torch.where(x_mask[:, :, None, None], x, masked_x)x = rearrange(x, "B T S C -> B (T S) C")