Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

better aspect ratio

Zangwei Zhengc553ee274f14
diff --git a/configs/opensora-v1-1/train/image.py b/configs/opensora-v1-1/train/image.py
index 9b6b311..588f3c2 100644
--- a/configs/opensora-v1-1/train/image.py
+++ b/configs/opensora-v1-1/train/image.py
@@ -8,9 +8,10 @@ dataset = dict(
transform_name="resize_crop",
)
bucket_config = {
- "240p": {1: (1.0, 128)},
- "360p": {1: (1.0, 2)},
- "720p": {1: (1.0, 2)},
+ "256": {1: (1.0, 128)},
+ "512": {1: (1.0, 2)},
+ "480p": {1: (1.0, 2)},
+ "1024": {1: (1.0, 2)},
"1080p": {1: (1.0, 2)},
}
diff --git a/configs/opensora-v1-1/train/test.py b/configs/opensora-v1-1/train/test.py
index 1877160..87f4d3b 100644
--- a/configs/opensora-v1-1/train/test.py
+++ b/configs/opensora-v1-1/train/test.py
@@ -8,7 +8,7 @@ dataset = dict(
transform_name="resize_crop",
)
bucket_config = {
- "240p": {1: (1.0, 128)},
+ "256": {1: (1.0, 64)}, # 32 ok, 64 broken
}
# Define acceleration
diff --git a/opensora/datasets/aspect.py b/opensora/datasets/aspect.py
index f0ead5b..0d772c9 100644
--- a/opensora/datasets/aspect.py
+++ b/opensora/datasets/aspect.py
@@ -11,69 +11,94 @@ def get_h_w(a, ts):
return h, w
-def get_aspect_ratios_dict(
- ts=360 * 640,
- ars=[0.39, 0.42, 0.48, 0.50, 0.52, 0.56, 0.66, 0.75, 1.00, 1.33, 1.52, 1.78, 1.92, 2.00, 2.10],
-):
+AR = [
+ 3 / 8,
+ 9 / 21,
+ 0.48,
+ 1 / 2,
+ 9 / 17,
+ 1 / 1.85,
+ 9 / 16,
+ 5 / 8,
+ 2 / 3,
+ 3 / 4,
+ 1 / 1,
+ 4 / 3,
+ 3 / 2,
+ 16 / 9,
+ 17 / 9,
+ 2 / 1,
+ 1 / 0.48,
+]
+ARV = [0.375, 0.43, 0.48, 0.50, 0.53, 0.54, 0.56, 0.62, 0.67, 0.75, 1.0, 1.33, 1.50, 1.78, 1.89, 2.0, 2.08]
+
+
+def get_aspect_ratios_dict(ts=360 * 640, ars=AR):
est = {f"{a:.2f}": get_h_w(a, ts) for a in ars}
return est
# S = 8294400
ASPECT_RATIO_4K = {
- "0.39": (1798, 4610),
- "0.42": (1866, 4442),
+ "0.38": (1764, 4704),
+ "0.43": (1886, 4400),
"0.48": (1996, 4158),
"0.50": (2036, 4072),
- "0.52": (2076, 3992),
- "0.56": (2160, 3840),
- "0.66": (2340, 3546),
+ "0.53": (2096, 3960),
+ "0.54": (2118, 3918),
+ "0.62": (2276, 3642),
+ "0.56": (2160, 3840), # base
+ "0.67": (2352, 3528),
"0.75": (2494, 3326),
"1.00": (2880, 2880),
- "1.33": (3322, 2498),
- "1.52": (3550, 2336),
- "1.78": (3842, 2158),
- "1.92": (3990, 2078),
+ "1.33": (3326, 2494),
+ "1.50": (3528, 2352),
+ "1.78": (3840, 2160),
+ "1.89": (3958, 2096),
"2.00": (4072, 2036),
- "2.10": (4174, 1988),
+ "2.08": (4156, 1994),
}
# S = 2073600
ASPECT_RATIO_1080P = {
- "0.39": (900, 2308),
- "0.42": (934, 2224),
+ "0.38": (882, 2352),
+ "0.43": (942, 2198),
"0.48": (998, 2080),
"0.50": (1018, 2036),
- "0.52": (1038, 1996),
+ "0.53": (1048, 1980),
+ "0.54": (1058, 1958),
"0.56": (1080, 1920), # base
- "0.66": (1170, 1772),
+ "0.62": (1138, 1820),
+ "0.67": (1176, 1764),
"0.75": (1248, 1664),
"1.00": (1440, 1440),
- "1.33": (1660, 1248),
- "1.52": (1776, 1168),
- "1.78": (1922, 1080),
- "1.92": (1996, 1040),
+ "1.33": (1662, 1246),
+ "1.50": (1764, 1176),
+ "1.78": (1920, 1080),
+ "1.89": (1980, 1048),
"2.00": (2036, 1018),
- "2.10": (2086, 994),
+ "2.08": (2078, 998),
}
# S = 921600
ASPECT_RATIO_720P = {
- "0.39": (600, 1538),
- "0.42": (622, 1480),
+ "0.38": (588, 1568),
+ "0.43": (628, 1466),
"0.48": (666, 1388),
"0.50": (678, 1356),
- "0.52": (692, 1330),
+ "0.53": (698, 1318),
+ "0.54": (706, 1306),
"0.56": (720, 1280), # base
- "0.66": (780, 1182),
+ "0.62": (758, 1212),
+ "0.67": (784, 1176),
"0.75": (832, 1110),
"1.00": (960, 960),
- "1.33": (1108, 834),
- "1.52": (1184, 778),
+ "1.33": (1108, 831),
+ "1.50": (1176, 784),
"1.78": (1280, 720),
- "1.92": (1330, 692),
+ "1.89": (1320, 698),
"2.00": (1358, 679),
- "2.10": (1392, 662),
+ "2.08": (1386, 666),
}
# S = 409920
@@ -97,174 +122,178 @@ ASPECT_RATIO_480P = {
# S = 230400
ASPECT_RATIO_360P = {
- "0.39": (300, 770),
- "0.42": (312, 742),
- "0.48": (332, 692),
- "0.50": (340, 680),
- "0.52": (346, 666),
- "0.56": (360, 640), # base
- "0.66": (390, 590),
- "0.75": (416, 554),
- "1.00": (480, 480),
- "1.33": (554, 416),
- "1.52": (592, 390),
- "1.78": (640, 360),
- "1.92": (666, 346),
- "2.00": (678, 339),
- "2.10": (696, 332),
+ "0.38": (392, 1046),
+ "0.43": (420, 980),
+ "0.48": (444, 925),
+ "0.50": (452, 904),
+ "0.53": (466, 880),
+ "0.54": (470, 870),
+ "0.56": (480, 854), # base
+ "0.62": (506, 810),
+ "0.67": (522, 783),
+ "0.75": (554, 738),
+ "1.00": (640, 640),
+ "1.33": (740, 555),
+ "1.50": (784, 522),
+ "1.78": (854, 480),
+ "1.89": (880, 466),
+ "2.00": (906, 453),
+ "2.08": (924, 444),
}
# S = 102240
ASPECT_RATIO_240P = {
- "0.39": (200, 512),
- "0.42": (208, 496),
+ "0.38": (196, 522),
+ "0.43": (210, 490),
"0.48": (222, 462),
"0.50": (226, 452),
- "0.52": (230, 442),
- "0.56": (240, 426),
- "0.66": (260, 394),
+ "0.53": (232, 438),
+ "0.54": (236, 436),
+ "0.56": (240, 426), # base
+ "0.62": (252, 404),
+ "0.67": (262, 393),
"0.75": (276, 368),
"1.00": (320, 320),
- "1.33": (368, 276),
- "1.52": (394, 260),
+ "1.33": (370, 278),
+ "1.50": (392, 262),
"1.78": (426, 240),
- "1.92": (444, 232),
+ "1.89": (440, 232),
"2.00": (452, 226),
- "2.10": (464, 220),
+ "2.08": (462, 222),
}
# from PixArt
# S = 1048576
ASPECT_RATIO_1024 = {
- "0.25": [512.0, 2048.0],
- "0.26": [512.0, 1984.0],
- "0.27": [512.0, 1920.0],
- "0.28": [512.0, 1856.0],
- "0.32": [576.0, 1792.0],
- "0.33": [576.0, 1728.0],
- "0.35": [576.0, 1664.0],
- "0.4": [640.0, 1600.0],
- "0.42": [640.0, 1536.0],
- "0.48": [704.0, 1472.0],
- "0.5": [704.0, 1408.0],
- "0.52": [704.0, 1344.0],
- "0.57": [768.0, 1344.0],
- "0.6": [768.0, 1280.0],
- "0.68": [832.0, 1216.0],
- "0.72": [832.0, 1152.0],
- "0.78": [896.0, 1152.0],
- "0.82": [896.0, 1088.0],
- "0.88": [960.0, 1088.0],
- "0.94": [960.0, 1024.0],
- "1.0": [1024.0, 1024.0],
- "1.07": [1024.0, 960.0],
- "1.13": [1088.0, 960.0],
- "1.21": [1088.0, 896.0],
- "1.29": [1152.0, 896.0],
- "1.38": [1152.0, 832.0],
- "1.46": [1216.0, 832.0],
- "1.67": [1280.0, 768.0],
- "1.75": [1344.0, 768.0],
- "2.0": [1408.0, 704.0],
- "2.09": [1472.0, 704.0],
- "2.4": [1536.0, 640.0],
- "2.5": [1600.0, 640.0],
- "2.89": [1664.0, 576.0],
- "3.0": [1728.0, 576.0],
- "3.11": [1792.0, 576.0],
- "3.62": [1856.0, 512.0],
- "3.75": [1920.0, 512.0],
- "3.88": [1984.0, 512.0],
- "4.0": [2048.0, 512.0],
+ "0.25": (512, 2048),
+ "0.26": (512, 1984),
+ "0.27": (512, 1920),
+ "0.28": (512, 1856),
+ "0.32": (576, 1792),
+ "0.33": (576, 1728),
+ "0.35": (576, 1664),
+ "0.4": (640, 1600),
+ "0.42": (640, 1536),
+ "0.48": (704, 1472),
+ "0.5": (704, 1408),
+ "0.52": (704, 1344),
+ "0.57": (768, 1344),
+ "0.6": (768, 1280),
+ "0.68": (832, 1216),
+ "0.72": (832, 1152),
+ "0.78": (896, 1152),
+ "0.82": (896, 1088),
+ "0.88": (960, 1088),
+ "0.94": (960, 1024),
+ "1.0": (1024, 1024),
+ "1.07": (1024, 960),
+ "1.13": (1088, 960),
+ "1.21": (1088, 896),
+ "1.29": (1152, 896),
+ "1.38": (1152, 832),
+ "1.46": (1216, 832),
+ "1.67": (1280, 768),
+ "1.75": (1344, 768),
+ "2.0": (1408, 704),
+ "2.09": (1472, 704),
+ "2.4": (1536, 640),
+ "2.5": (1600, 640),
+ "2.89": (1664, 576),
+ "3.0": (1728, 576),
+ "3.11": (1792, 576),
+ "3.62": (1856, 512),
+ "3.75": (1920, 512),
+ "3.88": (1984, 512),
+ "4.0": (2048, 512),
}
# S = 262144
ASPECT_RATIO_512 = {
- "0.25": [256.0, 1024.0],
- "0.26": [256.0, 992.0],
- "0.27": [256.0, 960.0],
- "0.28": [256.0, 928.0],
- "0.32": [288.0, 896.0],
- "0.33": [288.0, 864.0],
- "0.35": [288.0, 832.0],
- "0.4": [320.0, 800.0],
- "0.42": [320.0, 768.0],
- "0.48": [352.0, 736.0],
- "0.5": [352.0, 704.0],
- "0.52": [352.0, 672.0],
- "0.57": [384.0, 672.0],
- "0.6": [384.0, 640.0],
- "0.68": [416.0, 608.0],
- "0.72": [416.0, 576.0],
- "0.78": [448.0, 576.0],
- "0.82": [448.0, 544.0],
- "0.88": [480.0, 544.0],
- "0.94": [480.0, 512.0],
- "1.0": [512.0, 512.0],
- "1.07": [512.0, 480.0],
- "1.13": [544.0, 480.0],
- "1.21": [544.0, 448.0],
- "1.29": [576.0, 448.0],
- "1.38": [576.0, 416.0],
- "1.46": [608.0, 416.0],
- "1.67": [640.0, 384.0],
- "1.75": [672.0, 384.0],
- "2.0": [704.0, 352.0],
- "2.09": [736.0, 352.0],
- "2.4": [768.0, 320.0],
- "2.5": [800.0, 320.0],
- "2.89": [832.0, 288.0],
- "3.0": [864.0, 288.0],
- "3.11": [896.0, 288.0],
- "3.62": [928.0, 256.0],
- "3.75": [960.0, 256.0],
- "3.88": [992.0, 256.0],
- "4.0": [1024.0, 256.0],
+ "0.25": (256, 1024),
+ "0.26": (256, 992),
+ "0.27": (256, 960),
+ "0.28": (256, 928),
+ "0.32": (288, 896),
+ "0.33": (288, 864),
+ "0.35": (288, 832),
+ "0.4": (320, 800),
+ "0.42": (320, 768),
+ "0.48": (352, 736),
+ "0.5": (352, 704),
+ "0.52": (352, 672),
+ "0.57": (384, 672),
+ "0.6": (384, 640),
+ "0.68": (416, 608),
+ "0.72": (416, 576),
+ "0.78": (448, 576),
+ "0.82": (448, 544),
+ "0.88": (480, 544),
+ "0.94": (480, 512),
+ "1.0": (512, 512),
+ "1.07": (512, 480),
+ "1.13": (544, 480),
+ "1.21": (544, 448),
+ "1.29": (576, 448),
+ "1.38": (576, 416),
+ "1.46": (608, 416),
+ "1.67": (640, 384),
+ "1.75": (672, 384),
+ "2.0": (704, 352),
+ "2.09": (736, 352),
+ "2.4": (768, 320),
+ "2.5": (800, 320),
+ "2.89": (832, 288),
+ "3.0": (864, 288),
+ "3.11": (896, 288),
+ "3.62": (928, 256),
+ "3.75": (960, 256),
+ "3.88": (992, 256),
+ "4.0": (1024, 256),
}
# S = 65536
ASPECT_RATIO_256 = {
- "0.25": [128.0, 512.0],
- "0.26": [128.0, 496.0],
- "0.27": [128.0, 480.0],
- "0.28": [128.0, 464.0],
- "0.32": [144.0, 448.0],
- "0.33": [144.0, 432.0],
- "0.35": [144.0, 416.0],
- "0.4": [160.0, 400.0],
- "0.42": [160.0, 384.0],
- "0.48": [176.0, 368.0],
- "0.5": [176.0, 352.0],
- "0.52": [176.0, 336.0],
- "0.57": [192.0, 336.0],
- "0.6": [192.0, 320.0],
- "0.68": [208.0, 304.0],
- "0.72": [208.0, 288.0],
- "0.78": [224.0, 288.0],
- "0.82": [224.0, 272.0],
- "0.88": [240.0, 272.0],
- "0.94": [240.0, 256.0],
- "1.0": [256.0, 256.0],
- "1.07": [256.0, 240.0],
- "1.13": [272.0, 240.0],
- "1.21": [272.0, 224.0],
- "1.29": [288.0, 224.0],
- "1.38": [288.0, 208.0],
- "1.46": [304.0, 208.0],
- "1.67": [320.0, 192.0],
- "1.75": [336.0, 192.0],
- "2.0": [352.0, 176.0],
- "2.09": [368.0, 176.0],
- "2.4": [384.0, 160.0],
- "2.5": [400.0, 160.0],
- "2.89": [416.0, 144.0],
- "3.0": [432.0, 144.0],
- "3.11": [448.0, 144.0],
- "3.62": [464.0, 128.0],
- "3.75": [480.0, 128.0],
- "3.88": [496.0, 128.0],
- "4.0": [512.0, 128.0],
+ "0.25": (128, 512),
+ "0.26": (128, 496),
+ "0.27": (128, 480),
+ "0.28": (128, 464),
+ "0.32": (144, 448),
+ "0.33": (144, 432),
+ "0.35": (144, 416),
+ "0.4": (160, 400),
+ "0.42": (160, 384),
+ "0.48": (176, 368),
+ "0.5": (176, 352),
+ "0.52": (176, 336),
+ "0.57": (192, 336),
+ "0.6": (192, 320),
+ "0.68": (208, 304),
+ "0.72": (208, 288),
+ "0.78": (224, 288),
+ "0.82": (224, 272),
+ "0.88": (240, 272),
+ "0.94": (240, 256),
+ "1.0": (256, 256),
+ "1.07": (256, 240),
+ "1.13": (272, 240),
+ "1.21": (272, 224),
+ "1.29": (288, 224),
+ "1.38": (288, 208),
+ "1.46": (304, 208),
+ "1.67": (320, 192),
+ "1.75": (336, 192),
+ "2.0": (352, 176),
+ "2.09": (368, 176),
+ "2.4": (384, 160),
+ "2.5": (400, 160),
+ "2.89": (416, 144),
+ "3.0": (432, 144),
+ "3.11": (448, 144),
+ "3.62": (464, 128),
+ "3.75": (480, 128),
+ "3.88": (496, 128),
+ "4.0": (512, 128),
}
diff --git a/tools/caption/README.md b/tools/caption/README.md
index 8af5920..a980b29 100644
--- a/tools/caption/README.md
+++ b/tools/caption/README.md
@@ -77,7 +77,8 @@ Then use the output csv file to resume the process.
Run the following command to generate captions for videos with GPT-4V:
```bash
-python -m tools.caption.caption_gpt4 FOLDER_WITH_VIDEOS output.csv --key $OPENAI_API_KEY
+# output: DATA_caption.csv
+python -m tools.caption.caption_gpt4 DATA.csv --key $OPENAI_API_KEY
```
-The cost is approximately $0.01 per video (3 frames per video). The output is a CSV file with path and caption.
+The cost is approximately $0.01 per video (3 frames per video).