Open-Sora: Democratizing Efficient Video Production for All
Python
better aspect ratio
Zangwei Zhengc553ee274f14
diff --git a/configs/opensora-v1-1/train/image.py b/configs/opensora-v1-1/train/image.pyindex 9b6b311..588f3c2 100644--- a/configs/opensora-v1-1/train/image.py+++ b/configs/opensora-v1-1/train/image.py@@ -8,9 +8,10 @@ dataset = dict(transform_name="resize_crop",)bucket_config = {- "240p": {1: (1.0, 128)},- "360p": {1: (1.0, 2)},- "720p": {1: (1.0, 2)},+ "256": {1: (1.0, 128)},+ "512": {1: (1.0, 2)},+ "480p": {1: (1.0, 2)},+ "1024": {1: (1.0, 2)},"1080p": {1: (1.0, 2)},}diff --git a/configs/opensora-v1-1/train/test.py b/configs/opensora-v1-1/train/test.pyindex 1877160..87f4d3b 100644--- a/configs/opensora-v1-1/train/test.py+++ b/configs/opensora-v1-1/train/test.py@@ -8,7 +8,7 @@ dataset = dict(transform_name="resize_crop",)bucket_config = {- "240p": {1: (1.0, 128)},+ "256": {1: (1.0, 64)}, # 32 ok, 64 broken}# Define accelerationdiff --git a/opensora/datasets/aspect.py b/opensora/datasets/aspect.pyindex f0ead5b..0d772c9 100644--- a/opensora/datasets/aspect.py+++ b/opensora/datasets/aspect.py@@ -11,69 +11,94 @@ def get_h_w(a, ts):return h, w-def get_aspect_ratios_dict(- ts=360 * 640,- ars=[0.39, 0.42, 0.48, 0.50, 0.52, 0.56, 0.66, 0.75, 1.00, 1.33, 1.52, 1.78, 1.92, 2.00, 2.10],-):+AR = [+ 3 / 8,+ 9 / 21,+ 0.48,+ 1 / 2,+ 9 / 17,+ 1 / 1.85,+ 9 / 16,+ 5 / 8,+ 2 / 3,+ 3 / 4,+ 1 / 1,+ 4 / 3,+ 3 / 2,+ 16 / 9,+ 17 / 9,+ 2 / 1,+ 1 / 0.48,+]+ARV = [0.375, 0.43, 0.48, 0.50, 0.53, 0.54, 0.56, 0.62, 0.67, 0.75, 1.0, 1.33, 1.50, 1.78, 1.89, 2.0, 2.08]+++def get_aspect_ratios_dict(ts=360 * 640, ars=AR):est = {f"{a:.2f}": get_h_w(a, ts) for a in ars}return est# S = 8294400ASPECT_RATIO_4K = {- "0.39": (1798, 4610),- "0.42": (1866, 4442),+ "0.38": (1764, 4704),+ "0.43": (1886, 4400),"0.48": (1996, 4158),"0.50": (2036, 4072),- "0.52": (2076, 3992),- "0.56": (2160, 3840),- "0.66": (2340, 3546),+ "0.53": (2096, 3960),+ "0.54": (2118, 3918),+ "0.62": (2276, 3642),+ "0.56": (2160, 3840), # base+ "0.67": (2352, 3528),"0.75": (2494, 3326),"1.00": (2880, 2880),- "1.33": (3322, 2498),- "1.52": (3550, 2336),- "1.78": (3842, 2158),- "1.92": (3990, 2078),+ "1.33": (3326, 2494),+ "1.50": (3528, 2352),+ "1.78": (3840, 2160),+ "1.89": (3958, 2096),"2.00": (4072, 2036),- "2.10": (4174, 1988),+ "2.08": (4156, 1994),}# S = 2073600ASPECT_RATIO_1080P = {- "0.39": (900, 2308),- "0.42": (934, 2224),+ "0.38": (882, 2352),+ "0.43": (942, 2198),"0.48": (998, 2080),"0.50": (1018, 2036),- "0.52": (1038, 1996),+ "0.53": (1048, 1980),+ "0.54": (1058, 1958),"0.56": (1080, 1920), # base- "0.66": (1170, 1772),+ "0.62": (1138, 1820),+ "0.67": (1176, 1764),"0.75": (1248, 1664),"1.00": (1440, 1440),- "1.33": (1660, 1248),- "1.52": (1776, 1168),- "1.78": (1922, 1080),- "1.92": (1996, 1040),+ "1.33": (1662, 1246),+ "1.50": (1764, 1176),+ "1.78": (1920, 1080),+ "1.89": (1980, 1048),"2.00": (2036, 1018),- "2.10": (2086, 994),+ "2.08": (2078, 998),}# S = 921600ASPECT_RATIO_720P = {- "0.39": (600, 1538),- "0.42": (622, 1480),+ "0.38": (588, 1568),+ "0.43": (628, 1466),"0.48": (666, 1388),"0.50": (678, 1356),- "0.52": (692, 1330),+ "0.53": (698, 1318),+ "0.54": (706, 1306),"0.56": (720, 1280), # base- "0.66": (780, 1182),+ "0.62": (758, 1212),+ "0.67": (784, 1176),"0.75": (832, 1110),"1.00": (960, 960),- "1.33": (1108, 834),- "1.52": (1184, 778),+ "1.33": (1108, 831),+ "1.50": (1176, 784),"1.78": (1280, 720),- "1.92": (1330, 692),+ "1.89": (1320, 698),"2.00": (1358, 679),- "2.10": (1392, 662),+ "2.08": (1386, 666),}# S = 409920@@ -97,174 +122,178 @@ ASPECT_RATIO_480P = {# S = 230400ASPECT_RATIO_360P = {- "0.39": (300, 770),- "0.42": (312, 742),- "0.48": (332, 692),- "0.50": (340, 680),- "0.52": (346, 666),- "0.56": (360, 640), # base- "0.66": (390, 590),- "0.75": (416, 554),- "1.00": (480, 480),- "1.33": (554, 416),- "1.52": (592, 390),- "1.78": (640, 360),- "1.92": (666, 346),- "2.00": (678, 339),- "2.10": (696, 332),+ "0.38": (392, 1046),+ "0.43": (420, 980),+ "0.48": (444, 925),+ "0.50": (452, 904),+ "0.53": (466, 880),+ "0.54": (470, 870),+ "0.56": (480, 854), # base+ "0.62": (506, 810),+ "0.67": (522, 783),+ "0.75": (554, 738),+ "1.00": (640, 640),+ "1.33": (740, 555),+ "1.50": (784, 522),+ "1.78": (854, 480),+ "1.89": (880, 466),+ "2.00": (906, 453),+ "2.08": (924, 444),}# S = 102240ASPECT_RATIO_240P = {- "0.39": (200, 512),- "0.42": (208, 496),+ "0.38": (196, 522),+ "0.43": (210, 490),"0.48": (222, 462),"0.50": (226, 452),- "0.52": (230, 442),- "0.56": (240, 426),- "0.66": (260, 394),+ "0.53": (232, 438),+ "0.54": (236, 436),+ "0.56": (240, 426), # base+ "0.62": (252, 404),+ "0.67": (262, 393),"0.75": (276, 368),"1.00": (320, 320),- "1.33": (368, 276),- "1.52": (394, 260),+ "1.33": (370, 278),+ "1.50": (392, 262),"1.78": (426, 240),- "1.92": (444, 232),+ "1.89": (440, 232),"2.00": (452, 226),- "2.10": (464, 220),+ "2.08": (462, 222),}# from PixArt# S = 1048576ASPECT_RATIO_1024 = {- "0.25": [512.0, 2048.0],- "0.26": [512.0, 1984.0],- "0.27": [512.0, 1920.0],- "0.28": [512.0, 1856.0],- "0.32": [576.0, 1792.0],- "0.33": [576.0, 1728.0],- "0.35": [576.0, 1664.0],- "0.4": [640.0, 1600.0],- "0.42": [640.0, 1536.0],- "0.48": [704.0, 1472.0],- "0.5": [704.0, 1408.0],- "0.52": [704.0, 1344.0],- "0.57": [768.0, 1344.0],- "0.6": [768.0, 1280.0],- "0.68": [832.0, 1216.0],- "0.72": [832.0, 1152.0],- "0.78": [896.0, 1152.0],- "0.82": [896.0, 1088.0],- "0.88": [960.0, 1088.0],- "0.94": [960.0, 1024.0],- "1.0": [1024.0, 1024.0],- "1.07": [1024.0, 960.0],- "1.13": [1088.0, 960.0],- "1.21": [1088.0, 896.0],- "1.29": [1152.0, 896.0],- "1.38": [1152.0, 832.0],- "1.46": [1216.0, 832.0],- "1.67": [1280.0, 768.0],- "1.75": [1344.0, 768.0],- "2.0": [1408.0, 704.0],- "2.09": [1472.0, 704.0],- "2.4": [1536.0, 640.0],- "2.5": [1600.0, 640.0],- "2.89": [1664.0, 576.0],- "3.0": [1728.0, 576.0],- "3.11": [1792.0, 576.0],- "3.62": [1856.0, 512.0],- "3.75": [1920.0, 512.0],- "3.88": [1984.0, 512.0],- "4.0": [2048.0, 512.0],+ "0.25": (512, 2048),+ "0.26": (512, 1984),+ "0.27": (512, 1920),+ "0.28": (512, 1856),+ "0.32": (576, 1792),+ "0.33": (576, 1728),+ "0.35": (576, 1664),+ "0.4": (640, 1600),+ "0.42": (640, 1536),+ "0.48": (704, 1472),+ "0.5": (704, 1408),+ "0.52": (704, 1344),+ "0.57": (768, 1344),+ "0.6": (768, 1280),+ "0.68": (832, 1216),+ "0.72": (832, 1152),+ "0.78": (896, 1152),+ "0.82": (896, 1088),+ "0.88": (960, 1088),+ "0.94": (960, 1024),+ "1.0": (1024, 1024),+ "1.07": (1024, 960),+ "1.13": (1088, 960),+ "1.21": (1088, 896),+ "1.29": (1152, 896),+ "1.38": (1152, 832),+ "1.46": (1216, 832),+ "1.67": (1280, 768),+ "1.75": (1344, 768),+ "2.0": (1408, 704),+ "2.09": (1472, 704),+ "2.4": (1536, 640),+ "2.5": (1600, 640),+ "2.89": (1664, 576),+ "3.0": (1728, 576),+ "3.11": (1792, 576),+ "3.62": (1856, 512),+ "3.75": (1920, 512),+ "3.88": (1984, 512),+ "4.0": (2048, 512),}# S = 262144ASPECT_RATIO_512 = {- "0.25": [256.0, 1024.0],- "0.26": [256.0, 992.0],- "0.27": [256.0, 960.0],- "0.28": [256.0, 928.0],- "0.32": [288.0, 896.0],- "0.33": [288.0, 864.0],- "0.35": [288.0, 832.0],- "0.4": [320.0, 800.0],- "0.42": [320.0, 768.0],- "0.48": [352.0, 736.0],- "0.5": [352.0, 704.0],- "0.52": [352.0, 672.0],- "0.57": [384.0, 672.0],- "0.6": [384.0, 640.0],- "0.68": [416.0, 608.0],- "0.72": [416.0, 576.0],- "0.78": [448.0, 576.0],- "0.82": [448.0, 544.0],- "0.88": [480.0, 544.0],- "0.94": [480.0, 512.0],- "1.0": [512.0, 512.0],- "1.07": [512.0, 480.0],- "1.13": [544.0, 480.0],- "1.21": [544.0, 448.0],- "1.29": [576.0, 448.0],- "1.38": [576.0, 416.0],- "1.46": [608.0, 416.0],- "1.67": [640.0, 384.0],- "1.75": [672.0, 384.0],- "2.0": [704.0, 352.0],- "2.09": [736.0, 352.0],- "2.4": [768.0, 320.0],- "2.5": [800.0, 320.0],- "2.89": [832.0, 288.0],- "3.0": [864.0, 288.0],- "3.11": [896.0, 288.0],- "3.62": [928.0, 256.0],- "3.75": [960.0, 256.0],- "3.88": [992.0, 256.0],- "4.0": [1024.0, 256.0],+ "0.25": (256, 1024),+ "0.26": (256, 992),+ "0.27": (256, 960),+ "0.28": (256, 928),+ "0.32": (288, 896),+ "0.33": (288, 864),+ "0.35": (288, 832),+ "0.4": (320, 800),+ "0.42": (320, 768),+ "0.48": (352, 736),+ "0.5": (352, 704),+ "0.52": (352, 672),+ "0.57": (384, 672),+ "0.6": (384, 640),+ "0.68": (416, 608),+ "0.72": (416, 576),+ "0.78": (448, 576),+ "0.82": (448, 544),+ "0.88": (480, 544),+ "0.94": (480, 512),+ "1.0": (512, 512),+ "1.07": (512, 480),+ "1.13": (544, 480),+ "1.21": (544, 448),+ "1.29": (576, 448),+ "1.38": (576, 416),+ "1.46": (608, 416),+ "1.67": (640, 384),+ "1.75": (672, 384),+ "2.0": (704, 352),+ "2.09": (736, 352),+ "2.4": (768, 320),+ "2.5": (800, 320),+ "2.89": (832, 288),+ "3.0": (864, 288),+ "3.11": (896, 288),+ "3.62": (928, 256),+ "3.75": (960, 256),+ "3.88": (992, 256),+ "4.0": (1024, 256),}# S = 65536ASPECT_RATIO_256 = {- "0.25": [128.0, 512.0],- "0.26": [128.0, 496.0],- "0.27": [128.0, 480.0],- "0.28": [128.0, 464.0],- "0.32": [144.0, 448.0],- "0.33": [144.0, 432.0],- "0.35": [144.0, 416.0],- "0.4": [160.0, 400.0],- "0.42": [160.0, 384.0],- "0.48": [176.0, 368.0],- "0.5": [176.0, 352.0],- "0.52": [176.0, 336.0],- "0.57": [192.0, 336.0],- "0.6": [192.0, 320.0],- "0.68": [208.0, 304.0],- "0.72": [208.0, 288.0],- "0.78": [224.0, 288.0],- "0.82": [224.0, 272.0],- "0.88": [240.0, 272.0],- "0.94": [240.0, 256.0],- "1.0": [256.0, 256.0],- "1.07": [256.0, 240.0],- "1.13": [272.0, 240.0],- "1.21": [272.0, 224.0],- "1.29": [288.0, 224.0],- "1.38": [288.0, 208.0],- "1.46": [304.0, 208.0],- "1.67": [320.0, 192.0],- "1.75": [336.0, 192.0],- "2.0": [352.0, 176.0],- "2.09": [368.0, 176.0],- "2.4": [384.0, 160.0],- "2.5": [400.0, 160.0],- "2.89": [416.0, 144.0],- "3.0": [432.0, 144.0],- "3.11": [448.0, 144.0],- "3.62": [464.0, 128.0],- "3.75": [480.0, 128.0],- "3.88": [496.0, 128.0],- "4.0": [512.0, 128.0],+ "0.25": (128, 512),+ "0.26": (128, 496),+ "0.27": (128, 480),+ "0.28": (128, 464),+ "0.32": (144, 448),+ "0.33": (144, 432),+ "0.35": (144, 416),+ "0.4": (160, 400),+ "0.42": (160, 384),+ "0.48": (176, 368),+ "0.5": (176, 352),+ "0.52": (176, 336),+ "0.57": (192, 336),+ "0.6": (192, 320),+ "0.68": (208, 304),+ "0.72": (208, 288),+ "0.78": (224, 288),+ "0.82": (224, 272),+ "0.88": (240, 272),+ "0.94": (240, 256),+ "1.0": (256, 256),+ "1.07": (256, 240),+ "1.13": (272, 240),+ "1.21": (272, 224),+ "1.29": (288, 224),+ "1.38": (288, 208),+ "1.46": (304, 208),+ "1.67": (320, 192),+ "1.75": (336, 192),+ "2.0": (352, 176),+ "2.09": (368, 176),+ "2.4": (384, 160),+ "2.5": (400, 160),+ "2.89": (416, 144),+ "3.0": (432, 144),+ "3.11": (448, 144),+ "3.62": (464, 128),+ "3.75": (480, 128),+ "3.88": (496, 128),+ "4.0": (512, 128),}diff --git a/tools/caption/README.md b/tools/caption/README.mdindex 8af5920..a980b29 100644--- a/tools/caption/README.md+++ b/tools/caption/README.md@@ -77,7 +77,8 @@ Then use the output csv file to resume the process.Run the following command to generate captions for videos with GPT-4V:```bash-python -m tools.caption.caption_gpt4 FOLDER_WITH_VIDEOS output.csv --key $OPENAI_API_KEY+# output: DATA_caption.csv+python -m tools.caption.caption_gpt4 DATA.csv --key $OPENAI_API_KEY```-The cost is approximately $0.01 per video (3 frames per video). The output is a CSV file with path and caption.+The cost is approximately $0.01 per video (3 frames per video).