Open-Sora: Democratizing Efficient Video Production for All
Python
[exp] new config
Zangwei Zhengdce2ef4a1cd2
diff --git a/configs/opensora-v1-1/train/stage1.py b/configs/opensora-v1-1/train/stage1-1.pysimilarity index 99%rename from configs/opensora-v1-1/train/stage1.pyrename to configs/opensora-v1-1/train/stage1-1.pyindex bc55a0e..f9b1b60 100644--- a/configs/opensora-v1-1/train/stage1.py+++ b/configs/opensora-v1-1/train/stage1-1.py@@ -8,8 +8,8 @@ dataset = dict(transform_name="resize_crop",)bucket_config = { # 6s/it- "240p": {16: (1.0, 17), 32: (1.0, 9), 64: (1.0, 4), 128: (1.0, 2)},"256": {1: (1.0, 254)},+ "240p": {16: (1.0, 17), 32: (1.0, 9), 64: (1.0, 4), 128: (1.0, 2)},"512": {1: (0.5, 86)},"480p": {1: (0.4, 54), 16: (0.4, 4), 32: (0.0, None)},"720p": {16: (0.1, 2), 32: (0.0, None)},@@ -29,7 +29,7 @@ mask_ratios = {}# Define acceleration-num_workers = 4+num_workers = 8num_bucket_build_workers = 16dtype = "bf16"grad_checkpoint = Truediff --git a/configs/opensora-v1-1/train/stage1-2.py b/configs/opensora-v1-1/train/stage1-2.pyindex 2089420..6e6f7dc 100644--- a/configs/opensora-v1-1/train/stage1-2.py+++ b/configs/opensora-v1-1/train/stage1-2.py@@ -7,8 +7,13 @@ dataset = dict(image_size=(None, None),transform_name="resize_crop",)-bucket_config = { # 6s/it- "256": {1: (1.0, 254)},+# IMG: 1024 (20%) 512 (30%) 256 (50%) drop (50%)+bucket_config = { # 1s/it+ "144p": {16: (1.0, 8), 32: (1.0, 4), 64: (1.0, 2), 128: (1.0, 1)},+ "256": {1: (0.5, 32), 16: (0.5, 4), 32: (0.5, 2), 64: (0.5, 1), 128: (0.0, None)},+ "240p": {16: (0.3, 3), 32: (0.3, 1), 64: (0.0, None)},+ "512": {1: (0.4, 16)},+ "1024": {1: (0.3, 4)},}mask_ratios = {"mask_no": 0.9,@@ -23,10 +28,10 @@ mask_ratios = {}# Define acceleration-num_workers = 4+num_workers = 8num_bucket_build_workers = 16dtype = "bf16"-grad_checkpoint = True+grad_checkpoint = Falseplugin = "zero2"sp_size = 1diff --git a/opensora/datasets/aspect.py b/opensora/datasets/aspect.pyindex 7588cde..0a541a0 100644--- a/opensora/datasets/aspect.py+++ b/opensora/datasets/aspect.py@@ -164,6 +164,26 @@ ASPECT_RATIO_240P = {"2.08": (462, 222),}+# S = 36864+ASPECT_RATIO_144P = {+ "0.38": (118, 314),+ "0.43": (126, 294),+ "0.48": (134, 280),+ "0.50": (136, 272),+ "0.53": (140, 264),+ "0.54": (142, 262),+ "0.56": (144, 256), # base+ "0.62": (152, 244),+ "0.67": (156, 234),+ "0.75": (166, 222),+ "1.00": (192, 192),+ "1.33": (222, 166),+ "1.50": (236, 158),+ "1.78": (256, 144),+ "1.89": (264, 140),+ "2.00": (272, 136),+ "2.08": (278, 134),+}# from PixArt# S = 1048576@@ -306,6 +326,7 @@ def get_closest_ratio(height: float, width: float, ratios: dict):ASPECT_RATIOS = {+ "144p": (36864, ASPECT_RATIO_144P),"256": (65536, ASPECT_RATIO_256),"240p": (102240, ASPECT_RATIO_240P),"360p": (230400, ASPECT_RATIO_360P),diff --git a/opensora/datasets/bucket.py b/opensora/datasets/bucket.pyindex c60c6ff..01d9650 100644--- a/opensora/datasets/bucket.py+++ b/opensora/datasets/bucket.py@@ -1,8 +1,9 @@from collections import OrderedDict-from .aspect import ASPECT_RATIOS, get_closest_ratioimport numpy as np+from .aspect import ASPECT_RATIOS, get_closest_ratio+def find_approximate_hw(hw, hw_dict, approx=0.8):for k, v in hw_dict.items():@@ -80,9 +81,11 @@ class Bucket:# if sample is an imageif T == 1:if 1 in t_criteria:- fail = False- t_id = 1- break+ rng = np.random.default_rng(seed + self.bucket_id[hw_id][1])+ if rng.random() < t_criteria[1]:+ fail = False+ t_id = 1+ breakelse:continuediff --git a/opensora/datasets/sampler.py b/opensora/datasets/sampler.pyindex c0c0388..535b60d 100644--- a/opensora/datasets/sampler.py+++ b/opensora/datasets/sampler.py@@ -3,7 +3,6 @@ from collections import OrderedDict, defaultdictfrom pprint import pprintfrom typing import Iterator, List, Optional-import numpy as npimport torchimport torch.distributed as distfrom pandarallel import pandarallel@@ -53,7 +52,7 @@ class VariableVideoBatchSampler(DistributedSampler):def group_by_bucket(self) -> dict:bucket_sample_dict = OrderedDict()- pandarallel.initialize(progress_bar=True, nb_workers=self.num_bucket_build_workers)+ pandarallel.initialize(nb_workers=self.num_bucket_build_workers, progress_bar=False)bucket_ids = self.dataset.data.parallel_apply(apply,axis=1,diff --git a/scripts/inference-long.py b/scripts/inference-long.pyindex 33e3642..2860dee 100644--- a/scripts/inference-long.py+++ b/scripts/inference-long.py@@ -150,6 +150,8 @@ def main():width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)num_frames = torch.tensor([cfg.num_frames], device=device, dtype=dtype).repeat(cfg.batch_size)ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)+ if cfg.num_frames == 1:+ cfg.fps = IMG_FPSfps = torch.tensor([cfg.fps], device=device, dtype=dtype).repeat(cfg.batch_size)model_args["height"] = heightmodel_args["width"] = widthdiff --git a/scripts/inference.py b/scripts/inference.pyindex 7a66120..c59e560 100644--- a/scripts/inference.py+++ b/scripts/inference.py@@ -7,7 +7,7 @@ from colossalai.cluster import DistCoordinatorfrom mmengine.runner import set_random_seedfrom opensora.acceleration.parallel_states import set_sequence_parallel_group-from opensora.datasets import save_sample+from opensora.datasets import IMG_FPS, save_samplefrom opensora.models.text_encoder.t5 import text_preprocessingfrom opensora.registry import MODELS, SCHEDULERS, build_modulefrom opensora.utils.config_utils import parse_configs@@ -87,6 +87,8 @@ def main():width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)num_frames = torch.tensor([cfg.num_frames], device=device, dtype=dtype).repeat(cfg.batch_size)ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)+ if cfg.num_frames == 1:+ cfg.fps = IMG_FPSfps = torch.tensor([cfg.fps], device=device, dtype=dtype).repeat(cfg.batch_size)model_args["height"] = heightmodel_args["width"] = width