Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[exp] new config

Zangwei Zhengdce2ef4a1cd2
diff --git a/configs/opensora-v1-1/train/stage1.py b/configs/opensora-v1-1/train/stage1-1.py
similarity index 99%
rename from configs/opensora-v1-1/train/stage1.py
rename to configs/opensora-v1-1/train/stage1-1.py
index bc55a0e..f9b1b60 100644
--- a/configs/opensora-v1-1/train/stage1.py
+++ b/configs/opensora-v1-1/train/stage1-1.py
@@ -8,8 +8,8 @@ dataset = dict(
transform_name="resize_crop",
)
bucket_config = { # 6s/it
- "240p": {16: (1.0, 17), 32: (1.0, 9), 64: (1.0, 4), 128: (1.0, 2)},
"256": {1: (1.0, 254)},
+ "240p": {16: (1.0, 17), 32: (1.0, 9), 64: (1.0, 4), 128: (1.0, 2)},
"512": {1: (0.5, 86)},
"480p": {1: (0.4, 54), 16: (0.4, 4), 32: (0.0, None)},
"720p": {16: (0.1, 2), 32: (0.0, None)},
@@ -29,7 +29,7 @@ mask_ratios = {
}
# Define acceleration
-num_workers = 4
+num_workers = 8
num_bucket_build_workers = 16
dtype = "bf16"
grad_checkpoint = True
diff --git a/configs/opensora-v1-1/train/stage1-2.py b/configs/opensora-v1-1/train/stage1-2.py
index 2089420..6e6f7dc 100644
--- a/configs/opensora-v1-1/train/stage1-2.py
+++ b/configs/opensora-v1-1/train/stage1-2.py
@@ -7,8 +7,13 @@ dataset = dict(
image_size=(None, None),
transform_name="resize_crop",
)
-bucket_config = { # 6s/it
- "256": {1: (1.0, 254)},
+# IMG: 1024 (20%) 512 (30%) 256 (50%) drop (50%)
+bucket_config = { # 1s/it
+ "144p": {16: (1.0, 8), 32: (1.0, 4), 64: (1.0, 2), 128: (1.0, 1)},
+ "256": {1: (0.5, 32), 16: (0.5, 4), 32: (0.5, 2), 64: (0.5, 1), 128: (0.0, None)},
+ "240p": {16: (0.3, 3), 32: (0.3, 1), 64: (0.0, None)},
+ "512": {1: (0.4, 16)},
+ "1024": {1: (0.3, 4)},
}
mask_ratios = {
"mask_no": 0.9,
@@ -23,10 +28,10 @@ mask_ratios = {
}
# Define acceleration
-num_workers = 4
+num_workers = 8
num_bucket_build_workers = 16
dtype = "bf16"
-grad_checkpoint = True
+grad_checkpoint = False
plugin = "zero2"
sp_size = 1
diff --git a/opensora/datasets/aspect.py b/opensora/datasets/aspect.py
index 7588cde..0a541a0 100644
--- a/opensora/datasets/aspect.py
+++ b/opensora/datasets/aspect.py
@@ -164,6 +164,26 @@ ASPECT_RATIO_240P = {
"2.08": (462, 222),
}
+# S = 36864
+ASPECT_RATIO_144P = {
+ "0.38": (118, 314),
+ "0.43": (126, 294),
+ "0.48": (134, 280),
+ "0.50": (136, 272),
+ "0.53": (140, 264),
+ "0.54": (142, 262),
+ "0.56": (144, 256), # base
+ "0.62": (152, 244),
+ "0.67": (156, 234),
+ "0.75": (166, 222),
+ "1.00": (192, 192),
+ "1.33": (222, 166),
+ "1.50": (236, 158),
+ "1.78": (256, 144),
+ "1.89": (264, 140),
+ "2.00": (272, 136),
+ "2.08": (278, 134),
+}
# from PixArt
# S = 1048576
@@ -306,6 +326,7 @@ def get_closest_ratio(height: float, width: float, ratios: dict):
ASPECT_RATIOS = {
+ "144p": (36864, ASPECT_RATIO_144P),
"256": (65536, ASPECT_RATIO_256),
"240p": (102240, ASPECT_RATIO_240P),
"360p": (230400, ASPECT_RATIO_360P),
diff --git a/opensora/datasets/bucket.py b/opensora/datasets/bucket.py
index c60c6ff..01d9650 100644
--- a/opensora/datasets/bucket.py
+++ b/opensora/datasets/bucket.py
@@ -1,8 +1,9 @@
from collections import OrderedDict
-from .aspect import ASPECT_RATIOS, get_closest_ratio
import numpy as np
+from .aspect import ASPECT_RATIOS, get_closest_ratio
+
def find_approximate_hw(hw, hw_dict, approx=0.8):
for k, v in hw_dict.items():
@@ -80,9 +81,11 @@ class Bucket:
# if sample is an image
if T == 1:
if 1 in t_criteria:
- fail = False
- t_id = 1
- break
+ rng = np.random.default_rng(seed + self.bucket_id[hw_id][1])
+ if rng.random() < t_criteria[1]:
+ fail = False
+ t_id = 1
+ break
else:
continue
diff --git a/opensora/datasets/sampler.py b/opensora/datasets/sampler.py
index c0c0388..535b60d 100644
--- a/opensora/datasets/sampler.py
+++ b/opensora/datasets/sampler.py
@@ -3,7 +3,6 @@ from collections import OrderedDict, defaultdict
from pprint import pprint
from typing import Iterator, List, Optional
-import numpy as np
import torch
import torch.distributed as dist
from pandarallel import pandarallel
@@ -53,7 +52,7 @@ class VariableVideoBatchSampler(DistributedSampler):
def group_by_bucket(self) -> dict:
bucket_sample_dict = OrderedDict()
- pandarallel.initialize(progress_bar=True, nb_workers=self.num_bucket_build_workers)
+ pandarallel.initialize(nb_workers=self.num_bucket_build_workers, progress_bar=False)
bucket_ids = self.dataset.data.parallel_apply(
apply,
axis=1,
diff --git a/scripts/inference-long.py b/scripts/inference-long.py
index 33e3642..2860dee 100644
--- a/scripts/inference-long.py
+++ b/scripts/inference-long.py
@@ -150,6 +150,8 @@ def main():
width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
num_frames = torch.tensor([cfg.num_frames], device=device, dtype=dtype).repeat(cfg.batch_size)
ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
+ if cfg.num_frames == 1:
+ cfg.fps = IMG_FPS
fps = torch.tensor([cfg.fps], device=device, dtype=dtype).repeat(cfg.batch_size)
model_args["height"] = height
model_args["width"] = width
diff --git a/scripts/inference.py b/scripts/inference.py
index 7a66120..c59e560 100644
--- a/scripts/inference.py
+++ b/scripts/inference.py
@@ -7,7 +7,7 @@ from colossalai.cluster import DistCoordinator
from mmengine.runner import set_random_seed
from opensora.acceleration.parallel_states import set_sequence_parallel_group
-from opensora.datasets import save_sample
+from opensora.datasets import IMG_FPS, save_sample
from opensora.models.text_encoder.t5 import text_preprocessing
from opensora.registry import MODELS, SCHEDULERS, build_module
from opensora.utils.config_utils import parse_configs
@@ -87,6 +87,8 @@ def main():
width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
num_frames = torch.tensor([cfg.num_frames], device=device, dtype=dtype).repeat(cfg.batch_size)
ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
+ if cfg.num_frames == 1:
+ cfg.fps = IMG_FPS
fps = torch.tensor([cfg.fps], device=device, dtype=dtype).repeat(cfg.batch_size)
model_args["height"] = height
model_args["width"] = width