Open-Sora: Democratizing Efficient Video Production for All
Python
Feat/fast bucket (#54)
Zheng Zangwei (Alex Zheng)069ea0d6877b
diff --git a/configs/opensora-v1-1/train/stage1.py b/configs/opensora-v1-1/train/stage1.pyindex 0e1814d..bc55a0e 100644--- a/configs/opensora-v1-1/train/stage1.py+++ b/configs/opensora-v1-1/train/stage1.py@@ -12,7 +12,7 @@ bucket_config = { # 6s/it"256": {1: (1.0, 254)},"512": {1: (0.5, 86)},"480p": {1: (0.4, 54), 16: (0.4, 4), 32: (0.0, None)},- "720p": {16: (0.1, 2), 32: (0.0, None)}, # No examples now+ "720p": {16: (0.1, 2), 32: (0.0, None)},"1024": {1: (0.3, 20)},"1080p": {1: (0.4, 8)},}@@ -30,6 +30,7 @@ mask_ratios = {# Define accelerationnum_workers = 4+num_bucket_build_workers = 16dtype = "bf16"grad_checkpoint = Trueplugin = "zero2"diff --git a/opensora/datasets/bucket.py b/opensora/datasets/bucket.pyindex 117f5ee..c60c6ff 100644--- a/opensora/datasets/bucket.py+++ b/opensora/datasets/bucket.py@@ -1,8 +1,7 @@from collections import OrderedDict-import torch-from .aspect import ASPECT_RATIOS, get_closest_ratio+import numpy as npdef find_approximate_hw(hw, hw_dict, approx=0.8):@@ -44,12 +43,17 @@ class Bucket:hw_criteria = dict()t_criteria = dict()ar_criteria = dict()+ bucket_id = OrderedDict()+ bucket_id_cnt = 0for k1, v1 in bucket_probs.items():hw_criteria[k1] = ASPECT_RATIOS[k1][0]t_criteria[k1] = dict()ar_criteria[k1] = dict()+ bucket_id[k1] = dict()for k2, _ in v1.items():t_criteria[k1][k2] = k2+ bucket_id[k1][k2] = bucket_id_cnt+ bucket_id_cnt += 1ar_criteria[k1][k2] = dict()for k3, v3 in ASPECT_RATIOS[k1][1].items():ar_criteria[k1][k2][k3] = v3@@ -57,58 +61,52 @@ class Bucket:self.bucket_probs = bucket_probsself.bucket_bs = bucket_bs+ self.bucket_id = bucket_idself.hw_criteria = hw_criteriaself.t_criteria = t_criteriaself.ar_criteria = ar_criteriaself.num_bucket = num_bucketprint(f"Number of buckets: {num_bucket}")- def info_bucket(self, dataset, frame_interval=1):- infos = dict()- infos_ar = dict()- for i in range(len(dataset)):- T, H, W = dataset.get_data_info(i)- bucket_id = self.get_bucket_id(T, H, W, frame_interval)- if bucket_id is None:+ def get_bucket_id(self, T, H, W, frame_interval=1, seed=None):+ resolution = H * W+ approx = 0.8++ fail = True+ for hw_id, t_criteria in self.bucket_probs.items():+ if resolution < self.hw_criteria[hw_id] * approx:continue- if f"{(bucket_id[0], bucket_id[1])}" not in infos:- infos[f"{(bucket_id[0], bucket_id[1])}"] = 0- if f"{bucket_id[2]}" not in infos_ar:- infos_ar[f"{bucket_id[2]}"] = 0- infos[f"{(bucket_id[0], bucket_id[1])}"] += 1- infos_ar[f"{bucket_id[2]}"] += 1- print(f"Dataset contains {len(dataset)} samples.")- print("Bucket info:", infos)- print("Aspect ratio info:", infos_ar)-- def get_bucket_id(self, T, H, W, frame_interval=1, generator=None):- # hw- hw = H * W- hw_id = find_approximate_hw(hw, self.hw_criteria)- if hw_id is None:- return None- hw_id_index = list(self.hw_criteria.keys()).index(hw_id)-- # hw drops by probablity- while True:- # T- T_id = find_closet_smaller_bucket(T, self.t_criteria[hw_id], frame_interval)- if T_id is not None:- prob = self.get_prob((hw_id, T_id))- if torch.rand(1, generator=generator).item() < prob:++ # if sample is an image+ if T == 1:+ if 1 in t_criteria:+ fail = False+ t_id = 1break- hw_id_index += 1- if hw_id_index > len(self.hw_criteria) - 1:- break- hw_id = list(self.hw_criteria.keys())[hw_id_index]+ else:+ continue++ # otherwise, find suitable t_id for video+ t_fail = True+ for t_id, prob in t_criteria.items():+ if T > t_id * frame_interval and t_id != 1:+ t_fail = False+ break+ if t_fail:+ continue- if T_id is None or hw_id_index > len(self.hw_criteria) - 1:+ # leave the loop if prob is high enough+ rng = np.random.default_rng(seed + self.bucket_id[hw_id][t_id])+ if prob == 1 or rng.random() < prob:+ fail = False+ break+ if fail:return None- # ar- ar_criteria = self.ar_criteria[hw_id][T_id]+ # get aspect ratio id+ ar_criteria = self.ar_criteria[hw_id][t_id]ar_id = get_closest_ratio(H, W, ar_criteria)- return hw_id, T_id, ar_id+ return hw_id, t_id, ar_iddef get_thw(self, bucket_id):assert len(bucket_id) == 3diff --git a/opensora/datasets/dataloader.py b/opensora/datasets/dataloader.pyindex 518e64c..bf360b1 100644--- a/opensora/datasets/dataloader.py+++ b/opensora/datasets/dataloader.py@@ -107,6 +107,7 @@ def prepare_variable_dataloader(pin_memory=False,num_workers=0,process_group=None,+ num_bucket_build_workers=1,**kwargs,):_kwargs = kwargs.copy()@@ -120,6 +121,7 @@ def prepare_variable_dataloader(seed=seed,drop_last=drop_last,verbose=True,+ num_bucket_build_workers=num_bucket_build_workers,)# Deterministic dataloaderdiff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.pyindex 67791cb..ea0a61c 100644--- a/opensora/datasets/datasets.py+++ b/opensora/datasets/datasets.py@@ -113,6 +113,7 @@ class VariableVideoTextDataset(VideoTextDataset):):super().__init__(data_path, num_frames, frame_interval, image_size, transform_name=None)self.transform_name = transform_name+ self.data["id"] = np.arange(len(self.data))def get_data_info(self, index):T = self.data.iloc[index]["num_frames"]diff --git a/opensora/datasets/sampler.py b/opensora/datasets/sampler.pyindex 1478ff2..c0c0388 100644--- a/opensora/datasets/sampler.py+++ b/opensora/datasets/sampler.py@@ -3,14 +3,28 @@ from collections import OrderedDict, defaultdictfrom pprint import pprintfrom typing import Iterator, List, Optional+import numpy as npimport torchimport torch.distributed as dist+from pandarallel import pandarallelfrom torch.utils.data import DistributedSamplerfrom .bucket import Bucketfrom .datasets import VariableVideoTextDataset+# HACK: use pandarallel+# pandarallel should only access local variables+def apply(data, method=None, frame_interval=None, seed=None, num_bucket=None):+ return method(+ data["num_frames"],+ data["height"],+ data["width"],+ frame_interval,+ seed + data["id"] * num_bucket,+ )++class VariableVideoBatchSampler(DistributedSampler):def __init__(self,@@ -22,6 +36,7 @@ class VariableVideoBatchSampler(DistributedSampler):seed: int = 0,drop_last: bool = False,verbose: bool = False,+ num_bucket_build_workers: int = 1,) -> None:super().__init__(dataset=dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle, seed=seed, drop_last=drop_last@@ -32,48 +47,56 @@ class VariableVideoBatchSampler(DistributedSampler):self.last_micro_batch_access_index = 0self.approximate_num_batch = None- def get_num_batch(self) -> int:- g = torch.Generator()- g.manual_seed(self.seed + self.epoch)+ self._get_num_batch_cached_bucket_sample_dict = None+ self.num_bucket_build_workers = num_bucket_build_workers++ def group_by_bucket(self) -> dict:bucket_sample_dict = OrderedDict()+ pandarallel.initialize(progress_bar=True, nb_workers=self.num_bucket_build_workers)+ bucket_ids = self.dataset.data.parallel_apply(+ apply,+ axis=1,+ method=self.bucket.get_bucket_id,+ frame_interval=self.dataset.frame_interval,+ seed=self.seed + self.epoch,+ num_bucket=self.bucket.num_bucket,+ )+# group by bucket# each data sample is put into a bucket with a similar image/video sizefor i in range(len(self.dataset)):- t, h, w = self.dataset.get_data_info(i)- bucket_id = self.bucket.get_bucket_id(t, h, w, self.dataset.frame_interval, g)+ bucket_id = bucket_ids[i]if bucket_id is None:continueif bucket_id not in bucket_sample_dict:bucket_sample_dict[bucket_id] = []bucket_sample_dict[bucket_id].append(i)+ return bucket_sample_dict++ def get_num_batch(self) -> int:+ bucket_sample_dict = self.group_by_bucket()+ self._get_num_batch_cached_bucket_sample_dict = bucket_sample_dict# calculate the number of batches- self._print_bucket_info(bucket_sample_dict)+ if self.verbose:+ self._print_bucket_info(bucket_sample_dict)return self.approximate_num_batchdef __iter__(self) -> Iterator[List[int]]:+ if self._get_num_batch_cached_bucket_sample_dict is not None:+ bucket_sample_dict = self._get_num_batch_cached_bucket_sample_dict+ self._get_num_batch_cached_bucket_sample_dict = None+ else:+ bucket_sample_dict = self.group_by_bucket()+ if self.verbose:+ self._print_bucket_info(bucket_sample_dict)+g = torch.Generator()g.manual_seed(self.seed + self.epoch)- bucket_sample_dict = OrderedDict()bucket_micro_batch_count = OrderedDict()bucket_last_consumed = OrderedDict()- # group by bucket- # each data sample is put into a bucket with a similar image/video size- for i in range(len(self.dataset)):- t, h, w = self.dataset.get_data_info(i)- bucket_id = self.bucket.get_bucket_id(t, h, w, self.dataset.frame_interval, g)- if bucket_id is None:- continue- if bucket_id not in bucket_sample_dict:- bucket_sample_dict[bucket_id] = []- bucket_sample_dict[bucket_id].append(i)-- # print bucket info- if self.verbose:- self._print_bucket_info(bucket_sample_dict)-# process the samplesfor bucket_id, data_list in bucket_sample_dict.items():# handle droplastdiff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.pyindex ecd1b90..26ca2bc 100644--- a/opensora/utils/config_utils.py+++ b/opensora/utils/config_utils.py@@ -103,6 +103,8 @@ def merge_args(cfg, args, training=False):cfg["bucket_config"] = Noneif "transform_name" not in cfg.dataset:cfg.dataset["transform_name"] = "center"+ if "num_bucket_build_workers" not in cfg:+ cfg["num_bucket_build_workers"] = 1# Both training and inferenceif "multi_resolution" not in cfg:diff --git a/scripts/misc/sample.sh b/scripts/misc/sample.shindex 904505e..c3a91e8 100644--- a/scripts/misc/sample.sh+++ b/scripts/misc/sample.sh@@ -18,7 +18,7 @@ start=$(date +%s)### Functions-function run_image() {+function run_image() { # 10min# 1.1 1024x1024eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2i_samples.txt --save-dir $OUTPUT --num-frames 1 --image-size 1024 1024 --sample-name 1024x1024@@ -27,6 +27,9 @@ function run_image() {# 1.3 512x512eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2i_samples.txt --save-dir $OUTPUT --num-frames 1 --image-size 512 512 --sample-name 512x512 --end-index 3+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 1 --image-size 512 512 --sample-name 512x512 --end-index 3+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 1 --image-size 512 512 --sample-name 512x512 --end-index 3+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_sora.txt --save-dir $OUTPUT --num-frames 1 --image-size 512 512 --sample-name 512x512 --end-index 3# 1.4 720p multi-resolution# 1:1@@ -46,11 +49,9 @@ function run_image() {eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --num-frames 1 --image-size 600 1358 --sample-name 720p_2_1}-function run_video_1() {+function run_video_1() { # 20min# 2.1.1 16x240x426eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name sample_16x240x426- eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name short_16x240x426- eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_sora.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name sora_16x240x426# 2.1.2 16x720p multi-resolution# 1:1@@ -70,38 +71,52 @@ function run_video_1() {eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --num-frames 16 --image-size 600 1358 --sample-name 720p_2_1}-function run_video_2() {- # 2.2.1 64x240x426+function run_video_2() { # 60min+ # 2.2.1 16x240x426+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name short_16x240x426++ # 2.2.2 64x240x426eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 64 --image-size 240 426 --sample-name short_64x240x426+}- # 2.2.2 128x240x426+function run_video_3() { # 60min+ # 2.3.1 16x240x426+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_sora.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name sora_16x240x426++ # 2.3.2 128x240x426eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 128 --image-size 240 426 --sample-name short_128x240x426+}- # 2.2.3 16x480x854+function run_video_4() { # 120min+ # 2.4 16x480x854eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 16 --image-size 480 854 --sample-name short_16x480x854+}- # 2.2.4 64x480x854+function run_video_5() { # 120min+ # 2.5 64x480x854eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 64 --image-size 480 854 --sample-name short_64x480x854+}- # 2.2.5 16x720x1280+function run_video_6() { # 120min+ # 2.6 16x720x1280eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 16 --image-size 720 1280 --sample-name short_16x720x1280}-function run_video_edit() {+function run_video_edit() { # 23min# 3.1 image-conditioned long video generationeval $CMD_REF --ckpt-path $CKPT --save-dir $OUTPUT --sample-name ref_L10C4_16x240x426 \--prompt-path assets/texts/t2v_ref.txt --start-index 0 --end-index 3 \--num-frames 16 --image-size 240 426 \--loop 5 --condition-frame-length 4 \- --reference-path assets/images/condition/cliff.png assets/images/condition/wave.png \- --mask-strategy "0,0,0,1,0" "0,0,0,1,0"+ --reference-path assets/images/condition/cliff.png assets/images/condition/wave.png assets/images/condition/ship.png \+ --mask-strategy "0,0,0,1,0" "0,0,0,1,0" "0,0,0,1,0"eval $CMD_REF --ckpt-path $CKPT --save-dir $OUTPUT --sample-name ref_L10C4_64x240x426 \--prompt-path assets/texts/t2v_ref.txt --start-index 0 --end-index 3 \--num-frames 64 --image-size 240 426 \--loop 5 --condition-frame-length 16 \- --reference-path assets/images/condition/cliff.png assets/images/condition/wave.png \- --mask-strategy "0,0,0,1,0" "0,0,0,1,0"+ --reference-path assets/images/condition/cliff.png assets/images/condition/wave.png assets/images/condition/ship.png \+ --mask-strategy "0,0,0,1,0" "0,0,0,1,0" "0,0,0,1,0"# 3.2eval $CMD_REF --ckpt-path $CKPT --save-dir $OUTPUT --sample-name ref_L1_128x240x426 \@@ -116,20 +131,36 @@ function run_video_edit() {for arg in "$@"; doif [[ "$arg" = -1 ]] || [[ "$arg" = --image ]]; then- run_imageecho "Running image samples..."+ run_imagefi- if [[ "$arg" = -2_1 ]] || [[ "$arg" = --video ]]; then- run_video_1+ if [[ "$arg" = -2a ]] || [[ "$arg" = --video ]]; thenecho "Running video samples 1..."+ run_video_1fi- if [[ "$arg" = -2_2 ]] || [[ "$arg" = --video ]]; then- run_video_2+ if [[ "$arg" = -2b ]] || [[ "$arg" = --video ]]; thenecho "Running video samples 2..."+ run_video_2+ fi+ if [[ "$arg" = -2c ]] || [[ "$arg" = --video ]]; then+ echo "Running video samples 3..."+ run_video_3+ fi+ if [[ "$arg" = -2d ]] || [[ "$arg" = --video ]]; then+ echo "Running video samples 4..."+ run_video_4+ fi+ if [[ "$arg" = -2e ]] || [[ "$arg" = --video ]]; then+ echo "Running video samples 5..."+ run_video_5+ fi+ if [[ "$arg" = -2f ]] || [[ "$arg" = --video ]]; then+ echo "Running video samples 6..."+ run_video_6fiif [[ "$arg" = -3 ]] || [[ "$arg" = --video-edit ]]; then- run_video_editecho "Running video edit samples..."+ run_video_editfidonediff --git a/scripts/train.py b/scripts/train.pyindex 9d76bf6..1884fb3 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -4,6 +4,7 @@ from pprint import pprintimport torchimport torch.distributed as dist+import wandbfrom colossalai.booster import Boosterfrom colossalai.booster.plugin import LowLevelZeroPluginfrom colossalai.cluster import DistCoordinator@@ -11,7 +12,6 @@ from colossalai.nn.optimizer import HybridAdamfrom colossalai.utils import get_current_device, set_seedfrom tqdm import tqdm-import wandbfrom opensora.acceleration.checkpoint import set_grad_checkpointfrom opensora.acceleration.parallel_states import (get_data_parallel_group,@@ -95,6 +95,7 @@ def main():# 3. build dataset and dataloader# ======================================================dataset = build_module(cfg.dataset, DATASETS)+ logger.info(f"Dataset contains {len(dataset)} samples.")dataloader_args = dict(dataset=dataset,batch_size=cfg.batch_size,@@ -109,7 +110,11 @@ def main():if cfg.bucket_config is None:dataloader = prepare_dataloader(**dataloader_args)else:- dataloader = prepare_variable_dataloader(bucket_config=cfg.bucket_config, **dataloader_args)+ dataloader = prepare_variable_dataloader(+ bucket_config=cfg.bucket_config,+ num_bucket_build_workers=cfg.num_bucket_build_workers,+ **dataloader_args,+ )if cfg.dataset.type == "VideoTextDataset":total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_sizelogger.info(f"Total batch size: {total_batch_size}")