Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

Feat/fast bucket (#54)

Zheng Zangwei (Alex Zheng)069ea0d6877b
diff --git a/configs/opensora-v1-1/train/stage1.py b/configs/opensora-v1-1/train/stage1.py
index 0e1814d..bc55a0e 100644
--- a/configs/opensora-v1-1/train/stage1.py
+++ b/configs/opensora-v1-1/train/stage1.py
@@ -12,7 +12,7 @@ bucket_config = { # 6s/it
"256": {1: (1.0, 254)},
"512": {1: (0.5, 86)},
"480p": {1: (0.4, 54), 16: (0.4, 4), 32: (0.0, None)},
- "720p": {16: (0.1, 2), 32: (0.0, None)}, # No examples now
+ "720p": {16: (0.1, 2), 32: (0.0, None)},
"1024": {1: (0.3, 20)},
"1080p": {1: (0.4, 8)},
}
@@ -30,6 +30,7 @@ mask_ratios = {
# Define acceleration
num_workers = 4
+num_bucket_build_workers = 16
dtype = "bf16"
grad_checkpoint = True
plugin = "zero2"
diff --git a/opensora/datasets/bucket.py b/opensora/datasets/bucket.py
index 117f5ee..c60c6ff 100644
--- a/opensora/datasets/bucket.py
+++ b/opensora/datasets/bucket.py
@@ -1,8 +1,7 @@
from collections import OrderedDict
-import torch
-
from .aspect import ASPECT_RATIOS, get_closest_ratio
+import numpy as np
def find_approximate_hw(hw, hw_dict, approx=0.8):
@@ -44,12 +43,17 @@ class Bucket:
hw_criteria = dict()
t_criteria = dict()
ar_criteria = dict()
+ bucket_id = OrderedDict()
+ bucket_id_cnt = 0
for k1, v1 in bucket_probs.items():
hw_criteria[k1] = ASPECT_RATIOS[k1][0]
t_criteria[k1] = dict()
ar_criteria[k1] = dict()
+ bucket_id[k1] = dict()
for k2, _ in v1.items():
t_criteria[k1][k2] = k2
+ bucket_id[k1][k2] = bucket_id_cnt
+ bucket_id_cnt += 1
ar_criteria[k1][k2] = dict()
for k3, v3 in ASPECT_RATIOS[k1][1].items():
ar_criteria[k1][k2][k3] = v3
@@ -57,58 +61,52 @@ class Bucket:
self.bucket_probs = bucket_probs
self.bucket_bs = bucket_bs
+ self.bucket_id = bucket_id
self.hw_criteria = hw_criteria
self.t_criteria = t_criteria
self.ar_criteria = ar_criteria
self.num_bucket = num_bucket
print(f"Number of buckets: {num_bucket}")
- def info_bucket(self, dataset, frame_interval=1):
- infos = dict()
- infos_ar = dict()
- for i in range(len(dataset)):
- T, H, W = dataset.get_data_info(i)
- bucket_id = self.get_bucket_id(T, H, W, frame_interval)
- if bucket_id is None:
+ def get_bucket_id(self, T, H, W, frame_interval=1, seed=None):
+ resolution = H * W
+ approx = 0.8
+
+ fail = True
+ for hw_id, t_criteria in self.bucket_probs.items():
+ if resolution < self.hw_criteria[hw_id] * approx:
continue
- if f"{(bucket_id[0], bucket_id[1])}" not in infos:
- infos[f"{(bucket_id[0], bucket_id[1])}"] = 0
- if f"{bucket_id[2]}" not in infos_ar:
- infos_ar[f"{bucket_id[2]}"] = 0
- infos[f"{(bucket_id[0], bucket_id[1])}"] += 1
- infos_ar[f"{bucket_id[2]}"] += 1
- print(f"Dataset contains {len(dataset)} samples.")
- print("Bucket info:", infos)
- print("Aspect ratio info:", infos_ar)
-
- def get_bucket_id(self, T, H, W, frame_interval=1, generator=None):
- # hw
- hw = H * W
- hw_id = find_approximate_hw(hw, self.hw_criteria)
- if hw_id is None:
- return None
- hw_id_index = list(self.hw_criteria.keys()).index(hw_id)
-
- # hw drops by probablity
- while True:
- # T
- T_id = find_closet_smaller_bucket(T, self.t_criteria[hw_id], frame_interval)
- if T_id is not None:
- prob = self.get_prob((hw_id, T_id))
- if torch.rand(1, generator=generator).item() < prob:
+
+ # if sample is an image
+ if T == 1:
+ if 1 in t_criteria:
+ fail = False
+ t_id = 1
break
- hw_id_index += 1
- if hw_id_index > len(self.hw_criteria) - 1:
- break
- hw_id = list(self.hw_criteria.keys())[hw_id_index]
+ else:
+ continue
+
+ # otherwise, find suitable t_id for video
+ t_fail = True
+ for t_id, prob in t_criteria.items():
+ if T > t_id * frame_interval and t_id != 1:
+ t_fail = False
+ break
+ if t_fail:
+ continue
- if T_id is None or hw_id_index > len(self.hw_criteria) - 1:
+ # leave the loop if prob is high enough
+ rng = np.random.default_rng(seed + self.bucket_id[hw_id][t_id])
+ if prob == 1 or rng.random() < prob:
+ fail = False
+ break
+ if fail:
return None
- # ar
- ar_criteria = self.ar_criteria[hw_id][T_id]
+ # get aspect ratio id
+ ar_criteria = self.ar_criteria[hw_id][t_id]
ar_id = get_closest_ratio(H, W, ar_criteria)
- return hw_id, T_id, ar_id
+ return hw_id, t_id, ar_id
def get_thw(self, bucket_id):
assert len(bucket_id) == 3
diff --git a/opensora/datasets/dataloader.py b/opensora/datasets/dataloader.py
index 518e64c..bf360b1 100644
--- a/opensora/datasets/dataloader.py
+++ b/opensora/datasets/dataloader.py
@@ -107,6 +107,7 @@ def prepare_variable_dataloader(
pin_memory=False,
num_workers=0,
process_group=None,
+ num_bucket_build_workers=1,
**kwargs,
):
_kwargs = kwargs.copy()
@@ -120,6 +121,7 @@ def prepare_variable_dataloader(
seed=seed,
drop_last=drop_last,
verbose=True,
+ num_bucket_build_workers=num_bucket_build_workers,
)
# Deterministic dataloader
diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.py
index 67791cb..ea0a61c 100644
--- a/opensora/datasets/datasets.py
+++ b/opensora/datasets/datasets.py
@@ -113,6 +113,7 @@ class VariableVideoTextDataset(VideoTextDataset):
):
super().__init__(data_path, num_frames, frame_interval, image_size, transform_name=None)
self.transform_name = transform_name
+ self.data["id"] = np.arange(len(self.data))
def get_data_info(self, index):
T = self.data.iloc[index]["num_frames"]
diff --git a/opensora/datasets/sampler.py b/opensora/datasets/sampler.py
index 1478ff2..c0c0388 100644
--- a/opensora/datasets/sampler.py
+++ b/opensora/datasets/sampler.py
@@ -3,14 +3,28 @@ from collections import OrderedDict, defaultdict
from pprint import pprint
from typing import Iterator, List, Optional
+import numpy as np
import torch
import torch.distributed as dist
+from pandarallel import pandarallel
from torch.utils.data import DistributedSampler
from .bucket import Bucket
from .datasets import VariableVideoTextDataset
+# HACK: use pandarallel
+# pandarallel should only access local variables
+def apply(data, method=None, frame_interval=None, seed=None, num_bucket=None):
+ return method(
+ data["num_frames"],
+ data["height"],
+ data["width"],
+ frame_interval,
+ seed + data["id"] * num_bucket,
+ )
+
+
class VariableVideoBatchSampler(DistributedSampler):
def __init__(
self,
@@ -22,6 +36,7 @@ class VariableVideoBatchSampler(DistributedSampler):
seed: int = 0,
drop_last: bool = False,
verbose: bool = False,
+ num_bucket_build_workers: int = 1,
) -> None:
super().__init__(
dataset=dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle, seed=seed, drop_last=drop_last
@@ -32,48 +47,56 @@ class VariableVideoBatchSampler(DistributedSampler):
self.last_micro_batch_access_index = 0
self.approximate_num_batch = None
- def get_num_batch(self) -> int:
- g = torch.Generator()
- g.manual_seed(self.seed + self.epoch)
+ self._get_num_batch_cached_bucket_sample_dict = None
+ self.num_bucket_build_workers = num_bucket_build_workers
+
+ def group_by_bucket(self) -> dict:
bucket_sample_dict = OrderedDict()
+ pandarallel.initialize(progress_bar=True, nb_workers=self.num_bucket_build_workers)
+ bucket_ids = self.dataset.data.parallel_apply(
+ apply,
+ axis=1,
+ method=self.bucket.get_bucket_id,
+ frame_interval=self.dataset.frame_interval,
+ seed=self.seed + self.epoch,
+ num_bucket=self.bucket.num_bucket,
+ )
+
# group by bucket
# each data sample is put into a bucket with a similar image/video size
for i in range(len(self.dataset)):
- t, h, w = self.dataset.get_data_info(i)
- bucket_id = self.bucket.get_bucket_id(t, h, w, self.dataset.frame_interval, g)
+ bucket_id = bucket_ids[i]
if bucket_id is None:
continue
if bucket_id not in bucket_sample_dict:
bucket_sample_dict[bucket_id] = []
bucket_sample_dict[bucket_id].append(i)
+ return bucket_sample_dict
+
+ def get_num_batch(self) -> int:
+ bucket_sample_dict = self.group_by_bucket()
+ self._get_num_batch_cached_bucket_sample_dict = bucket_sample_dict
# calculate the number of batches
- self._print_bucket_info(bucket_sample_dict)
+ if self.verbose:
+ self._print_bucket_info(bucket_sample_dict)
return self.approximate_num_batch
def __iter__(self) -> Iterator[List[int]]:
+ if self._get_num_batch_cached_bucket_sample_dict is not None:
+ bucket_sample_dict = self._get_num_batch_cached_bucket_sample_dict
+ self._get_num_batch_cached_bucket_sample_dict = None
+ else:
+ bucket_sample_dict = self.group_by_bucket()
+ if self.verbose:
+ self._print_bucket_info(bucket_sample_dict)
+
g = torch.Generator()
g.manual_seed(self.seed + self.epoch)
- bucket_sample_dict = OrderedDict()
bucket_micro_batch_count = OrderedDict()
bucket_last_consumed = OrderedDict()
- # group by bucket
- # each data sample is put into a bucket with a similar image/video size
- for i in range(len(self.dataset)):
- t, h, w = self.dataset.get_data_info(i)
- bucket_id = self.bucket.get_bucket_id(t, h, w, self.dataset.frame_interval, g)
- if bucket_id is None:
- continue
- if bucket_id not in bucket_sample_dict:
- bucket_sample_dict[bucket_id] = []
- bucket_sample_dict[bucket_id].append(i)
-
- # print bucket info
- if self.verbose:
- self._print_bucket_info(bucket_sample_dict)
-
# process the samples
for bucket_id, data_list in bucket_sample_dict.items():
# handle droplast
diff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.py
index ecd1b90..26ca2bc 100644
--- a/opensora/utils/config_utils.py
+++ b/opensora/utils/config_utils.py
@@ -103,6 +103,8 @@ def merge_args(cfg, args, training=False):
cfg["bucket_config"] = None
if "transform_name" not in cfg.dataset:
cfg.dataset["transform_name"] = "center"
+ if "num_bucket_build_workers" not in cfg:
+ cfg["num_bucket_build_workers"] = 1
# Both training and inference
if "multi_resolution" not in cfg:
diff --git a/scripts/misc/sample.sh b/scripts/misc/sample.sh
index 904505e..c3a91e8 100644
--- a/scripts/misc/sample.sh
+++ b/scripts/misc/sample.sh
@@ -18,7 +18,7 @@ start=$(date +%s)
### Functions
-function run_image() {
+function run_image() { # 10min
# 1.1 1024x1024
eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2i_samples.txt --save-dir $OUTPUT --num-frames 1 --image-size 1024 1024 --sample-name 1024x1024
@@ -27,6 +27,9 @@ function run_image() {
# 1.3 512x512
eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2i_samples.txt --save-dir $OUTPUT --num-frames 1 --image-size 512 512 --sample-name 512x512 --end-index 3
+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 1 --image-size 512 512 --sample-name 512x512 --end-index 3
+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 1 --image-size 512 512 --sample-name 512x512 --end-index 3
+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_sora.txt --save-dir $OUTPUT --num-frames 1 --image-size 512 512 --sample-name 512x512 --end-index 3
# 1.4 720p multi-resolution
# 1:1
@@ -46,11 +49,9 @@ function run_image() {
eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --num-frames 1 --image-size 600 1358 --sample-name 720p_2_1
}
-function run_video_1() {
+function run_video_1() { # 20min
# 2.1.1 16x240x426
eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name sample_16x240x426
- eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name short_16x240x426
- eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_sora.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name sora_16x240x426
# 2.1.2 16x720p multi-resolution
# 1:1
@@ -70,38 +71,52 @@ function run_video_1() {
eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --num-frames 16 --image-size 600 1358 --sample-name 720p_2_1
}
-function run_video_2() {
- # 2.2.1 64x240x426
+function run_video_2() { # 60min
+ # 2.2.1 16x240x426
+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name short_16x240x426
+
+ # 2.2.2 64x240x426
eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 64 --image-size 240 426 --sample-name short_64x240x426
+}
- # 2.2.2 128x240x426
+function run_video_3() { # 60min
+ # 2.3.1 16x240x426
+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_sora.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name sora_16x240x426
+
+ # 2.3.2 128x240x426
eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 128 --image-size 240 426 --sample-name short_128x240x426
+}
- # 2.2.3 16x480x854
+function run_video_4() { # 120min
+ # 2.4 16x480x854
eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 16 --image-size 480 854 --sample-name short_16x480x854
+}
- # 2.2.4 64x480x854
+function run_video_5() { # 120min
+ # 2.5 64x480x854
eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 64 --image-size 480 854 --sample-name short_64x480x854
+}
- # 2.2.5 16x720x1280
+function run_video_6() { # 120min
+ # 2.6 16x720x1280
eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_short.txt --save-dir $OUTPUT --num-frames 16 --image-size 720 1280 --sample-name short_16x720x1280
}
-function run_video_edit() {
+function run_video_edit() { # 23min
# 3.1 image-conditioned long video generation
eval $CMD_REF --ckpt-path $CKPT --save-dir $OUTPUT --sample-name ref_L10C4_16x240x426 \
--prompt-path assets/texts/t2v_ref.txt --start-index 0 --end-index 3 \
--num-frames 16 --image-size 240 426 \
--loop 5 --condition-frame-length 4 \
- --reference-path assets/images/condition/cliff.png assets/images/condition/wave.png \
- --mask-strategy "0,0,0,1,0" "0,0,0,1,0"
+ --reference-path assets/images/condition/cliff.png assets/images/condition/wave.png assets/images/condition/ship.png \
+ --mask-strategy "0,0,0,1,0" "0,0,0,1,0" "0,0,0,1,0"
eval $CMD_REF --ckpt-path $CKPT --save-dir $OUTPUT --sample-name ref_L10C4_64x240x426 \
--prompt-path assets/texts/t2v_ref.txt --start-index 0 --end-index 3 \
--num-frames 64 --image-size 240 426 \
--loop 5 --condition-frame-length 16 \
- --reference-path assets/images/condition/cliff.png assets/images/condition/wave.png \
- --mask-strategy "0,0,0,1,0" "0,0,0,1,0"
+ --reference-path assets/images/condition/cliff.png assets/images/condition/wave.png assets/images/condition/ship.png \
+ --mask-strategy "0,0,0,1,0" "0,0,0,1,0" "0,0,0,1,0"
# 3.2
eval $CMD_REF --ckpt-path $CKPT --save-dir $OUTPUT --sample-name ref_L1_128x240x426 \
@@ -116,20 +131,36 @@ function run_video_edit() {
for arg in "$@"; do
if [[ "$arg" = -1 ]] || [[ "$arg" = --image ]]; then
- run_image
echo "Running image samples..."
+ run_image
fi
- if [[ "$arg" = -2_1 ]] || [[ "$arg" = --video ]]; then
- run_video_1
+ if [[ "$arg" = -2a ]] || [[ "$arg" = --video ]]; then
echo "Running video samples 1..."
+ run_video_1
fi
- if [[ "$arg" = -2_2 ]] || [[ "$arg" = --video ]]; then
- run_video_2
+ if [[ "$arg" = -2b ]] || [[ "$arg" = --video ]]; then
echo "Running video samples 2..."
+ run_video_2
+ fi
+ if [[ "$arg" = -2c ]] || [[ "$arg" = --video ]]; then
+ echo "Running video samples 3..."
+ run_video_3
+ fi
+ if [[ "$arg" = -2d ]] || [[ "$arg" = --video ]]; then
+ echo "Running video samples 4..."
+ run_video_4
+ fi
+ if [[ "$arg" = -2e ]] || [[ "$arg" = --video ]]; then
+ echo "Running video samples 5..."
+ run_video_5
+ fi
+ if [[ "$arg" = -2f ]] || [[ "$arg" = --video ]]; then
+ echo "Running video samples 6..."
+ run_video_6
fi
if [[ "$arg" = -3 ]] || [[ "$arg" = --video-edit ]]; then
- run_video_edit
echo "Running video edit samples..."
+ run_video_edit
fi
done
diff --git a/scripts/train.py b/scripts/train.py
index 9d76bf6..1884fb3 100644
--- a/scripts/train.py
+++ b/scripts/train.py
@@ -4,6 +4,7 @@ from pprint import pprint
import torch
import torch.distributed as dist
+import wandb
from colossalai.booster import Booster
from colossalai.booster.plugin import LowLevelZeroPlugin
from colossalai.cluster import DistCoordinator
@@ -11,7 +12,6 @@ from colossalai.nn.optimizer import HybridAdam
from colossalai.utils import get_current_device, set_seed
from tqdm import tqdm
-import wandb
from opensora.acceleration.checkpoint import set_grad_checkpoint
from opensora.acceleration.parallel_states import (
get_data_parallel_group,
@@ -95,6 +95,7 @@ def main():
# 3. build dataset and dataloader
# ======================================================
dataset = build_module(cfg.dataset, DATASETS)
+ logger.info(f"Dataset contains {len(dataset)} samples.")
dataloader_args = dict(
dataset=dataset,
batch_size=cfg.batch_size,
@@ -109,7 +110,11 @@ def main():
if cfg.bucket_config is None:
dataloader = prepare_dataloader(**dataloader_args)
else:
- dataloader = prepare_variable_dataloader(bucket_config=cfg.bucket_config, **dataloader_args)
+ dataloader = prepare_variable_dataloader(
+ bucket_config=cfg.bucket_config,
+ num_bucket_build_workers=cfg.num_bucket_build_workers,
+ **dataloader_args,
+ )
if cfg.dataset.type == "VideoTextDataset":
total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size
logger.info(f"Total batch size: {total_batch_size}")