Open-Sora: Democratizing Efficient Video Production for All
Python
update csvutil & mask
Zangwei Zheng12dcbeee8eb9
diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.pyindex 9df37e7..d45c2a5 100644--- a/configs/opensora-v1-1/train/benchmark.py+++ b/configs/opensora-v1-1/train/benchmark.py@@ -8,7 +8,7 @@ dataset = dict(transform_name="resize_crop",)bucket_config = {- "240p": {16: (1.0, 16)},+ "240p": {16: (1.0, 16)}, # 4.6s/it# "256": {1: (1.0, 256)}, # 4.5s/it# "512": {1: (1.0, 96)}, # 4.7s/it# "512": {1: (1.0, 128)}, # 6.3s/it@@ -18,6 +18,10 @@ bucket_config = {# "1080p": {1: (1.0, 16)}, # 8.6s/it# "1080p": {1: (1.0, 8)}, # 4.4s/it}+mask_ratios = {+ "mask_no": 0.0,+ "mask_random": 1.0,+}# Define accelerationnum_workers = 4diff --git a/opensora/utils/train_utils.py b/opensora/utils/train_utils.pyindex 1bd4ea4..8adf50c 100644--- a/opensora/utils/train_utils.py+++ b/opensora/utils/train_utils.py@@ -34,36 +34,55 @@ def update_ema(class MaskGenerator:- def __init__(self, mask_ratios, condition_frames=4):- self.mask_name = ["mask_no", "mask_random", "mask_head", "mask_tail", "mask_head_tail"]- assert len(mask_ratios) == len(self.mask_name)- assert math.isclose(sum(mask_ratios), 1.0, abs_tol=1e-6)- self.mask_prob = mask_ratios- print(self.mask_prob)- self.mask_acc_prob = [sum(self.mask_prob[: i + 1]) for i in range(len(self.mask_prob))]- self.condition_frames = condition_frames+ def __init__(self, mask_ratios):+ valid_mask_names = ["mask_no", "mask_random", "mask_head", "mask_tail", "mask_head_tail"]+ assert all(+ mask_name in valid_mask_names for mask_name in mask_ratios.keys()+ ), f"mask_name should be one of {valid_mask_names}, got {mask_ratios.keys()}"+ assert all(+ mask_ratio >= 0 for mask_ratio in mask_ratios.values()+ ), f"mask_ratio should be greater than or equal to 0, got {mask_ratios.values()}"+ assert all(+ mask_ratio <= 1 for mask_ratio in mask_ratios.values()+ ), f"mask_ratio should be less than or equal to 1, got {mask_ratios.values()}"+ # sum of mask_ratios should be 1+ assert math.isclose(+ sum(mask_ratios.values()), 1.0, abs_tol=1e-6+ ), f"sum of mask_ratios should be 1, got {sum(mask_ratios.values())}"+ print(f"mask ratios: {mask_ratios}")+ self.mask_ratios = mask_ratiosdef get_mask(self, x):mask_type = random.random()- for i, acc_prob in enumerate(self.mask_acc_prob):- if mask_type <= acc_prob:- mask_name = self.mask_name[i]+ mask_name = None+ prob_acc = 0.0+ for mask, mask_ratio in self.mask_ratios.items():+ prob_acc += mask_ratio+ if mask_type < prob_acc:+ mask_name = maskbreak- mask = torch.ones(x.shape[2], dtype=torch.bool, device=x.device)+ num_frames = x.shape[2]+ # Hardcoded condition_frames+ condition_frames_max = num_frames // 4++ mask = torch.ones(num_frames, dtype=torch.bool, device=x.device)+ if num_frames <= 1:+ return mask+if mask_name == "mask_random":- random_size = random.randint(1, self.condition_frames)+ random_size = random.randint(1, condition_frames_max)random_pos = random.randint(0, x.shape[2] - random_size)mask[random_pos : random_pos + random_size] = 0return maskelif mask_name == "mask_head":- random_size = random.randint(1, self.condition_frames)+ random_size = random.randint(1, condition_frames_max)mask[:random_size] = 0elif mask_name == "mask_tail":- random_size = random.randint(1, self.condition_frames)+ random_size = random.randint(1, condition_frames_max)mask[-random_size:] = 0elif mask_name == "mask_head_tail":- random_size = random.randint(1, self.condition_frames)+ random_size = random.randint(1, condition_frames_max)mask[:random_size] = 0mask[-random_size:] = 0diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex 32865f2..e7eb0a5 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -395,6 +395,10 @@ def main(args):if args.lang is not None:assert "text" in data.columnsdata = data[data["text"].progress_apply(detect_lang)] # cannot parallelize+ if args.remove_empty_caption:+ assert "text" in data.columns+ data = data[data["text"].str.len() > 0]+ data = data[~data["text"].isna()]# processingif args.relpath is not None: