Open-Sora: Democratizing Efficient Video Production for All
Python
hotfix inference
Zangwei Zheng63e86f6dd651
diff --git a/configs/opensora-v1-1/train/stage1.py b/configs/opensora-v1-1/train/stage1.pynew file mode 100644index 0000000..2c06757--- /dev/null+++ b/configs/opensora-v1-1/train/stage1.py@@ -0,0 +1,70 @@+# Define dataset+dataset = dict(+ type="VariableVideoTextDataset",+ data_path=None,+ num_frames=None,+ frame_interval=3,+ image_size=(None, None),+ transform_name="resize_crop",+)+bucket_config = { # 6s/it+ "240p": {16: (1.0, 17), 32: (1.0, 9), 64: (1.0, 4), 128: (1.0, 2)},+ "256": {1: (1.0, 254)},+ "512": {1: (0.5, 86)},+ "480p": {1: (0.4, 54), 16: (0.4, 4), 32: (0.0, None)},+ "720p": {16: (0.1, 2), 32: (0.0, None)}, # No examples now+ "1024": {1: (0.3, 20)},+ "1080p": {1: (0.4, 8)},+}+mask_ratios = {+ "mask_no": 0.9,+ "mask_random": 0.06,+ "mask_head": 0.01,+ "mask_tail": 0.01,+ "mask_head_tail": 0.02,+}++# Define acceleration+num_workers = 4+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1++# Define model+model = dict(+ type="STDiT2-XL/2",+ from_pretrained=None,+ input_sq_size=512, # pretrained model is trained on 512x512+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+vae = dict(+ type="VideoAutoencoderKL",+ from_pretrained="stabilityai/sd-vae-ft-ema",+ micro_batch_size=4,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=200,+ shardformer=True,+)+scheduler = dict(+ type="iddpm-speed",+ timestep_respacing="",+)++# Others+seed = 42+outputs = "outputs"+wandb = False++epochs = 1000+log_every = 10+ckpt_every = 500+load = None++batch_size = 17 # only for logging+lr = 2e-5+grad_clip = 1.0diff --git a/opensora/models/text_encoder/t5.py b/opensora/models/text_encoder/t5.pyindex 807d2ad..7364eb9 100644--- a/opensora/models/text_encoder/t5.py+++ b/opensora/models/text_encoder/t5.py@@ -21,7 +21,10 @@# T5: https://github.com/google-research/text-to-text-transfer-transformer# --------------------------------------------------------+import html+import re+import ftfyimport torchfrom transformers import AutoTokenizer, T5EncoderModel@@ -96,12 +99,8 @@ class T5Embedder:self.hf_token = hf_tokenassert from_pretrained in self.available_models- self.tokenizer = AutoTokenizer.from_pretrained(- from_pretrained, cache_dir=cache_dir- )- self.model = T5EncoderModel.from_pretrained(- from_pretrained, cache_dir=cache_dir, **t5_model_kwargs- ).eval()+ self.tokenizer = AutoTokenizer.from_pretrained(from_pretrained, cache_dir=cache_dir)+ self.model = T5EncoderModel.from_pretrained(from_pretrained, cache_dir=cache_dir, **t5_model_kwargs).eval()self.model_max_length = model_max_lengthdef get_text_embeddings(self, texts):@@ -185,3 +184,142 @@ class T5Encoder:def null(self, n):null_y = self.y_embedder.y_embedding[None].repeat(n, 1, 1)[:, None]return null_y+++def basic_clean(text):+ text = ftfy.fix_text(text)+ text = html.unescape(html.unescape(text))+ return text.strip()+++BAD_PUNCT_REGEX = re.compile(+ r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"+) # noqa+++def clean_caption(caption):+ import urllib.parse as ul++ from bs4 import BeautifulSoup++ caption = str(caption)+ caption = ul.unquote_plus(caption)+ caption = caption.strip().lower()+ caption = re.sub("<person>", "person", caption)+ # urls:+ caption = re.sub(+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ caption = re.sub(+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ # html:+ caption = BeautifulSoup(caption, features="html.parser").text++ # @<nickname>+ caption = re.sub(r"@[\w\d]+\b", "", caption)++ # 31C0—31EF CJK Strokes+ # 31F0—31FF Katakana Phonetic Extensions+ # 3200—32FF Enclosed CJK Letters and Months+ # 3300—33FF CJK Compatibility+ # 3400—4DBF CJK Unified Ideographs Extension A+ # 4DC0—4DFF Yijing Hexagram Symbols+ # 4E00—9FFF CJK Unified Ideographs+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)+ #######################################################++ # все виды тире / all types of dash --> "-"+ caption = re.sub(+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa+ "-",+ caption,+ )++ # кавычки к одному стандарту+ caption = re.sub(r"[`´«»“”¨]", '"', caption)+ caption = re.sub(r"[‘’]", "'", caption)++ # "+ caption = re.sub(r""?", "", caption)+ # &+ caption = re.sub(r"&", "", caption)++ # ip adresses:+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)++ # article ids:+ caption = re.sub(r"\d:\d\d\s+$", "", caption)++ # \n+ caption = re.sub(r"\\n", " ", caption)++ # "#123"+ caption = re.sub(r"#\d{1,3}\b", "", caption)+ # "#12345.."+ caption = re.sub(r"#\d{5,}\b", "", caption)+ # "123456.."+ caption = re.sub(r"\b\d{6,}\b", "", caption)+ # filenames:+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)++ #+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""++ caption = re.sub(BAD_PUNCT_REGEX, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "++ # this-is-my-cute-cat / this_is_my_cute_cat+ regex2 = re.compile(r"(?:\-|\_)")+ if len(re.findall(regex2, caption)) > 3:+ caption = re.sub(regex2, " ", caption)++ caption = basic_clean(caption)++ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231++ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)+ caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)++ caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...++ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)++ caption = re.sub(r"\b\s+\:\s+", r": ", caption)+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)+ caption = re.sub(r"\s+", " ", caption)++ caption.strip()++ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)+ caption = re.sub(r"^\.\S+$", "", caption)++ return caption.strip()+++def text_preprocessing(text, use_text_preprocessing: bool = True):+ if use_text_preprocessing:+ # The exact text cleaning as was in the training stage:+ text = clean_caption(text)+ text = clean_caption(text)+ return text+ else:+ return text.lower().strip()diff --git a/scripts/inference-long.py b/scripts/inference-long.pyindex 0784aef..b6a6e51 100644--- a/scripts/inference-long.py+++ b/scripts/inference-long.py@@ -9,6 +9,7 @@ from mmengine.runner import set_random_seedfrom opensora.acceleration.parallel_states import set_sequence_parallel_groupfrom opensora.datasets import save_samplefrom opensora.datasets.utils import read_from_path+from opensora.models.text_encoder.t5 import text_preprocessingfrom opensora.registry import MODELS, SCHEDULERS, build_modulefrom opensora.utils.config_utils import parse_configsfrom opensora.utils.misc import to_torch_dtype@@ -65,11 +66,13 @@ def process_prompts(prompts, num_loop):for i in range(0, len(prompt_list), 2):start_loop = int(prompt_list[i])text = prompt_list[i + 1]+ text = text_preprocessing(text)end_loop = int(prompt_list[i + 2]) if i + 2 < len(prompt_list) else num_looptext_list.extend([text] * (end_loop - start_loop))assert len(text_list) == num_loopret_prompts.append(text_list)else:+ prompt = text_preprocessing(prompt)ret_prompts.append([prompt] * num_loop)return ret_promptsdiff --git a/scripts/inference.py b/scripts/inference.pyindex e83ff5a..76d1a59 100644--- a/scripts/inference.py+++ b/scripts/inference.py@@ -8,6 +8,7 @@ from mmengine.runner import set_random_seedfrom opensora.acceleration.parallel_states import set_sequence_parallel_groupfrom opensora.datasets import save_sample+from opensora.models.text_encoder.t5 import text_preprocessingfrom opensora.registry import MODELS, SCHEDULERS, build_modulefrom opensora.utils.config_utils import parse_configsfrom opensora.utils.misc import to_torch_dtype@@ -97,6 +98,7 @@ def main():for i in range(0, len(prompts), cfg.batch_size):# 4.2 sample in hidden spacebatch_prompts = prompts[i : i + cfg.batch_size]+ batch_prompts = [text_preprocessing(prompt) for prompt in batch_prompts]z = torch.randn(len(batch_prompts), vae.out_channels, *latent_size, device=device, dtype=dtype)# 4.3. diffusion samplingdiff --git a/tools/datasets/datautil.py b/tools/datasets/datautil.pyindex a014a7b..2b7d17c 100644--- a/tools/datasets/datautil.py+++ b/tools/datasets/datautil.py@@ -74,7 +74,7 @@ def get_info(path):def get_video_info(path):try:vframes, _, _ = torchvision.io.read_video(filename=path, pts_unit="sec", output_format="TCHW")- num_frames, height, width = vframes.shape[0], vframes.shape[1], vframes.shape[2]+ num_frames, height, width = vframes.shape[0], vframes.shape[2], vframes.shape[3]aspect_ratio = height / widthfps = np.nanresolution = height * width@@ -358,6 +358,9 @@ def read_file(input_path):def save_file(data, output_path):+ output_dir = os.path.dirname(output_path)+ if not os.path.exists(output_dir):+ os.makedirs(output_dir)if output_path.endswith(".csv"):return data.to_csv(output_path, index=False)elif output_path.endswith(".parquet"):@@ -476,9 +479,6 @@ def main(args):if args.refine_llm_caption:assert "text" in data.columnsdata["text"] = apply(data["text"], remove_caption_prefix)- if args.unescape:- assert "text" in data.columns- data["text"] = apply(data["text"], html.unescape)if args.clean_caption:assert "text" in data.columnsdata["text"] = apply(@@ -581,7 +581,6 @@ def parse_args():parser.add_argument("--clean-caption", action="store_true", help="modify the caption according to T5 pipeline to suit training")- parser.add_argument("--unescape", action="store_true", help="unescape the caption")parser.add_argument("--merge-cmotion", action="store_true", help="merge the camera motion to the caption")parser.add_argument("--count-num-token", type=str, choices=["t5"], default=None, help="Count the number of tokens in the caption"@@ -644,8 +643,6 @@ def get_output_path(args, input_name):# caption processingif args.refine_llm_caption:name += "_llm"- if args.unescape:- name += "_unescape"if args.clean_caption:name += "_clean"if args.merge_cmotion: