Open-Sora: Democratizing Efficient Video Production for All
Python
[feature] move t5 text processing to csvutil (#22)
Hongxin Liuc618f77bdc70
diff --git a/opensora/models/text_encoder/t5.py b/opensora/models/text_encoder/t5.pyindex 1bd535d..807d2ad 100644--- a/opensora/models/text_encoder/t5.py+++ b/opensora/models/text_encoder/t5.py@@ -22,13 +22,7 @@# ---------------------------------------------------------import html-import re-import urllib.parse as ul--import ftfyimport torch-from bs4 import BeautifulSoupfrom transformers import AutoTokenizer, T5EncoderModelfrom opensora.registry import MODELS@@ -36,9 +30,6 @@ from opensora.registry import MODELSclass T5Embedder:available_models = ["DeepFloyd/t5-v1_1-xxl"]- bad_punct_regex = re.compile(- r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"- ) # noqadef __init__(self,@@ -58,7 +49,10 @@ class T5Embedder:self.cache_dir = cache_dirif t5_model_kwargs is None:- t5_model_kwargs = {"low_cpu_mem_usage": True, "torch_dtype": self.torch_dtype}+ t5_model_kwargs = {+ "low_cpu_mem_usage": True,+ "torch_dtype": self.torch_dtype,+ }if use_offload_folder is not None:t5_model_kwargs["offload_folder"] = use_offload_folder@@ -93,19 +87,24 @@ class T5Embedder:"encoder.dropout": "disk",}else:- t5_model_kwargs["device_map"] = {"shared": self.device, "encoder": self.device}+ t5_model_kwargs["device_map"] = {+ "shared": self.device,+ "encoder": self.device,+ }self.use_text_preprocessing = use_text_preprocessingself.hf_token = hf_tokenassert from_pretrained in self.available_models- self.tokenizer = AutoTokenizer.from_pretrained(from_pretrained, cache_dir=cache_dir)- self.model = T5EncoderModel.from_pretrained(from_pretrained, cache_dir=cache_dir, **t5_model_kwargs).eval()+ self.tokenizer = AutoTokenizer.from_pretrained(+ from_pretrained, cache_dir=cache_dir+ )+ self.model = T5EncoderModel.from_pretrained(+ from_pretrained, cache_dir=cache_dir, **t5_model_kwargs+ ).eval()self.model_max_length = model_max_lengthdef get_text_embeddings(self, texts):- texts = [self.text_preprocessing(text) for text in texts]-text_tokens_and_mask = self.tokenizer(texts,max_length=self.model_max_length,@@ -116,143 +115,14 @@ class T5Embedder:return_tensors="pt",)- text_tokens_and_mask["input_ids"] = text_tokens_and_mask["input_ids"]- text_tokens_and_mask["attention_mask"] = text_tokens_and_mask["attention_mask"]-+ input_ids = text_tokens_and_mask["input_ids"].to(self.device)+ attention_mask = text_tokens_and_mask["attention_mask"].to(self.device)with torch.no_grad():text_encoder_embs = self.model(- input_ids=text_tokens_and_mask["input_ids"].to(self.device),- attention_mask=text_tokens_and_mask["attention_mask"].to(self.device),+ input_ids=input_ids,+ attention_mask=attention_mask,)["last_hidden_state"].detach()- return text_encoder_embs, text_tokens_and_mask["attention_mask"].to(self.device)-- def text_preprocessing(self, text):- if self.use_text_preprocessing:- # The exact text cleaning as was in the training stage:- text = self.clean_caption(text)- text = self.clean_caption(text)- return text- else:- return text.lower().strip()-- @staticmethod- def basic_clean(text):- text = ftfy.fix_text(text)- text = html.unescape(html.unescape(text))- return text.strip()-- def clean_caption(self, caption):- caption = str(caption)- caption = ul.unquote_plus(caption)- caption = caption.strip().lower()- caption = re.sub("<person>", "person", caption)- # urls:- caption = re.sub(- r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa- "",- caption,- ) # regex for urls- caption = re.sub(- r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa- "",- caption,- ) # regex for urls- # html:- caption = BeautifulSoup(caption, features="html.parser").text-- # @<nickname>- caption = re.sub(r"@[\w\d]+\b", "", caption)-- # 31C0—31EF CJK Strokes- # 31F0—31FF Katakana Phonetic Extensions- # 3200—32FF Enclosed CJK Letters and Months- # 3300—33FF CJK Compatibility- # 3400—4DBF CJK Unified Ideographs Extension A- # 4DC0—4DFF Yijing Hexagram Symbols- # 4E00—9FFF CJK Unified Ideographs- caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)- caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)- caption = re.sub(r"[\u3200-\u32ff]+", "", caption)- caption = re.sub(r"[\u3300-\u33ff]+", "", caption)- caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)- caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)- caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)- #######################################################-- # все виды тире / all types of dash --> "-"- caption = re.sub(- r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa- "-",- caption,- )-- # кавычки к одному стандарту- caption = re.sub(r"[`´«»“”¨]", '"', caption)- caption = re.sub(r"[‘’]", "'", caption)-- # "- caption = re.sub(r""?", "", caption)- # &- caption = re.sub(r"&", "", caption)-- # ip adresses:- caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)-- # article ids:- caption = re.sub(r"\d:\d\d\s+$", "", caption)-- # \n- caption = re.sub(r"\\n", " ", caption)-- # "#123"- caption = re.sub(r"#\d{1,3}\b", "", caption)- # "#12345.."- caption = re.sub(r"#\d{5,}\b", "", caption)- # "123456.."- caption = re.sub(r"\b\d{6,}\b", "", caption)- # filenames:- caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)-- #- caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""- caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""-- caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT- caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "-- # this-is-my-cute-cat / this_is_my_cute_cat- regex2 = re.compile(r"(?:\-|\_)")- if len(re.findall(regex2, caption)) > 3:- caption = re.sub(regex2, " ", caption)-- caption = self.basic_clean(caption)-- caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640- caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc- caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231-- caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)- caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)- caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)- caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)- caption = re.sub(r"\bpage\s+\d+\b", "", caption)-- caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...-- caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)-- caption = re.sub(r"\b\s+\:\s+", r": ", caption)- caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)- caption = re.sub(r"\s+", " ", caption)-- caption.strip()-- caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)- caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)- caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)- caption = re.sub(r"^\.\S+$", "", caption)-- return caption.strip()+ return text_encoder_embs, attention_mask@MODELS.register_module("t5")diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex ba68d42..a7002ff 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -1,6 +1,8 @@import argparseimport htmlimport os+import re+from functools import partialfrom glob import globimport numpy as np@@ -24,7 +26,17 @@ def apply(df, func):return df.progress_apply(func)-IMG_EXTENSIONS = (".jpg", ".jpeg", ".png", ".ppm", ".bmp", ".pgm", ".tif", ".tiff", ".webp")+IMG_EXTENSIONS = (+ ".jpg",+ ".jpeg",+ ".png",+ ".ppm",+ ".bmp",+ ".pgm",+ ".tif",+ ".tiff",+ ".webp",+)def get_video_info(path):@@ -87,7 +99,11 @@ def build_lang_detector(lang_to_detect):lang_dict = dict(en=Language.ENGLISH)assert lang_to_detect in lang_dictvalid_lang = lang_dict[lang_to_detect]- detector = LanguageDetectorBuilder.from_all_spoken_languages().with_low_accuracy_mode().build()+ detector = (+ LanguageDetectorBuilder.from_all_spoken_languages()+ .with_low_accuracy_mode()+ .build()+ )def detect_lang(caption):confidence_values = detector.compute_language_confidence_values(caption)@@ -99,6 +115,163 @@ def build_lang_detector(lang_to_detect):return detect_lang+def basic_clean(text):+ import ftfy++ text = ftfy.fix_text(text)+ text = html.unescape(html.unescape(text))+ return text.strip()+++BAD_PUNCT_REGEX = re.compile(+ r"["+ + "#®•©™&@·º½¾¿¡§~"+ + "\)"+ + "\("+ + "\]"+ + "\["+ + "\}"+ + "\{"+ + "\|"+ + "\\"+ + "\/"+ + "\*"+ + r"]{1,}"+) # noqa+++def clean_caption(caption):+ import urllib.parse as ul++ from bs4 import BeautifulSoup++ caption = str(caption)+ caption = ul.unquote_plus(caption)+ caption = caption.strip().lower()+ caption = re.sub("<person>", "person", caption)+ # urls:+ caption = re.sub(+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ caption = re.sub(+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ # html:+ caption = BeautifulSoup(caption, features="html.parser").text++ # @<nickname>+ caption = re.sub(r"@[\w\d]+\b", "", caption)++ # 31C0—31EF CJK Strokes+ # 31F0—31FF Katakana Phonetic Extensions+ # 3200—32FF Enclosed CJK Letters and Months+ # 3300—33FF CJK Compatibility+ # 3400—4DBF CJK Unified Ideographs Extension A+ # 4DC0—4DFF Yijing Hexagram Symbols+ # 4E00—9FFF CJK Unified Ideographs+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)+ #######################################################++ # все виды тире / all types of dash --> "-"+ caption = re.sub(+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa+ "-",+ caption,+ )++ # кавычки к одному стандарту+ caption = re.sub(r"[`´«»“”¨]", '"', caption)+ caption = re.sub(r"[‘’]", "'", caption)++ # "+ caption = re.sub(r""?", "", caption)+ # &+ caption = re.sub(r"&", "", caption)++ # ip adresses:+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)++ # article ids:+ caption = re.sub(r"\d:\d\d\s+$", "", caption)++ # \n+ caption = re.sub(r"\\n", " ", caption)++ # "#123"+ caption = re.sub(r"#\d{1,3}\b", "", caption)+ # "#12345.."+ caption = re.sub(r"#\d{5,}\b", "", caption)+ # "123456.."+ caption = re.sub(r"\b\d{6,}\b", "", caption)+ # filenames:+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)++ #+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""++ caption = re.sub(BAD_PUNCT_REGEX, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "++ # this-is-my-cute-cat / this_is_my_cute_cat+ regex2 = re.compile(r"(?:\-|\_)")+ if len(re.findall(regex2, caption)) > 3:+ caption = re.sub(regex2, " ", caption)++ caption = basic_clean(caption)++ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231++ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)+ caption = re.sub(+ r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption+ )+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)++ caption = re.sub(+ r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption+ ) # j2d1a2a...++ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)++ caption = re.sub(r"\b\s+\:\s+", r": ", caption)+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)+ caption = re.sub(r"\s+", " ", caption)++ caption.strip()++ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)+ caption = re.sub(r"^\.\S+$", "", caption)++ return caption.strip()+++def text_preprocessing(text, use_text_preprocessing: bool = True):+ if use_text_preprocessing:+ # The exact text cleaning as was in the training stage:+ text = clean_caption(text)+ text = clean_caption(text)+ return text+ else:+ return text.lower().strip()++def parse_args():parser = argparse.ArgumentParser()parser.add_argument("input", type=str, nargs="+")@@ -123,6 +296,7 @@ def parse_args():# caption processingparser.add_argument("--remove-caption-prefix", action="store_true")parser.add_argument("--unescape", action="store_true")+ parser.add_argument("--clean-caption", action="store_true")# num_frames processingparser.add_argument("--info", action="store_true")# num_frames filtering@@ -162,6 +336,8 @@ def get_output_path(args, input_name):name += "_rcp"if args.unescape:name += "_unescape"+ if args.clean_caption:+ name += "_clean"# num_frames processingif args.info:name += "_info"@@ -254,9 +430,20 @@ def main(args):if args.unescape:assert "text" in data.columnsdata["text"] = apply(data["text"], html.unescape)+ if "text" in data.columns:+ data["text"] = apply(+ data["text"],+ partial(text_preprocessing, use_text_preprocessing=args.clean_caption),+ )if args.info:info = apply(data["path"], get_video_info)- data["num_frames"], data["height"], data["width"], data["aspect_ratio"], data["fps"] = zip(*info)+ (+ data["num_frames"],+ data["height"],+ data["width"],+ data["aspect_ratio"],+ data["fps"],+ ) = zip(*info)# filteringif args.fmin is not None: