Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[feature] move t5 text processing to csvutil (#22)

Hongxin Liuc618f77bdc70
diff --git a/opensora/models/text_encoder/t5.py b/opensora/models/text_encoder/t5.py
index 1bd535d..807d2ad 100644
--- a/opensora/models/text_encoder/t5.py
+++ b/opensora/models/text_encoder/t5.py
@@ -22,13 +22,7 @@
# --------------------------------------------------------
-import html
-import re
-import urllib.parse as ul
-
-import ftfy
import torch
-from bs4 import BeautifulSoup
from transformers import AutoTokenizer, T5EncoderModel
from opensora.registry import MODELS
@@ -36,9 +30,6 @@ from opensora.registry import MODELS
class T5Embedder:
available_models = ["DeepFloyd/t5-v1_1-xxl"]
- bad_punct_regex = re.compile(
- r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"
- ) # noqa
def __init__(
self,
@@ -58,7 +49,10 @@ class T5Embedder:
self.cache_dir = cache_dir
if t5_model_kwargs is None:
- t5_model_kwargs = {"low_cpu_mem_usage": True, "torch_dtype": self.torch_dtype}
+ t5_model_kwargs = {
+ "low_cpu_mem_usage": True,
+ "torch_dtype": self.torch_dtype,
+ }
if use_offload_folder is not None:
t5_model_kwargs["offload_folder"] = use_offload_folder
@@ -93,19 +87,24 @@ class T5Embedder:
"encoder.dropout": "disk",
}
else:
- t5_model_kwargs["device_map"] = {"shared": self.device, "encoder": self.device}
+ t5_model_kwargs["device_map"] = {
+ "shared": self.device,
+ "encoder": self.device,
+ }
self.use_text_preprocessing = use_text_preprocessing
self.hf_token = hf_token
assert from_pretrained in self.available_models
- self.tokenizer = AutoTokenizer.from_pretrained(from_pretrained, cache_dir=cache_dir)
- self.model = T5EncoderModel.from_pretrained(from_pretrained, cache_dir=cache_dir, **t5_model_kwargs).eval()
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ from_pretrained, cache_dir=cache_dir
+ )
+ self.model = T5EncoderModel.from_pretrained(
+ from_pretrained, cache_dir=cache_dir, **t5_model_kwargs
+ ).eval()
self.model_max_length = model_max_length
def get_text_embeddings(self, texts):
- texts = [self.text_preprocessing(text) for text in texts]
-
text_tokens_and_mask = self.tokenizer(
texts,
max_length=self.model_max_length,
@@ -116,143 +115,14 @@ class T5Embedder:
return_tensors="pt",
)
- text_tokens_and_mask["input_ids"] = text_tokens_and_mask["input_ids"]
- text_tokens_and_mask["attention_mask"] = text_tokens_and_mask["attention_mask"]
-
+ input_ids = text_tokens_and_mask["input_ids"].to(self.device)
+ attention_mask = text_tokens_and_mask["attention_mask"].to(self.device)
with torch.no_grad():
text_encoder_embs = self.model(
- input_ids=text_tokens_and_mask["input_ids"].to(self.device),
- attention_mask=text_tokens_and_mask["attention_mask"].to(self.device),
+ input_ids=input_ids,
+ attention_mask=attention_mask,
)["last_hidden_state"].detach()
- return text_encoder_embs, text_tokens_and_mask["attention_mask"].to(self.device)
-
- def text_preprocessing(self, text):
- if self.use_text_preprocessing:
- # The exact text cleaning as was in the training stage:
- text = self.clean_caption(text)
- text = self.clean_caption(text)
- return text
- else:
- return text.lower().strip()
-
- @staticmethod
- def basic_clean(text):
- text = ftfy.fix_text(text)
- text = html.unescape(html.unescape(text))
- return text.strip()
-
- def clean_caption(self, caption):
- caption = str(caption)
- caption = ul.unquote_plus(caption)
- caption = caption.strip().lower()
- caption = re.sub("<person>", "person", caption)
- # urls:
- caption = re.sub(
- r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
- "",
- caption,
- ) # regex for urls
- caption = re.sub(
- r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
- "",
- caption,
- ) # regex for urls
- # html:
- caption = BeautifulSoup(caption, features="html.parser").text
-
- # @<nickname>
- caption = re.sub(r"@[\w\d]+\b", "", caption)
-
- # 31C0—31EF CJK Strokes
- # 31F0—31FF Katakana Phonetic Extensions
- # 3200—32FF Enclosed CJK Letters and Months
- # 3300—33FF CJK Compatibility
- # 3400—4DBF CJK Unified Ideographs Extension A
- # 4DC0—4DFF Yijing Hexagram Symbols
- # 4E00—9FFF CJK Unified Ideographs
- caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)
- caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)
- caption = re.sub(r"[\u3200-\u32ff]+", "", caption)
- caption = re.sub(r"[\u3300-\u33ff]+", "", caption)
- caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)
- caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)
- caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)
- #######################################################
-
- # все виды тире / all types of dash --> "-"
- caption = re.sub(
- r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa
- "-",
- caption,
- )
-
- # кавычки к одному стандарту
- caption = re.sub(r"[`´«»“”¨]", '"', caption)
- caption = re.sub(r"[‘’]", "'", caption)
-
- # &quot;
- caption = re.sub(r"&quot;?", "", caption)
- # &amp
- caption = re.sub(r"&amp", "", caption)
-
- # ip adresses:
- caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)
-
- # article ids:
- caption = re.sub(r"\d:\d\d\s+$", "", caption)
-
- # \n
- caption = re.sub(r"\\n", " ", caption)
-
- # "#123"
- caption = re.sub(r"#\d{1,3}\b", "", caption)
- # "#12345.."
- caption = re.sub(r"#\d{5,}\b", "", caption)
- # "123456.."
- caption = re.sub(r"\b\d{6,}\b", "", caption)
- # filenames:
- caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)
-
- #
- caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""
- caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""
-
- caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT
- caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "
-
- # this-is-my-cute-cat / this_is_my_cute_cat
- regex2 = re.compile(r"(?:\-|\_)")
- if len(re.findall(regex2, caption)) > 3:
- caption = re.sub(regex2, " ", caption)
-
- caption = self.basic_clean(caption)
-
- caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640
- caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc
- caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231
-
- caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)
- caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)
- caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)
- caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)
- caption = re.sub(r"\bpage\s+\d+\b", "", caption)
-
- caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...
-
- caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)
-
- caption = re.sub(r"\b\s+\:\s+", r": ", caption)
- caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)
- caption = re.sub(r"\s+", " ", caption)
-
- caption.strip()
-
- caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)
- caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)
- caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)
- caption = re.sub(r"^\.\S+$", "", caption)
-
- return caption.strip()
+ return text_encoder_embs, attention_mask
@MODELS.register_module("t5")
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.py
index ba68d42..a7002ff 100644
--- a/tools/datasets/csvutil.py
+++ b/tools/datasets/csvutil.py
@@ -1,6 +1,8 @@
import argparse
import html
import os
+import re
+from functools import partial
from glob import glob
import numpy as np
@@ -24,7 +26,17 @@ def apply(df, func):
return df.progress_apply(func)
-IMG_EXTENSIONS = (".jpg", ".jpeg", ".png", ".ppm", ".bmp", ".pgm", ".tif", ".tiff", ".webp")
+IMG_EXTENSIONS = (
+ ".jpg",
+ ".jpeg",
+ ".png",
+ ".ppm",
+ ".bmp",
+ ".pgm",
+ ".tif",
+ ".tiff",
+ ".webp",
+)
def get_video_info(path):
@@ -87,7 +99,11 @@ def build_lang_detector(lang_to_detect):
lang_dict = dict(en=Language.ENGLISH)
assert lang_to_detect in lang_dict
valid_lang = lang_dict[lang_to_detect]
- detector = LanguageDetectorBuilder.from_all_spoken_languages().with_low_accuracy_mode().build()
+ detector = (
+ LanguageDetectorBuilder.from_all_spoken_languages()
+ .with_low_accuracy_mode()
+ .build()
+ )
def detect_lang(caption):
confidence_values = detector.compute_language_confidence_values(caption)
@@ -99,6 +115,163 @@ def build_lang_detector(lang_to_detect):
return detect_lang
+def basic_clean(text):
+ import ftfy
+
+ text = ftfy.fix_text(text)
+ text = html.unescape(html.unescape(text))
+ return text.strip()
+
+
+BAD_PUNCT_REGEX = re.compile(
+ r"["
+ + "#®•©™&@·º½¾¿¡§~"
+ + "\)"
+ + "\("
+ + "\]"
+ + "\["
+ + "\}"
+ + "\{"
+ + "\|"
+ + "\\"
+ + "\/"
+ + "\*"
+ + r"]{1,}"
+) # noqa
+
+
+def clean_caption(caption):
+ import urllib.parse as ul
+
+ from bs4 import BeautifulSoup
+
+ caption = str(caption)
+ caption = ul.unquote_plus(caption)
+ caption = caption.strip().lower()
+ caption = re.sub("<person>", "person", caption)
+ # urls:
+ caption = re.sub(
+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
+ "",
+ caption,
+ ) # regex for urls
+ caption = re.sub(
+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
+ "",
+ caption,
+ ) # regex for urls
+ # html:
+ caption = BeautifulSoup(caption, features="html.parser").text
+
+ # @<nickname>
+ caption = re.sub(r"@[\w\d]+\b", "", caption)
+
+ # 31C0—31EF CJK Strokes
+ # 31F0—31FF Katakana Phonetic Extensions
+ # 3200—32FF Enclosed CJK Letters and Months
+ # 3300—33FF CJK Compatibility
+ # 3400—4DBF CJK Unified Ideographs Extension A
+ # 4DC0—4DFF Yijing Hexagram Symbols
+ # 4E00—9FFF CJK Unified Ideographs
+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)
+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)
+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)
+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)
+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)
+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)
+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)
+ #######################################################
+
+ # все виды тире / all types of dash --> "-"
+ caption = re.sub(
+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa
+ "-",
+ caption,
+ )
+
+ # кавычки к одному стандарту
+ caption = re.sub(r"[`´«»“”¨]", '"', caption)
+ caption = re.sub(r"[‘’]", "'", caption)
+
+ # &quot;
+ caption = re.sub(r"&quot;?", "", caption)
+ # &amp
+ caption = re.sub(r"&amp", "", caption)
+
+ # ip adresses:
+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)
+
+ # article ids:
+ caption = re.sub(r"\d:\d\d\s+$", "", caption)
+
+ # \n
+ caption = re.sub(r"\\n", " ", caption)
+
+ # "#123"
+ caption = re.sub(r"#\d{1,3}\b", "", caption)
+ # "#12345.."
+ caption = re.sub(r"#\d{5,}\b", "", caption)
+ # "123456.."
+ caption = re.sub(r"\b\d{6,}\b", "", caption)
+ # filenames:
+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)
+
+ #
+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""
+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""
+
+ caption = re.sub(BAD_PUNCT_REGEX, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT
+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "
+
+ # this-is-my-cute-cat / this_is_my_cute_cat
+ regex2 = re.compile(r"(?:\-|\_)")
+ if len(re.findall(regex2, caption)) > 3:
+ caption = re.sub(regex2, " ", caption)
+
+ caption = basic_clean(caption)
+
+ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640
+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc
+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231
+
+ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)
+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)
+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)
+ caption = re.sub(
+ r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption
+ )
+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)
+
+ caption = re.sub(
+ r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption
+ ) # j2d1a2a...
+
+ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)
+
+ caption = re.sub(r"\b\s+\:\s+", r": ", caption)
+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)
+ caption = re.sub(r"\s+", " ", caption)
+
+ caption.strip()
+
+ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)
+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)
+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)
+ caption = re.sub(r"^\.\S+$", "", caption)
+
+ return caption.strip()
+
+
+def text_preprocessing(text, use_text_preprocessing: bool = True):
+ if use_text_preprocessing:
+ # The exact text cleaning as was in the training stage:
+ text = clean_caption(text)
+ text = clean_caption(text)
+ return text
+ else:
+ return text.lower().strip()
+
+
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("input", type=str, nargs="+")
@@ -123,6 +296,7 @@ def parse_args():
# caption processing
parser.add_argument("--remove-caption-prefix", action="store_true")
parser.add_argument("--unescape", action="store_true")
+ parser.add_argument("--clean-caption", action="store_true")
# num_frames processing
parser.add_argument("--info", action="store_true")
# num_frames filtering
@@ -162,6 +336,8 @@ def get_output_path(args, input_name):
name += "_rcp"
if args.unescape:
name += "_unescape"
+ if args.clean_caption:
+ name += "_clean"
# num_frames processing
if args.info:
name += "_info"
@@ -254,9 +430,20 @@ def main(args):
if args.unescape:
assert "text" in data.columns
data["text"] = apply(data["text"], html.unescape)
+ if "text" in data.columns:
+ data["text"] = apply(
+ data["text"],
+ partial(text_preprocessing, use_text_preprocessing=args.clean_caption),
+ )
if args.info:
info = apply(data["path"], get_video_info)
- data["num_frames"], data["height"], data["width"], data["aspect_ratio"], data["fps"] = zip(*info)
+ (
+ data["num_frames"],
+ data["height"],
+ data["width"],
+ data["aspect_ratio"],
+ data["fps"],
+ ) = zip(*info)
# filtering
if args.fmin is not None: