Open-Sora: Democratizing Efficient Video Production for All
Python
[hotfix] fix caption processing (#27)
Hongxin Liuc18ad83f016e
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex 5144ae1..3ce1b2c 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -100,7 +100,11 @@ def build_lang_detector(lang_to_detect):lang_dict = dict(en=Language.ENGLISH)assert lang_to_detect in lang_dictvalid_lang = lang_dict[lang_to_detect]- detector = LanguageDetectorBuilder.from_all_spoken_languages().with_low_accuracy_mode().build()+ detector = (+ LanguageDetectorBuilder.from_all_spoken_languages()+ .with_low_accuracy_mode()+ .build()+ )def detect_lang(caption):confidence_values = detector.compute_language_confidence_values(caption)@@ -121,7 +125,19 @@ def basic_clean(text):BAD_PUNCT_REGEX = re.compile(- r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"+ r"["+ + "#®•©™&@·º½¾¿¡§~"+ + "\)"+ + "\("+ + "\]"+ + "\["+ + "\}"+ + "\{"+ + "\|"+ + "\\"+ + "\/"+ + "\*"+ + r"]{1,}") # noqa@@ -222,10 +238,14 @@ def clean_caption(caption):caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)- caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)+ caption = re.sub(+ r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption+ )caption = re.sub(r"\bpage\s+\d+\b", "", caption)- caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...+ caption = re.sub(+ r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption+ ) # j2d1a2a...caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)@@ -389,10 +409,6 @@ def main(args):if args.ext:assert "path" in data.columnsdata = data[apply(data["path"], os.path.exists)]- if args.remove_empty_caption:- assert "text" in data.columns- data = data[data["text"].str.len() > 0]- data = data[~data["text"].isna()]if args.remove_url:assert "text" in data.columnsdata = data[~data["text"].str.contains(r"(?P<url>https?://[^\s]+)", regex=True)]@@ -417,6 +433,11 @@ def main(args):data["text"],partial(text_preprocessing, use_text_preprocessing=True),)+ # this should be the last step of caption processing+ if args.remove_empty_caption:+ assert "text" in data.columns+ data = data[data["text"].str.len() > 0]+ data = data[~data["text"].isna()]if args.info:info = apply(data["path"], get_video_info)(