Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

hotfix inference

Zangwei Zheng63e86f6dd651
diff --git a/configs/opensora-v1-1/train/stage1.py b/configs/opensora-v1-1/train/stage1.py
new file mode 100644
index 0000000..2c06757
--- /dev/null
+++ b/configs/opensora-v1-1/train/stage1.py
@@ -0,0 +1,70 @@
+# Define dataset
+dataset = dict(
+ type="VariableVideoTextDataset",
+ data_path=None,
+ num_frames=None,
+ frame_interval=3,
+ image_size=(None, None),
+ transform_name="resize_crop",
+)
+bucket_config = { # 6s/it
+ "240p": {16: (1.0, 17), 32: (1.0, 9), 64: (1.0, 4), 128: (1.0, 2)},
+ "256": {1: (1.0, 254)},
+ "512": {1: (0.5, 86)},
+ "480p": {1: (0.4, 54), 16: (0.4, 4), 32: (0.0, None)},
+ "720p": {16: (0.1, 2), 32: (0.0, None)}, # No examples now
+ "1024": {1: (0.3, 20)},
+ "1080p": {1: (0.4, 8)},
+}
+mask_ratios = {
+ "mask_no": 0.9,
+ "mask_random": 0.06,
+ "mask_head": 0.01,
+ "mask_tail": 0.01,
+ "mask_head_tail": 0.02,
+}
+
+# Define acceleration
+num_workers = 4
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+# Define model
+model = dict(
+ type="STDiT2-XL/2",
+ from_pretrained=None,
+ input_sq_size=512, # pretrained model is trained on 512x512
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+)
+vae = dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="stabilityai/sd-vae-ft-ema",
+ micro_batch_size=4,
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=200,
+ shardformer=True,
+)
+scheduler = dict(
+ type="iddpm-speed",
+ timestep_respacing="",
+)
+
+# Others
+seed = 42
+outputs = "outputs"
+wandb = False
+
+epochs = 1000
+log_every = 10
+ckpt_every = 500
+load = None
+
+batch_size = 17 # only for logging
+lr = 2e-5
+grad_clip = 1.0
diff --git a/opensora/models/text_encoder/t5.py b/opensora/models/text_encoder/t5.py
index 807d2ad..7364eb9 100644
--- a/opensora/models/text_encoder/t5.py
+++ b/opensora/models/text_encoder/t5.py
@@ -21,7 +21,10 @@
# T5: https://github.com/google-research/text-to-text-transfer-transformer
# --------------------------------------------------------
+import html
+import re
+import ftfy
import torch
from transformers import AutoTokenizer, T5EncoderModel
@@ -96,12 +99,8 @@ class T5Embedder:
self.hf_token = hf_token
assert from_pretrained in self.available_models
- self.tokenizer = AutoTokenizer.from_pretrained(
- from_pretrained, cache_dir=cache_dir
- )
- self.model = T5EncoderModel.from_pretrained(
- from_pretrained, cache_dir=cache_dir, **t5_model_kwargs
- ).eval()
+ self.tokenizer = AutoTokenizer.from_pretrained(from_pretrained, cache_dir=cache_dir)
+ self.model = T5EncoderModel.from_pretrained(from_pretrained, cache_dir=cache_dir, **t5_model_kwargs).eval()
self.model_max_length = model_max_length
def get_text_embeddings(self, texts):
@@ -185,3 +184,142 @@ class T5Encoder:
def null(self, n):
null_y = self.y_embedder.y_embedding[None].repeat(n, 1, 1)[:, None]
return null_y
+
+
+def basic_clean(text):
+ text = ftfy.fix_text(text)
+ text = html.unescape(html.unescape(text))
+ return text.strip()
+
+
+BAD_PUNCT_REGEX = re.compile(
+ r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"
+) # noqa
+
+
+def clean_caption(caption):
+ import urllib.parse as ul
+
+ from bs4 import BeautifulSoup
+
+ caption = str(caption)
+ caption = ul.unquote_plus(caption)
+ caption = caption.strip().lower()
+ caption = re.sub("<person>", "person", caption)
+ # urls:
+ caption = re.sub(
+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
+ "",
+ caption,
+ ) # regex for urls
+ caption = re.sub(
+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
+ "",
+ caption,
+ ) # regex for urls
+ # html:
+ caption = BeautifulSoup(caption, features="html.parser").text
+
+ # @<nickname>
+ caption = re.sub(r"@[\w\d]+\b", "", caption)
+
+ # 31C0—31EF CJK Strokes
+ # 31F0—31FF Katakana Phonetic Extensions
+ # 3200—32FF Enclosed CJK Letters and Months
+ # 3300—33FF CJK Compatibility
+ # 3400—4DBF CJK Unified Ideographs Extension A
+ # 4DC0—4DFF Yijing Hexagram Symbols
+ # 4E00—9FFF CJK Unified Ideographs
+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)
+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)
+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)
+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)
+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)
+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)
+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)
+ #######################################################
+
+ # все виды тире / all types of dash --> "-"
+ caption = re.sub(
+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa
+ "-",
+ caption,
+ )
+
+ # кавычки к одному стандарту
+ caption = re.sub(r"[`´«»“”¨]", '"', caption)
+ caption = re.sub(r"[‘’]", "'", caption)
+
+ # &quot;
+ caption = re.sub(r"&quot;?", "", caption)
+ # &amp
+ caption = re.sub(r"&amp", "", caption)
+
+ # ip adresses:
+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)
+
+ # article ids:
+ caption = re.sub(r"\d:\d\d\s+$", "", caption)
+
+ # \n
+ caption = re.sub(r"\\n", " ", caption)
+
+ # "#123"
+ caption = re.sub(r"#\d{1,3}\b", "", caption)
+ # "#12345.."
+ caption = re.sub(r"#\d{5,}\b", "", caption)
+ # "123456.."
+ caption = re.sub(r"\b\d{6,}\b", "", caption)
+ # filenames:
+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)
+
+ #
+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""
+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""
+
+ caption = re.sub(BAD_PUNCT_REGEX, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT
+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "
+
+ # this-is-my-cute-cat / this_is_my_cute_cat
+ regex2 = re.compile(r"(?:\-|\_)")
+ if len(re.findall(regex2, caption)) > 3:
+ caption = re.sub(regex2, " ", caption)
+
+ caption = basic_clean(caption)
+
+ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640
+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc
+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231
+
+ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)
+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)
+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)
+ caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)
+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)
+
+ caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...
+
+ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)
+
+ caption = re.sub(r"\b\s+\:\s+", r": ", caption)
+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)
+ caption = re.sub(r"\s+", " ", caption)
+
+ caption.strip()
+
+ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)
+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)
+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)
+ caption = re.sub(r"^\.\S+$", "", caption)
+
+ return caption.strip()
+
+
+def text_preprocessing(text, use_text_preprocessing: bool = True):
+ if use_text_preprocessing:
+ # The exact text cleaning as was in the training stage:
+ text = clean_caption(text)
+ text = clean_caption(text)
+ return text
+ else:
+ return text.lower().strip()
diff --git a/scripts/inference-long.py b/scripts/inference-long.py
index 0784aef..b6a6e51 100644
--- a/scripts/inference-long.py
+++ b/scripts/inference-long.py
@@ -9,6 +9,7 @@ from mmengine.runner import set_random_seed
from opensora.acceleration.parallel_states import set_sequence_parallel_group
from opensora.datasets import save_sample
from opensora.datasets.utils import read_from_path
+from opensora.models.text_encoder.t5 import text_preprocessing
from opensora.registry import MODELS, SCHEDULERS, build_module
from opensora.utils.config_utils import parse_configs
from opensora.utils.misc import to_torch_dtype
@@ -65,11 +66,13 @@ def process_prompts(prompts, num_loop):
for i in range(0, len(prompt_list), 2):
start_loop = int(prompt_list[i])
text = prompt_list[i + 1]
+ text = text_preprocessing(text)
end_loop = int(prompt_list[i + 2]) if i + 2 < len(prompt_list) else num_loop
text_list.extend([text] * (end_loop - start_loop))
assert len(text_list) == num_loop
ret_prompts.append(text_list)
else:
+ prompt = text_preprocessing(prompt)
ret_prompts.append([prompt] * num_loop)
return ret_prompts
diff --git a/scripts/inference.py b/scripts/inference.py
index e83ff5a..76d1a59 100644
--- a/scripts/inference.py
+++ b/scripts/inference.py
@@ -8,6 +8,7 @@ from mmengine.runner import set_random_seed
from opensora.acceleration.parallel_states import set_sequence_parallel_group
from opensora.datasets import save_sample
+from opensora.models.text_encoder.t5 import text_preprocessing
from opensora.registry import MODELS, SCHEDULERS, build_module
from opensora.utils.config_utils import parse_configs
from opensora.utils.misc import to_torch_dtype
@@ -97,6 +98,7 @@ def main():
for i in range(0, len(prompts), cfg.batch_size):
# 4.2 sample in hidden space
batch_prompts = prompts[i : i + cfg.batch_size]
+ batch_prompts = [text_preprocessing(prompt) for prompt in batch_prompts]
z = torch.randn(len(batch_prompts), vae.out_channels, *latent_size, device=device, dtype=dtype)
# 4.3. diffusion sampling
diff --git a/tools/datasets/datautil.py b/tools/datasets/datautil.py
index a014a7b..2b7d17c 100644
--- a/tools/datasets/datautil.py
+++ b/tools/datasets/datautil.py
@@ -74,7 +74,7 @@ def get_info(path):
def get_video_info(path):
try:
vframes, _, _ = torchvision.io.read_video(filename=path, pts_unit="sec", output_format="TCHW")
- num_frames, height, width = vframes.shape[0], vframes.shape[1], vframes.shape[2]
+ num_frames, height, width = vframes.shape[0], vframes.shape[2], vframes.shape[3]
aspect_ratio = height / width
fps = np.nan
resolution = height * width
@@ -358,6 +358,9 @@ def read_file(input_path):
def save_file(data, output_path):
+ output_dir = os.path.dirname(output_path)
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
if output_path.endswith(".csv"):
return data.to_csv(output_path, index=False)
elif output_path.endswith(".parquet"):
@@ -476,9 +479,6 @@ def main(args):
if args.refine_llm_caption:
assert "text" in data.columns
data["text"] = apply(data["text"], remove_caption_prefix)
- if args.unescape:
- assert "text" in data.columns
- data["text"] = apply(data["text"], html.unescape)
if args.clean_caption:
assert "text" in data.columns
data["text"] = apply(
@@ -581,7 +581,6 @@ def parse_args():
parser.add_argument(
"--clean-caption", action="store_true", help="modify the caption according to T5 pipeline to suit training"
)
- parser.add_argument("--unescape", action="store_true", help="unescape the caption")
parser.add_argument("--merge-cmotion", action="store_true", help="merge the camera motion to the caption")
parser.add_argument(
"--count-num-token", type=str, choices=["t5"], default=None, help="Count the number of tokens in the caption"
@@ -644,8 +643,6 @@ def get_output_path(args, input_name):
# caption processing
if args.refine_llm_caption:
name += "_llm"
- if args.unescape:
- name += "_unescape"
if args.clean_caption:
name += "_clean"
if args.merge_cmotion: