Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[fix] update inference and datautil

zhengzangw6194da650459
diff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.py
index cee23dd..8b61c83 100644
--- a/opensora/utils/config_utils.py
+++ b/opensora/utils/config_utils.py
@@ -7,12 +7,6 @@ from mmengine.config import Config
from torch.utils.tensorboard import SummaryWriter
-def load_prompts(prompt_path):
- with open(prompt_path, "r") as f:
- prompts = [line.strip() for line in f.readlines()]
- return prompts
-
-
def parse_args(training=False):
parser = argparse.ArgumentParser()
@@ -86,11 +80,13 @@ def merge_args(cfg, args, training=False):
if args.data_path is not None:
cfg.dataset["data_path"] = args.data_path
args.data_path = None
- if not training:
- if args.image_size is not None and "dataset" in cfg:
+ # NOTE: for vae inference (reconstruction)
+ if not training and "dataset" in cfg:
+ if args.image_size is not None:
cfg.dataset["image_size"] = args.image_size
- if args.num_frames is not None and "dataset" in cfg:
+ if args.num_frames is not None:
cfg.dataset["num_frames"] = args.num_frames
+ if not training:
if args.cfg_scale is not None:
cfg.scheduler["cfg_scale"] = args.cfg_scale
args.cfg_scale = None
@@ -103,32 +99,10 @@ def merge_args(cfg, args, training=False):
cfg[k] = v
if not training:
- # Inference only
- # - Allow not set
if "reference_path" not in cfg:
cfg["reference_path"] = None
if "loop" not in cfg:
cfg["loop"] = 1
- if "frame_interval" not in cfg:
- cfg["frame_interval"] = 1
- if "sample_name" not in cfg:
- cfg["sample_name"] = None
- if "num_sample" not in cfg:
- cfg["num_sample"] = 1
- if "prompt_as_path" not in cfg:
- cfg["prompt_as_path"] = False
- # - Prompt handling
- if "prompt" not in cfg or cfg["prompt"] is None:
- if ("prompt" not in cfg or cfg["prompt"] is None) and cfg.get("prompt_path", None) is not None:
- cfg["prompt"] = load_prompts(cfg["prompt_path"])
- if args.start_index is not None and args.end_index is not None:
- cfg["prompt"] = cfg["prompt"][args.start_index : args.end_index]
- elif args.start_index is not None:
- cfg["prompt"] = cfg["prompt"][args.start_index :]
- elif args.end_index is not None:
- cfg["prompt"] = cfg["prompt"][: args.end_index]
- if "multi_resolution" not in cfg:
- cfg["multi_resolution"] = False
return cfg
diff --git a/opensora/utils/inference_utils.py b/opensora/utils/inference_utils.py
new file mode 100644
index 0000000..48792b4
--- /dev/null
+++ b/opensora/utils/inference_utils.py
@@ -0,0 +1,49 @@
+import os
+
+import torch
+
+from opensora.datasets import IMG_FPS
+
+
+def prepare_multi_resolution_info(info_type, batch_size, image_size, num_frames, fps, device, dtype):
+ if info_type is None:
+ return dict()
+ elif info_type == "PixArtMS":
+ hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(batch_size, 1)
+ ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(batch_size, 1)
+ return dict(ar=ar, hw=hw)
+ elif info_type in ["STDiT2", "OpenSora"]:
+ height = torch.tensor([image_size[0]], device=device, dtype=dtype).repeat(batch_size)
+ width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(batch_size)
+ num_frames = torch.tensor([num_frames], device=device, dtype=dtype).repeat(batch_size)
+ ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(batch_size)
+ fps = fps if num_frames > 1 else IMG_FPS
+ fps = torch.tensor([fps], device=device, dtype=dtype).repeat(batch_size)
+ return dict(height=height, width=width, num_frames=num_frames, ar=ar, fps=fps)
+ else:
+ raise NotImplementedError
+
+
+def load_prompts(prompt_path, start_idx=None, end_idx=None):
+ with open(prompt_path, "r") as f:
+ prompts = [line.strip() for line in f.readlines()]
+ prompts = prompts[start_idx:end_idx]
+ return prompts
+
+
+def get_save_path_name(
+ save_dir,
+ sample_name=None, # prefix
+ sample_idx=None, # sample index
+ prompt=None, # used prompt
+ prompt_as_path=False, # use prompt as path
+ num_sample=1, # number of samples to generate for one prompt
+ k=None, # kth sample
+):
+ if sample_name is None:
+ sample_name = "" if prompt_as_path else "sample"
+ sample_name_suffix = prompt if prompt_as_path else f"_{sample_idx}"
+ save_path = os.path.join(save_dir, f"{sample_name}{sample_name_suffix}")
+ if num_sample != 1:
+ save_path = f"{save_path}-{k}"
+ return save_path
diff --git a/opensora/utils/misc.py b/opensora/utils/misc.py
index e492bdd..d191581 100644
--- a/opensora/utils/misc.py
+++ b/opensora/utils/misc.py
@@ -318,3 +318,7 @@ def transpose(x):
"""
ret = list(map(list, zip(*x)))
return ret
+
+
+def all_exists(paths):
+ return all(os.path.exists(path) for path in paths)
diff --git a/scripts/inference.py b/scripts/inference.py
index a839ca5..2887fce 100644
--- a/scripts/inference.py
+++ b/scripts/inference.py
@@ -1,4 +1,5 @@
import os
+from pprint import pformat
import colossalai
import torch
@@ -8,17 +9,18 @@ from mmengine.runner import set_random_seed
from tqdm import tqdm
from opensora.acceleration.parallel_states import set_sequence_parallel_group
-from opensora.datasets import IMG_FPS, save_sample
+from opensora.datasets import save_sample
from opensora.models.text_encoder.t5 import text_preprocessing
from opensora.registry import MODELS, SCHEDULERS, build_module
from opensora.utils.config_utils import parse_configs
-from opensora.utils.misc import create_logger, is_distributed, is_main_process, to_torch_dtype
+from opensora.utils.inference_utils import get_save_path_name, load_prompts, prepare_multi_resolution_info
+from opensora.utils.misc import all_exists, create_logger, is_distributed, is_main_process, to_torch_dtype
def main():
torch.set_grad_enabled(False)
# ======================================================
- # 1. configs & runtime variables
+ # configs & runtime variables
# ======================================================
# == parse configs ==
cfg = parse_configs(training=False)
@@ -27,7 +29,7 @@ def main():
device = "cuda" if torch.cuda.is_available() else "cpu"
cfg_dtype = cfg.get("dtype", "fp32")
assert cfg_dtype in ["fp16", "bf16", "fp32"], f"Unknown mixed precision {cfg_dtype}"
- dtype = to_torch_dtype(cfg.dtype)
+ dtype = to_torch_dtype(cfg.get("dtype", "bf16"))
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
@@ -39,120 +41,102 @@ def main():
if enable_sequence_parallelism:
set_sequence_parallel_group(dist.group.WORLD)
else:
+ coordinator = None
enable_sequence_parallelism = False
set_random_seed(seed=cfg.seed)
# == init logger ==
- create_logger()
+ logger = create_logger()
+ logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))
verbose = cfg.get("verbose", 1)
- breakpoint()
-
- print(cfg)
+ progress_wrap = tqdm if verbose == 1 else (lambda x: x)
# ======================================================
- # 2. runtime variables
+ # build model & load weights
# ======================================================
+ logger.info("Building models...")
+ # == build text-encoder and vae ==
+ text_encoder = build_module(cfg.text_encoder, MODELS, device=device)
+ vae = build_module(cfg.vae, MODELS).to(device, dtype).eval()
- prompts = cfg.prompt
-
- # ======================================================
- # 3. build model & load weights
- # ======================================================
- # 3.1. build model
- input_size = (cfg.num_frames, *cfg.image_size)
- vae = build_module(cfg.vae, MODELS)
+ # == build diffusion model ==
+ input_size = (cfg.get("num_frames", None), *cfg.get("image_size", (None, None)))
latent_size = vae.get_latent_size(input_size)
- text_encoder = build_module(cfg.text_encoder, MODELS, device=device) # T5 must be fp32
-
- model = build_module(
- cfg.model,
- MODELS,
- input_size=latent_size,
- in_channels=vae.out_channels,
- caption_channels=text_encoder.output_dim,
- model_max_length=text_encoder.model_max_length,
- enable_sequence_parallelism=enable_sequence_parallelism,
+ model = (
+ build_module(
+ cfg.model,
+ MODELS,
+ input_size=latent_size,
+ in_channels=vae.out_channels,
+ caption_channels=text_encoder.output_dim,
+ model_max_length=text_encoder.model_max_length,
+ enable_sequence_parallelism=enable_sequence_parallelism,
+ )
+ .to(device, dtype)
+ .eval()
)
- text_encoder.y_embedder = model.y_embedder # hack for classifier-free guidance
-
- # 3.2. move to device & eval
- vae = vae.to(device, dtype).eval()
- model = model.to(device, dtype).eval()
+ text_encoder.y_embedder = model.y_embedder # HACK: for classifier-free guidance
- # 3.3. build scheduler
+ # == build scheduler ==
scheduler = build_module(cfg.scheduler, SCHEDULERS)
- # 3.4. support for multi-resolution
- model_args = dict()
- if cfg.multi_resolution == "PixArtMS":
- image_size = cfg.image_size
- hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
- ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
- model_args["data_info"] = dict(ar=ar, hw=hw)
- elif cfg.multi_resolution == "STDiT2":
- image_size = cfg.image_size
- height = torch.tensor([image_size[0]], device=device, dtype=dtype).repeat(cfg.batch_size)
- width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
- num_frames = torch.tensor([cfg.num_frames], device=device, dtype=dtype).repeat(cfg.batch_size)
- ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
- if cfg.num_frames == 1:
- cfg.fps = IMG_FPS
- fps = torch.tensor([cfg.fps], device=device, dtype=dtype).repeat(cfg.batch_size)
- model_args["height"] = height
- model_args["width"] = width
- model_args["num_frames"] = num_frames
- model_args["ar"] = ar
- model_args["fps"] = fps
-
# ======================================================
- # 4. inference
+ # inference
# ======================================================
- sample_idx = cfg.get("start_index", 0)
- if cfg.sample_name is not None:
- sample_name = cfg.sample_name
- elif cfg.prompt_as_path:
- sample_name = ""
- else:
- sample_name = "sample"
+ # == load prompts ==
+ prompts = cfg.get("prompt", None)
+ start_idx = cfg.get("start_index", 0)
+ if prompts is None:
+ assert cfg.get("prompt_path", None) is not None, "Prompt or prompt_path must be provided"
+ prompts = load_prompts(cfg.prompt_path, start_idx, cfg.get("end_index", None))
+
+ # == prepare arguments ==
+ image_size = cfg.image_size
+ num_frames = cfg.num_frames
+ fps = cfg.fps
+ save_fps = cfg.fps // cfg.get("frame_interval", 1)
+ multi_resolution = cfg.get("multi_resolution", None)
+ batch_size = cfg.get("batch_size", 1)
+ num_sample = cfg.get("num_sample", 1)
+
save_dir = cfg.save_dir
os.makedirs(save_dir, exist_ok=True)
+ sample_name = cfg.get("sample_name", None)
+ prompt_as_path = cfg.get("prompt_as_path", False)
- # 4.1. batch generation
- progress_wrap = tqdm if verbose == 1 else (lambda x: x)
- for i in progress_wrap(range(0, len(prompts), cfg.batch_size)):
- # 4.2 sample in hidden space
- batch_prompts_raw = prompts[i : i + cfg.batch_size]
+ # == Iter over all samples ==
+ for i in progress_wrap(range(0, len(prompts), batch_size)):
+ # == prepare batch prompts ==
+ batch_prompts_raw = prompts[i : i + batch_size]
batch_prompts = [text_preprocessing(prompt) for prompt in batch_prompts_raw]
- # handle the last batch
- if len(batch_prompts_raw) < cfg.batch_size and cfg.multi_resolution == "STDiT2":
- model_args["height"] = model_args["height"][: len(batch_prompts_raw)]
- model_args["width"] = model_args["width"][: len(batch_prompts_raw)]
- model_args["num_frames"] = model_args["num_frames"][: len(batch_prompts_raw)]
- model_args["ar"] = model_args["ar"][: len(batch_prompts_raw)]
- model_args["fps"] = model_args["fps"][: len(batch_prompts_raw)]
-
- # 4.3. diffusion sampling
- old_sample_idx = sample_idx
- # generate multiple samples for each prompt
- for k in range(cfg.num_sample):
- sample_idx = old_sample_idx
-
- # Skip if the sample already exists
+
+ # == multi-resolution info ==
+ model_args = prepare_multi_resolution_info(
+ multi_resolution, len(batch_prompts), image_size, num_frames, fps, device, dtype
+ )
+
+ # == Iter over number of sampling for one prompt ==
+ for k in range(num_sample):
+ # == prepare save paths ==
+ save_paths = [
+ get_save_path_name(
+ save_dir,
+ sample_name=sample_name,
+ sample_idx=start_idx + idx,
+ prompt=batch_prompts_raw[idx],
+ prompt_as_path=prompt_as_path,
+ num_sample=num_sample,
+ k=k,
+ )
+ for idx in range(len(batch_prompts))
+ ]
+
+ # NOTE: Skip if the sample already exists
# This is useful for resuming sampling VBench
- if cfg.prompt_as_path:
- skip = True
- for batch_prompt in batch_prompts_raw:
- path = os.path.join(save_dir, f"{sample_name}{batch_prompt}")
- if cfg.num_sample != 1:
- path = f"{path}-{k}"
- path = f"{path}.mp4"
- if not os.path.exists(path):
- skip = False
- break
- if skip:
- continue
-
- # sampling
+ if prompt_as_path and all_exists(save_paths):
+ continue
+
+ # == sampling ==
z = torch.randn(len(batch_prompts), vae.out_channels, *latent_size, device=device, dtype=dtype)
samples = scheduler.sample(
model,
@@ -163,27 +147,23 @@ def main():
additional_args=model_args,
progress=verbose >= 2,
)
- samples = vae.decode(samples.to(dtype), num_frames=cfg.num_frames)
+ samples = vae.decode(samples.to(dtype), num_frames=num_frames)
- # 4.4. save samples
+ # == save samples ==
if is_main_process():
for idx, sample in enumerate(samples):
if verbose >= 2:
- print(f"Prompt: {batch_prompts_raw[idx]}")
- if cfg.prompt_as_path:
- sample_name_suffix = batch_prompts_raw[idx]
- else:
- sample_name_suffix = f"_{sample_idx}"
- save_path = os.path.join(save_dir, f"{sample_name}{sample_name_suffix}")
- if cfg.num_sample != 1:
- save_path = f"{save_path}-{k}"
+ logger.info("Prompt: %s", batch_prompts_raw[idx])
+ save_path = save_paths[idx]
save_sample(
sample,
- fps=cfg.fps // cfg.frame_interval,
+ fps=save_fps,
save_path=save_path,
verbose=verbose >= 2,
)
- sample_idx += 1
+ start_idx += len(batch_prompts)
+ logger.info("Inference finished.")
+ logger.info("Saved samples to %s", save_dir)
if __name__ == "__main__":
diff --git a/scripts/train.py b/scripts/train.py
index 0647432..e115b53 100644
--- a/scripts/train.py
+++ b/scripts/train.py
@@ -67,10 +67,9 @@ def main():
# == init logger, tensorboard & wandb ==
logger = create_logger(exp_dir)
+ logger.info("Experiment directory created at %s", exp_dir)
+ logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))
if coordinator.is_master():
- logger.info("Experiment directory created at %s", exp_dir)
- logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))
-
tb_writer = create_tensorboard_writer(exp_dir)
if cfg.get("wandb", False):
wandb.init(project="minisora", name=exp_name, config=cfg.to_dict(), dir="./outputs/wandb")
@@ -124,21 +123,23 @@ def main():
logger.info("Building models...")
# == build text-encoder and vae ==
text_encoder = build_module(cfg.text_encoder, MODELS, device=device)
- vae = build_module(cfg.vae, MODELS).to(device, dtype)
- vae.eval()
+ vae = build_module(cfg.vae, MODELS).to(device, dtype).eval()
# == build diffusion model ==
input_size = (dataset.num_frames, *dataset.image_size)
latent_size = vae.get_latent_size(input_size)
- model = build_module(
- cfg.model,
- MODELS,
- input_size=latent_size,
- in_channels=vae.out_channels,
- caption_channels=text_encoder.output_dim,
- model_max_length=text_encoder.model_max_length,
- ).to(device, dtype)
- model.train()
+ model = (
+ build_module(
+ cfg.model,
+ MODELS,
+ input_size=latent_size,
+ in_channels=vae.out_channels,
+ caption_channels=text_encoder.output_dim,
+ model_max_length=text_encoder.model_max_length,
+ )
+ .to(device, dtype)
+ .train()
+ )
model_numel, model_numel_trainable = get_model_numel(model)
logger.info(
"[Diffusion] Trainable model params: %s, Total model params: %s",
diff --git a/scripts/train_vae.py b/scripts/train_vae.py
index a0ef76e..5db369f 100644
--- a/scripts/train_vae.py
+++ b/scripts/train_vae.py
@@ -62,10 +62,9 @@ def main():
# == init logger, tensorboard & wandb ==
logger = create_logger(exp_dir)
+ logger.info("Experiment directory created at %s", exp_dir)
+ logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))
if coordinator.is_master():
- logger.info("Experiment directory created at %s", exp_dir)
- logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))
-
tb_writer = create_tensorboard_writer(exp_dir)
if cfg.get("wandb", False):
wandb.init(project="minisora", name=exp_name, config=cfg.to_dict(), dir="./outputs/wandb")
@@ -109,8 +108,7 @@ def main():
# ======================================================
logger.info("Building models...")
# == build vae model ==
- model = build_module(cfg.model, MODELS).to(device, dtype)
- model.train()
+ model = build_module(cfg.model, MODELS).to(device, dtype).train()
model_numel, model_numel_trainable = get_model_numel(model)
logger.info(
"[VAE] Trainable model params: %s, Total model params: %s",
@@ -121,8 +119,7 @@ def main():
# == build discriminator model ==
use_discriminator = cfg.get("discriminator", None) is not None
if use_discriminator:
- discriminator = build_module(cfg.discriminator, MODELS).to(device, dtype)
- discriminator.train()
+ discriminator = build_module(cfg.discriminator, MODELS).to(device, dtype).train()
discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)
logger.info(
"[Discriminator] Trainable model params: %s, Total model params: %s",
diff --git a/tools/datasets/datautil.py b/tools/datasets/datautil.py
index bf34aa9..d06b22f 100644
--- a/tools/datasets/datautil.py
+++ b/tools/datasets/datautil.py
@@ -74,10 +74,13 @@ def get_info(path):
def get_video_info(path):
try:
- vframes, _, _ = torchvision.io.read_video(filename=path, pts_unit="sec", output_format="TCHW")
+ vframes, _, infos = torchvision.io.read_video(filename=path, pts_unit="sec", output_format="TCHW")
num_frames, height, width = vframes.shape[0], vframes.shape[2], vframes.shape[3]
aspect_ratio = height / width
- fps = np.nan
+ if "video_fps" in infos:
+ fps = infos["video_fps"]
+ else:
+ fps = np.nan
resolution = height * width
return num_frames, height, width, aspect_ratio, fps, resolution
except: