Open-Sora: Democratizing Efficient Video Production for All
Python
[fix] update inference and datautil
zhengzangw6194da650459
diff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.pyindex cee23dd..8b61c83 100644--- a/opensora/utils/config_utils.py+++ b/opensora/utils/config_utils.py@@ -7,12 +7,6 @@ from mmengine.config import Configfrom torch.utils.tensorboard import SummaryWriter-def load_prompts(prompt_path):- with open(prompt_path, "r") as f:- prompts = [line.strip() for line in f.readlines()]- return prompts--def parse_args(training=False):parser = argparse.ArgumentParser()@@ -86,11 +80,13 @@ def merge_args(cfg, args, training=False):if args.data_path is not None:cfg.dataset["data_path"] = args.data_pathargs.data_path = None- if not training:- if args.image_size is not None and "dataset" in cfg:+ # NOTE: for vae inference (reconstruction)+ if not training and "dataset" in cfg:+ if args.image_size is not None:cfg.dataset["image_size"] = args.image_size- if args.num_frames is not None and "dataset" in cfg:+ if args.num_frames is not None:cfg.dataset["num_frames"] = args.num_frames+ if not training:if args.cfg_scale is not None:cfg.scheduler["cfg_scale"] = args.cfg_scaleargs.cfg_scale = None@@ -103,32 +99,10 @@ def merge_args(cfg, args, training=False):cfg[k] = vif not training:- # Inference only- # - Allow not setif "reference_path" not in cfg:cfg["reference_path"] = Noneif "loop" not in cfg:cfg["loop"] = 1- if "frame_interval" not in cfg:- cfg["frame_interval"] = 1- if "sample_name" not in cfg:- cfg["sample_name"] = None- if "num_sample" not in cfg:- cfg["num_sample"] = 1- if "prompt_as_path" not in cfg:- cfg["prompt_as_path"] = False- # - Prompt handling- if "prompt" not in cfg or cfg["prompt"] is None:- if ("prompt" not in cfg or cfg["prompt"] is None) and cfg.get("prompt_path", None) is not None:- cfg["prompt"] = load_prompts(cfg["prompt_path"])- if args.start_index is not None and args.end_index is not None:- cfg["prompt"] = cfg["prompt"][args.start_index : args.end_index]- elif args.start_index is not None:- cfg["prompt"] = cfg["prompt"][args.start_index :]- elif args.end_index is not None:- cfg["prompt"] = cfg["prompt"][: args.end_index]- if "multi_resolution" not in cfg:- cfg["multi_resolution"] = Falsereturn cfgdiff --git a/opensora/utils/inference_utils.py b/opensora/utils/inference_utils.pynew file mode 100644index 0000000..48792b4--- /dev/null+++ b/opensora/utils/inference_utils.py@@ -0,0 +1,49 @@+import os++import torch++from opensora.datasets import IMG_FPS+++def prepare_multi_resolution_info(info_type, batch_size, image_size, num_frames, fps, device, dtype):+ if info_type is None:+ return dict()+ elif info_type == "PixArtMS":+ hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(batch_size, 1)+ ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(batch_size, 1)+ return dict(ar=ar, hw=hw)+ elif info_type in ["STDiT2", "OpenSora"]:+ height = torch.tensor([image_size[0]], device=device, dtype=dtype).repeat(batch_size)+ width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(batch_size)+ num_frames = torch.tensor([num_frames], device=device, dtype=dtype).repeat(batch_size)+ ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(batch_size)+ fps = fps if num_frames > 1 else IMG_FPS+ fps = torch.tensor([fps], device=device, dtype=dtype).repeat(batch_size)+ return dict(height=height, width=width, num_frames=num_frames, ar=ar, fps=fps)+ else:+ raise NotImplementedError+++def load_prompts(prompt_path, start_idx=None, end_idx=None):+ with open(prompt_path, "r") as f:+ prompts = [line.strip() for line in f.readlines()]+ prompts = prompts[start_idx:end_idx]+ return prompts+++def get_save_path_name(+ save_dir,+ sample_name=None, # prefix+ sample_idx=None, # sample index+ prompt=None, # used prompt+ prompt_as_path=False, # use prompt as path+ num_sample=1, # number of samples to generate for one prompt+ k=None, # kth sample+):+ if sample_name is None:+ sample_name = "" if prompt_as_path else "sample"+ sample_name_suffix = prompt if prompt_as_path else f"_{sample_idx}"+ save_path = os.path.join(save_dir, f"{sample_name}{sample_name_suffix}")+ if num_sample != 1:+ save_path = f"{save_path}-{k}"+ return save_pathdiff --git a/opensora/utils/misc.py b/opensora/utils/misc.pyindex e492bdd..d191581 100644--- a/opensora/utils/misc.py+++ b/opensora/utils/misc.py@@ -318,3 +318,7 @@ def transpose(x):"""ret = list(map(list, zip(*x)))return ret+++def all_exists(paths):+ return all(os.path.exists(path) for path in paths)diff --git a/scripts/inference.py b/scripts/inference.pyindex a839ca5..2887fce 100644--- a/scripts/inference.py+++ b/scripts/inference.py@@ -1,4 +1,5 @@import os+from pprint import pformatimport colossalaiimport torch@@ -8,17 +9,18 @@ from mmengine.runner import set_random_seedfrom tqdm import tqdmfrom opensora.acceleration.parallel_states import set_sequence_parallel_group-from opensora.datasets import IMG_FPS, save_sample+from opensora.datasets import save_samplefrom opensora.models.text_encoder.t5 import text_preprocessingfrom opensora.registry import MODELS, SCHEDULERS, build_modulefrom opensora.utils.config_utils import parse_configs-from opensora.utils.misc import create_logger, is_distributed, is_main_process, to_torch_dtype+from opensora.utils.inference_utils import get_save_path_name, load_prompts, prepare_multi_resolution_info+from opensora.utils.misc import all_exists, create_logger, is_distributed, is_main_process, to_torch_dtypedef main():torch.set_grad_enabled(False)# ======================================================- # 1. configs & runtime variables+ # configs & runtime variables# ======================================================# == parse configs ==cfg = parse_configs(training=False)@@ -27,7 +29,7 @@ def main():device = "cuda" if torch.cuda.is_available() else "cpu"cfg_dtype = cfg.get("dtype", "fp32")assert cfg_dtype in ["fp16", "bf16", "fp32"], f"Unknown mixed precision {cfg_dtype}"- dtype = to_torch_dtype(cfg.dtype)+ dtype = to_torch_dtype(cfg.get("dtype", "bf16"))torch.backends.cuda.matmul.allow_tf32 = Truetorch.backends.cudnn.allow_tf32 = True@@ -39,120 +41,102 @@ def main():if enable_sequence_parallelism:set_sequence_parallel_group(dist.group.WORLD)else:+ coordinator = Noneenable_sequence_parallelism = Falseset_random_seed(seed=cfg.seed)# == init logger ==- create_logger()+ logger = create_logger()+ logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))verbose = cfg.get("verbose", 1)- breakpoint()-- print(cfg)+ progress_wrap = tqdm if verbose == 1 else (lambda x: x)# ======================================================- # 2. runtime variables+ # build model & load weights# ======================================================+ logger.info("Building models...")+ # == build text-encoder and vae ==+ text_encoder = build_module(cfg.text_encoder, MODELS, device=device)+ vae = build_module(cfg.vae, MODELS).to(device, dtype).eval()- prompts = cfg.prompt-- # ======================================================- # 3. build model & load weights- # ======================================================- # 3.1. build model- input_size = (cfg.num_frames, *cfg.image_size)- vae = build_module(cfg.vae, MODELS)+ # == build diffusion model ==+ input_size = (cfg.get("num_frames", None), *cfg.get("image_size", (None, None)))latent_size = vae.get_latent_size(input_size)- text_encoder = build_module(cfg.text_encoder, MODELS, device=device) # T5 must be fp32-- model = build_module(- cfg.model,- MODELS,- input_size=latent_size,- in_channels=vae.out_channels,- caption_channels=text_encoder.output_dim,- model_max_length=text_encoder.model_max_length,- enable_sequence_parallelism=enable_sequence_parallelism,+ model = (+ build_module(+ cfg.model,+ MODELS,+ input_size=latent_size,+ in_channels=vae.out_channels,+ caption_channels=text_encoder.output_dim,+ model_max_length=text_encoder.model_max_length,+ enable_sequence_parallelism=enable_sequence_parallelism,+ )+ .to(device, dtype)+ .eval())- text_encoder.y_embedder = model.y_embedder # hack for classifier-free guidance-- # 3.2. move to device & eval- vae = vae.to(device, dtype).eval()- model = model.to(device, dtype).eval()+ text_encoder.y_embedder = model.y_embedder # HACK: for classifier-free guidance- # 3.3. build scheduler+ # == build scheduler ==scheduler = build_module(cfg.scheduler, SCHEDULERS)- # 3.4. support for multi-resolution- model_args = dict()- if cfg.multi_resolution == "PixArtMS":- image_size = cfg.image_size- hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)- ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)- model_args["data_info"] = dict(ar=ar, hw=hw)- elif cfg.multi_resolution == "STDiT2":- image_size = cfg.image_size- height = torch.tensor([image_size[0]], device=device, dtype=dtype).repeat(cfg.batch_size)- width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)- num_frames = torch.tensor([cfg.num_frames], device=device, dtype=dtype).repeat(cfg.batch_size)- ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)- if cfg.num_frames == 1:- cfg.fps = IMG_FPS- fps = torch.tensor([cfg.fps], device=device, dtype=dtype).repeat(cfg.batch_size)- model_args["height"] = height- model_args["width"] = width- model_args["num_frames"] = num_frames- model_args["ar"] = ar- model_args["fps"] = fps-# ======================================================- # 4. inference+ # inference# ======================================================- sample_idx = cfg.get("start_index", 0)- if cfg.sample_name is not None:- sample_name = cfg.sample_name- elif cfg.prompt_as_path:- sample_name = ""- else:- sample_name = "sample"+ # == load prompts ==+ prompts = cfg.get("prompt", None)+ start_idx = cfg.get("start_index", 0)+ if prompts is None:+ assert cfg.get("prompt_path", None) is not None, "Prompt or prompt_path must be provided"+ prompts = load_prompts(cfg.prompt_path, start_idx, cfg.get("end_index", None))++ # == prepare arguments ==+ image_size = cfg.image_size+ num_frames = cfg.num_frames+ fps = cfg.fps+ save_fps = cfg.fps // cfg.get("frame_interval", 1)+ multi_resolution = cfg.get("multi_resolution", None)+ batch_size = cfg.get("batch_size", 1)+ num_sample = cfg.get("num_sample", 1)+save_dir = cfg.save_diros.makedirs(save_dir, exist_ok=True)+ sample_name = cfg.get("sample_name", None)+ prompt_as_path = cfg.get("prompt_as_path", False)- # 4.1. batch generation- progress_wrap = tqdm if verbose == 1 else (lambda x: x)- for i in progress_wrap(range(0, len(prompts), cfg.batch_size)):- # 4.2 sample in hidden space- batch_prompts_raw = prompts[i : i + cfg.batch_size]+ # == Iter over all samples ==+ for i in progress_wrap(range(0, len(prompts), batch_size)):+ # == prepare batch prompts ==+ batch_prompts_raw = prompts[i : i + batch_size]batch_prompts = [text_preprocessing(prompt) for prompt in batch_prompts_raw]- # handle the last batch- if len(batch_prompts_raw) < cfg.batch_size and cfg.multi_resolution == "STDiT2":- model_args["height"] = model_args["height"][: len(batch_prompts_raw)]- model_args["width"] = model_args["width"][: len(batch_prompts_raw)]- model_args["num_frames"] = model_args["num_frames"][: len(batch_prompts_raw)]- model_args["ar"] = model_args["ar"][: len(batch_prompts_raw)]- model_args["fps"] = model_args["fps"][: len(batch_prompts_raw)]-- # 4.3. diffusion sampling- old_sample_idx = sample_idx- # generate multiple samples for each prompt- for k in range(cfg.num_sample):- sample_idx = old_sample_idx-- # Skip if the sample already exists++ # == multi-resolution info ==+ model_args = prepare_multi_resolution_info(+ multi_resolution, len(batch_prompts), image_size, num_frames, fps, device, dtype+ )++ # == Iter over number of sampling for one prompt ==+ for k in range(num_sample):+ # == prepare save paths ==+ save_paths = [+ get_save_path_name(+ save_dir,+ sample_name=sample_name,+ sample_idx=start_idx + idx,+ prompt=batch_prompts_raw[idx],+ prompt_as_path=prompt_as_path,+ num_sample=num_sample,+ k=k,+ )+ for idx in range(len(batch_prompts))+ ]++ # NOTE: Skip if the sample already exists# This is useful for resuming sampling VBench- if cfg.prompt_as_path:- skip = True- for batch_prompt in batch_prompts_raw:- path = os.path.join(save_dir, f"{sample_name}{batch_prompt}")- if cfg.num_sample != 1:- path = f"{path}-{k}"- path = f"{path}.mp4"- if not os.path.exists(path):- skip = False- break- if skip:- continue-- # sampling+ if prompt_as_path and all_exists(save_paths):+ continue++ # == sampling ==z = torch.randn(len(batch_prompts), vae.out_channels, *latent_size, device=device, dtype=dtype)samples = scheduler.sample(model,@@ -163,27 +147,23 @@ def main():additional_args=model_args,progress=verbose >= 2,)- samples = vae.decode(samples.to(dtype), num_frames=cfg.num_frames)+ samples = vae.decode(samples.to(dtype), num_frames=num_frames)- # 4.4. save samples+ # == save samples ==if is_main_process():for idx, sample in enumerate(samples):if verbose >= 2:- print(f"Prompt: {batch_prompts_raw[idx]}")- if cfg.prompt_as_path:- sample_name_suffix = batch_prompts_raw[idx]- else:- sample_name_suffix = f"_{sample_idx}"- save_path = os.path.join(save_dir, f"{sample_name}{sample_name_suffix}")- if cfg.num_sample != 1:- save_path = f"{save_path}-{k}"+ logger.info("Prompt: %s", batch_prompts_raw[idx])+ save_path = save_paths[idx]save_sample(sample,- fps=cfg.fps // cfg.frame_interval,+ fps=save_fps,save_path=save_path,verbose=verbose >= 2,)- sample_idx += 1+ start_idx += len(batch_prompts)+ logger.info("Inference finished.")+ logger.info("Saved samples to %s", save_dir)if __name__ == "__main__":diff --git a/scripts/train.py b/scripts/train.pyindex 0647432..e115b53 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -67,10 +67,9 @@ def main():# == init logger, tensorboard & wandb ==logger = create_logger(exp_dir)+ logger.info("Experiment directory created at %s", exp_dir)+ logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))if coordinator.is_master():- logger.info("Experiment directory created at %s", exp_dir)- logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))-tb_writer = create_tensorboard_writer(exp_dir)if cfg.get("wandb", False):wandb.init(project="minisora", name=exp_name, config=cfg.to_dict(), dir="./outputs/wandb")@@ -124,21 +123,23 @@ def main():logger.info("Building models...")# == build text-encoder and vae ==text_encoder = build_module(cfg.text_encoder, MODELS, device=device)- vae = build_module(cfg.vae, MODELS).to(device, dtype)- vae.eval()+ vae = build_module(cfg.vae, MODELS).to(device, dtype).eval()# == build diffusion model ==input_size = (dataset.num_frames, *dataset.image_size)latent_size = vae.get_latent_size(input_size)- model = build_module(- cfg.model,- MODELS,- input_size=latent_size,- in_channels=vae.out_channels,- caption_channels=text_encoder.output_dim,- model_max_length=text_encoder.model_max_length,- ).to(device, dtype)- model.train()+ model = (+ build_module(+ cfg.model,+ MODELS,+ input_size=latent_size,+ in_channels=vae.out_channels,+ caption_channels=text_encoder.output_dim,+ model_max_length=text_encoder.model_max_length,+ )+ .to(device, dtype)+ .train()+ )model_numel, model_numel_trainable = get_model_numel(model)logger.info("[Diffusion] Trainable model params: %s, Total model params: %s",diff --git a/scripts/train_vae.py b/scripts/train_vae.pyindex a0ef76e..5db369f 100644--- a/scripts/train_vae.py+++ b/scripts/train_vae.py@@ -62,10 +62,9 @@ def main():# == init logger, tensorboard & wandb ==logger = create_logger(exp_dir)+ logger.info("Experiment directory created at %s", exp_dir)+ logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))if coordinator.is_master():- logger.info("Experiment directory created at %s", exp_dir)- logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))-tb_writer = create_tensorboard_writer(exp_dir)if cfg.get("wandb", False):wandb.init(project="minisora", name=exp_name, config=cfg.to_dict(), dir="./outputs/wandb")@@ -109,8 +108,7 @@ def main():# ======================================================logger.info("Building models...")# == build vae model ==- model = build_module(cfg.model, MODELS).to(device, dtype)- model.train()+ model = build_module(cfg.model, MODELS).to(device, dtype).train()model_numel, model_numel_trainable = get_model_numel(model)logger.info("[VAE] Trainable model params: %s, Total model params: %s",@@ -121,8 +119,7 @@ def main():# == build discriminator model ==use_discriminator = cfg.get("discriminator", None) is not Noneif use_discriminator:- discriminator = build_module(cfg.discriminator, MODELS).to(device, dtype)- discriminator.train()+ discriminator = build_module(cfg.discriminator, MODELS).to(device, dtype).train()discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)logger.info("[Discriminator] Trainable model params: %s, Total model params: %s",diff --git a/tools/datasets/datautil.py b/tools/datasets/datautil.pyindex bf34aa9..d06b22f 100644--- a/tools/datasets/datautil.py+++ b/tools/datasets/datautil.py@@ -74,10 +74,13 @@ def get_info(path):def get_video_info(path):try:- vframes, _, _ = torchvision.io.read_video(filename=path, pts_unit="sec", output_format="TCHW")+ vframes, _, infos = torchvision.io.read_video(filename=path, pts_unit="sec", output_format="TCHW")num_frames, height, width = vframes.shape[0], vframes.shape[2], vframes.shape[3]aspect_ratio = height / width- fps = np.nan+ if "video_fps" in infos:+ fps = infos["video_fps"]+ else:+ fps = np.nanresolution = height * widthreturn num_frames, height, width, aspect_ratio, fps, resolutionexcept: