Open-Sora: Democratizing Efficient Video Production for All
Python
Merge branch 'main' into vae
zhengzangwa4652a8aef0b
diff --cc opensora/utils/ckpt_utils.pyindex b28debb,746f42e..2b0655b--- a/opensora/utils/ckpt_utils.py+++ b/opensora/utils/ckpt_utils.py@@@ -80,10 -108,11 +108,11 @@@ def download_model(model_name=None, locreturn model-def load_from_sharded_state_dict(model, ckpt_path):+def load_from_sharded_state_dict(model, ckpt_path, model_name="model"):ckpt_io = GeneralCheckpointIO()- ckpt_io.load_model(model, os.path.join(ckpt_path, "model"))+ ckpt_io.load_model(model, os.path.join(ckpt_path, model_name))+def model_sharding(model: torch.nn.Module):global_rank = dist.get_rank()world_size = dist.get_world_size()@@@ -203,16 -256,16 +256,16 @@@ def create_logger(logging_dir)return logger-def load_checkpoint(model, ckpt_path, save_as_pt=False):+def load_checkpoint(model, ckpt_path, save_as_pt=True, model_name="model"):if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):- state_dict = find_model(ckpt_path)+ state_dict = find_model(ckpt_path, model=model)missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)print(f"Missing keys: {missing_keys}")print(f"Unexpected keys: {unexpected_keys}")elif os.path.isdir(ckpt_path):- load_from_sharded_state_dict(model, ckpt_path)+ load_from_sharded_state_dict(model, ckpt_path, model_name)if save_as_pt:- save_path = os.path.join(ckpt_path, model_name+"_ckpt.pt")- save_path = os.path.join(ckpt_path, "model_ckpt.pt")++ save_path = os.path.join(ckpt_path, model_name + "_ckpt.pt")torch.save(model.state_dict(), save_path)print(f"Model checkpoint saved to {save_path}")else:diff --cc opensora/utils/config_utils.pyindex c3dc5e0,9e60a69..efbed9b--- a/opensora/utils/config_utils.py+++ b/opensora/utils/config_utils.py@@@ -27,13 -29,24 +30,25 @@@ def parse_args(training=False)# ======================================================# Inference# ======================================================-if not training:- # prompt- parser.add_argument("--prompt-path", default=None, type=str, help="path to prompt txt file")+ # outputparser.add_argument("--save-dir", default=None, type=str, help="path to save generated samples")+ parser.add_argument("--sample-name", default=None, type=str, help="sample name, default is sample_idx")+ parser.add_argument("--start-index", default=None, type=int, help="start index for sample name")+ parser.add_argument("--end-index", default=None, type=int, help="end index for sample name")+ parser.add_argument("--num-sample", default=None, type=int, help="number of samples to generate for one prompt")+ parser.add_argument("--prompt-as-path", action="store_true", help="use prompt as path to save samples")++ parser.add_argument("--data-path", default=None, type=str, help="path to data csv")-+ # prompt+ parser.add_argument("--prompt-path", default=None, type=str, help="path to prompt txt file")+ parser.add_argument("--prompt", default=None, type=str, nargs="+", help="prompt list")++ # image/video+ parser.add_argument("--num-frames", default=None, type=int, help="number of frames")+ parser.add_argument("--fps", default=None, type=int, help="fps")+ parser.add_argument("--image-size", default=None, type=int, nargs=2, help="image size")+# hyperparametersparser.add_argument("--num-sampling-steps", default=None, type=int, help="sampling steps")parser.add_argument("--cfg-scale", default=None, type=float, help="balance between cond & uncond")@@@ -50,12 -71,19 +73,21 @@@def merge_args(cfg, args, training=False):if args.ckpt_path is not None:cfg.model["from_pretrained"] = args.ckpt_path+ if cfg.get("discriminator") is not None:+ cfg.discriminator["from_pretrained"] = args.ckpt_pathargs.ckpt_path = None+ if training and args.data_path is not None:+ cfg.dataset["data_path"] = args.data_path+ args.data_path = None+ if not training and args.cfg_scale is not None:+ cfg.scheduler["cfg_scale"] = args.cfg_scale+ args.cfg_scale = None+ if not training and args.num_sampling_steps is not None:+ cfg.scheduler["num_sampling_steps"] = args.num_sampling_steps+ args.num_sampling_steps = Nonefor k, v in vars(args).items():- if k in cfg and v is not None:+ if v is not None:cfg[k] = vif not training:@@@ -64,13 -93,37 +97,38 @@@cfg["reference_path"] = Noneif "loop" not in cfg:cfg["loop"] = 1+ if "frame_interval" not in cfg:+ cfg["frame_interval"] = 1+ if "sample_name" not in cfg:+ cfg["sample_name"] = None+ if "num_sample" not in cfg:+ cfg["num_sample"] = 1+ if "prompt_as_path" not in cfg:+ cfg["prompt_as_path"] = False+ # - Prompt handling- if "prompt" not in cfg or cfg["prompt"] is None:- assert cfg["prompt_path"] is not None, "prompt or prompt_path must be provided"- cfg["prompt"] = load_prompts(cfg["prompt_path"])- if args.start_index is not None and args.end_index is not None:- cfg["prompt"] = cfg["prompt"][args.start_index : args.end_index]- elif args.start_index is not None:- cfg["prompt"] = cfg["prompt"][args.start_index :]- elif args.end_index is not None:- cfg["prompt"] = cfg["prompt"][: args.end_index]+ if not "is_vae" in cfg and ("prompt" not in cfg or cfg["prompt"] is None):- assert cfg["prompt_path"] is not None, "prompt or prompt_path must be provided"- cfg["prompt"] = load_prompts(cfg["prompt_path"])++ if "prompt" not in cfg or cfg["prompt"] is None:++ assert cfg["prompt_path"] is not None, "prompt or prompt_path must be provided"++ cfg["prompt"] = load_prompts(cfg["prompt_path"])++ if args.start_index is not None and args.end_index is not None:++ cfg["prompt"] = cfg["prompt"][args.start_index : args.end_index]++ elif args.start_index is not None:++ cfg["prompt"] = cfg["prompt"][args.start_index :]++ elif args.end_index is not None:++ cfg["prompt"] = cfg["prompt"][: args.end_index]else:- # Training only+ # Training only+ # - Allow not setif "mask_ratios" not in cfg:cfg["mask_ratios"] = None+ if "start_from_scratch" not in cfg:+ cfg["start_from_scratch"] = False+ if "bucket_config" not in cfg:+ cfg["bucket_config"] = None+ if "transform_name" not in cfg.dataset:+ cfg.dataset["transform_name"] = "center"+ if "num_bucket_build_workers" not in cfg:+ cfg["num_bucket_build_workers"] = 1# Both training and inferenceif "multi_resolution" not in cfg: