Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

Merge branch 'main' into vae

zhengzangwa4652a8aef0b
diff --cc opensora/utils/ckpt_utils.py
index b28debb,746f42e..2b0655b
--- a/opensora/utils/ckpt_utils.py
+++ b/opensora/utils/ckpt_utils.py
@@@ -80,10 -108,11 +108,11 @@@ def download_model(model_name=None, loc
return model
-def load_from_sharded_state_dict(model, ckpt_path):
+def load_from_sharded_state_dict(model, ckpt_path, model_name="model"):
ckpt_io = GeneralCheckpointIO()
- ckpt_io.load_model(model, os.path.join(ckpt_path, "model"))
+ ckpt_io.load_model(model, os.path.join(ckpt_path, model_name))
+
def model_sharding(model: torch.nn.Module):
global_rank = dist.get_rank()
world_size = dist.get_world_size()
@@@ -203,16 -256,16 +256,16 @@@ def create_logger(logging_dir)
return logger
-def load_checkpoint(model, ckpt_path, save_as_pt=False):
+def load_checkpoint(model, ckpt_path, save_as_pt=True, model_name="model"):
if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):
- state_dict = find_model(ckpt_path)
+ state_dict = find_model(ckpt_path, model=model)
missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)
print(f"Missing keys: {missing_keys}")
print(f"Unexpected keys: {unexpected_keys}")
elif os.path.isdir(ckpt_path):
- load_from_sharded_state_dict(model, ckpt_path)
+ load_from_sharded_state_dict(model, ckpt_path, model_name)
if save_as_pt:
- save_path = os.path.join(ckpt_path, model_name+"_ckpt.pt")
- save_path = os.path.join(ckpt_path, "model_ckpt.pt")
++ save_path = os.path.join(ckpt_path, model_name + "_ckpt.pt")
torch.save(model.state_dict(), save_path)
print(f"Model checkpoint saved to {save_path}")
else:
diff --cc opensora/utils/config_utils.py
index c3dc5e0,9e60a69..efbed9b
--- a/opensora/utils/config_utils.py
+++ b/opensora/utils/config_utils.py
@@@ -27,13 -29,24 +30,25 @@@ def parse_args(training=False)
# ======================================================
# Inference
# ======================================================
-
if not training:
- # prompt
- parser.add_argument("--prompt-path", default=None, type=str, help="path to prompt txt file")
+ # output
parser.add_argument("--save-dir", default=None, type=str, help="path to save generated samples")
+ parser.add_argument("--sample-name", default=None, type=str, help="sample name, default is sample_idx")
+ parser.add_argument("--start-index", default=None, type=int, help="start index for sample name")
+ parser.add_argument("--end-index", default=None, type=int, help="end index for sample name")
+ parser.add_argument("--num-sample", default=None, type=int, help="number of samples to generate for one prompt")
+ parser.add_argument("--prompt-as-path", action="store_true", help="use prompt as path to save samples")
+
+ parser.add_argument("--data-path", default=None, type=str, help="path to data csv")
-
+ # prompt
+ parser.add_argument("--prompt-path", default=None, type=str, help="path to prompt txt file")
+ parser.add_argument("--prompt", default=None, type=str, nargs="+", help="prompt list")
+
+ # image/video
+ parser.add_argument("--num-frames", default=None, type=int, help="number of frames")
+ parser.add_argument("--fps", default=None, type=int, help="fps")
+ parser.add_argument("--image-size", default=None, type=int, nargs=2, help="image size")
+
# hyperparameters
parser.add_argument("--num-sampling-steps", default=None, type=int, help="sampling steps")
parser.add_argument("--cfg-scale", default=None, type=float, help="balance between cond & uncond")
@@@ -50,12 -71,19 +73,21 @@@
def merge_args(cfg, args, training=False):
if args.ckpt_path is not None:
cfg.model["from_pretrained"] = args.ckpt_path
+ if cfg.get("discriminator") is not None:
+ cfg.discriminator["from_pretrained"] = args.ckpt_path
args.ckpt_path = None
+ if training and args.data_path is not None:
+ cfg.dataset["data_path"] = args.data_path
+ args.data_path = None
+ if not training and args.cfg_scale is not None:
+ cfg.scheduler["cfg_scale"] = args.cfg_scale
+ args.cfg_scale = None
+ if not training and args.num_sampling_steps is not None:
+ cfg.scheduler["num_sampling_steps"] = args.num_sampling_steps
+ args.num_sampling_steps = None
for k, v in vars(args).items():
- if k in cfg and v is not None:
+ if v is not None:
cfg[k] = v
if not training:
@@@ -64,13 -93,37 +97,38 @@@
cfg["reference_path"] = None
if "loop" not in cfg:
cfg["loop"] = 1
+ if "frame_interval" not in cfg:
+ cfg["frame_interval"] = 1
+ if "sample_name" not in cfg:
+ cfg["sample_name"] = None
+ if "num_sample" not in cfg:
+ cfg["num_sample"] = 1
+ if "prompt_as_path" not in cfg:
+ cfg["prompt_as_path"] = False
+ # - Prompt handling
- if "prompt" not in cfg or cfg["prompt"] is None:
- assert cfg["prompt_path"] is not None, "prompt or prompt_path must be provided"
- cfg["prompt"] = load_prompts(cfg["prompt_path"])
- if args.start_index is not None and args.end_index is not None:
- cfg["prompt"] = cfg["prompt"][args.start_index : args.end_index]
- elif args.start_index is not None:
- cfg["prompt"] = cfg["prompt"][args.start_index :]
- elif args.end_index is not None:
- cfg["prompt"] = cfg["prompt"][: args.end_index]
+ if not "is_vae" in cfg and ("prompt" not in cfg or cfg["prompt"] is None):
- assert cfg["prompt_path"] is not None, "prompt or prompt_path must be provided"
- cfg["prompt"] = load_prompts(cfg["prompt_path"])
++ if "prompt" not in cfg or cfg["prompt"] is None:
++ assert cfg["prompt_path"] is not None, "prompt or prompt_path must be provided"
++ cfg["prompt"] = load_prompts(cfg["prompt_path"])
++ if args.start_index is not None and args.end_index is not None:
++ cfg["prompt"] = cfg["prompt"][args.start_index : args.end_index]
++ elif args.start_index is not None:
++ cfg["prompt"] = cfg["prompt"][args.start_index :]
++ elif args.end_index is not None:
++ cfg["prompt"] = cfg["prompt"][: args.end_index]
else:
- # Training only
+ # Training only
+ # - Allow not set
if "mask_ratios" not in cfg:
cfg["mask_ratios"] = None
+ if "start_from_scratch" not in cfg:
+ cfg["start_from_scratch"] = False
+ if "bucket_config" not in cfg:
+ cfg["bucket_config"] = None
+ if "transform_name" not in cfg.dataset:
+ cfg.dataset["transform_name"] = "center"
+ if "num_bucket_build_workers" not in cfg:
+ cfg["num_bucket_build_workers"] = 1
# Both training and inference
if "multi_resolution" not in cfg: