Open-Sora: Democratizing Efficient Video Production for All
Python
[checkpoint] adapted inference to hf pretrained weights (#143)
Frank Lee93ba7719ece7
diff --git a/configs/opensora-v1-2/inference/sample.py b/configs/opensora-v1-2/inference/sample.pyindex fd04d23..72cb47b 100644--- a/configs/opensora-v1-2/inference/sample.py+++ b/configs/opensora-v1-2/inference/sample.py@@ -26,7 +26,7 @@ align = 5model = dict(type="STDiT3-XL/2",- from_pretrained="/mnt/jfs/sora_checkpoints/042-STDiT3-XL-2/epoch0-global_step7200/ema.pt",+ from_pretrained="hpcai-tech/OpenSora-STDiT-v3",qk_norm=True,enable_flash_attn=True,enable_layernorm_kernel=True,diff --git a/gradio/app.py b/gradio/app.pyindex 55d40e6..7d1a88c 100644--- a/gradio/app.py+++ b/gradio/app.py@@ -25,10 +25,7 @@ CONFIG_MAP = {"v1.2-stage3": "configs/opensora-v1-2/inference/sample.py",}HF_STDIT_MAP = {- "v1.2-stage3": {- "ema": "/mnt/jfs-hdd/sora/checkpoints/outputs/042-STDiT3-XL-2/epoch1-global_step16200/ema.pt",- "model": "/mnt/jfs-hdd/sora/checkpoints/outputs/042-STDiT3-XL-2/epoch1-global_step16200/model"- }+ "v1.2-stage3": "hpcai-tech/OpenSora-STDiT-v3"}# ============================@@ -104,11 +101,8 @@ def build_models(model_type, config, enable_optimization=False):# build stdit# we load model from HuggingFace directly so that we don't need to# handle model download logic in HuggingFace Space- from opensora.models.stdit.stdit3 import STDiT3, STDiT3Config- stdit3_config = STDiT3Config.from_pretrained(HF_STDIT_MAP[model_type]['model'])- stdit = STDiT3(stdit3_config)- ckpt = torch.load(HF_STDIT_MAP[model_type]['ema'])- stdit.load_state_dict(ckpt)+ from opensora.models.stdit.stdit3 import STDiT3+ stdit = STDiT3.from_pretrained(HF_STDIT_MAP[model_type])stdit = stdit.cuda()# build schedulerdiff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.pyindex 2a64ab9..dd9fe86 100644--- a/opensora/models/stdit/stdit3.py+++ b/opensora/models/stdit/stdit3.py@@ -1,3 +1,4 @@+import osimport numpy as npimport torchimport torch.nn as nn@@ -444,17 +445,24 @@ class STDiT3(PreTrainedModel):@MODELS.register_module("STDiT3-XL/2")def STDiT3_XL_2(from_pretrained=None, **kwargs):- config = STDiT3Config(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)- model = STDiT3(config)- if from_pretrained is not None:- load_checkpoint(model, from_pretrained)+ if from_pretrained is not None and not os.path.isdir(from_pretrained):+ model = STDiT3.from_pretrained(from_pretrained, **kwargs)+ else:+ config = STDiT3Config(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)+ model = STDiT3(config)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)return model@MODELS.register_module("STDiT3-3B/2")def STDiT3_3B_2(from_pretrained=None, **kwargs):- config = STDiT3Config(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)- model = STDiT3(config)- if from_pretrained is not None:- load_checkpoint(model, from_pretrained)+ # check if from_pretrained is a path+ if from_pretrained is not None and not os.path.isdir(from_pretrained):+ model = STDiT3.from_pretrained(from_pretrained, **kwargs)+ else:+ config = STDiT3Config(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)+ model = STDiT3(config)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)return model