Open-Sora: Democratizing Efficient Video Production for All
Python
add config for PixArt 1.6B
zhengzangw84465c0e44c1
diff --git a/configs/opensora-v1-2/inference/pixart.py b/configs/opensora-v1-2/inference/pixart.pynew file mode 100644index 0000000..32f4f46--- /dev/null+++ b/configs/opensora-v1-2/inference/pixart.py@@ -0,0 +1,36 @@+num_frames = 1+fps = 1+image_size = (2560, 1536)+# image_size = (2048, 2048)++model = dict(+ type="PixArtMS-1B/2",+ # from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",+ space_scale=4,+ no_temporal_pos_emb=True,+ enable_flashattn=True,+ enable_layernorm_kernel=True,+ base_size=2048 // 8,+)+vae = dict(+ type="VideoAutoencoderKL",+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+)+scheduler = dict(+ type="dpm-solver",+ num_sampling_steps=14,+ cfg_scale=4.5,+)+dtype = "bf16"++# Others+batch_size = 1+seed = 42+prompt_path = "./assets/texts/t2i_sigma.txt"+save_dir = "./samples/samples/"diff --git a/configs/opensora-v1-2/train/pixart.py b/configs/opensora-v1-2/train/pixart.pynew file mode 100644index 0000000..e69de29diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.pyindex 2451d91..5fec4d0 100644--- a/opensora/models/pixart/pixart.py+++ b/opensora/models/pixart/pixart.py@@ -393,3 +393,11 @@ def PixArtMS_XL_2(from_pretrained=None, **kwargs):if from_pretrained is not None:load_checkpoint(model, from_pretrained)return model++[email protected]_module("PixArtMS-1B/2")+def PixArtMS_1B_2(from_pretrained=None, **kwargs):+ model = PixArtMS(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return modeldiff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.pynew file mode 100644index 0000000..e69de29diff --git a/scripts/inference.py b/scripts/inference.pyindex 7a83ead..179ef94 100644--- a/scripts/inference.py+++ b/scripts/inference.py@@ -70,6 +70,7 @@ def main():# 3.2. move to device & evalvae = vae.to(device, dtype).eval()model = model.to(device, dtype).eval()+ breakpoint()# 3.3. build schedulerscheduler = build_module(cfg.scheduler, SCHEDULERS)