Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

add config for PixArt 1.6B

zhengzangw84465c0e44c1
diff --git a/configs/opensora-v1-2/inference/pixart.py b/configs/opensora-v1-2/inference/pixart.py
new file mode 100644
index 0000000..32f4f46
--- /dev/null
+++ b/configs/opensora-v1-2/inference/pixart.py
@@ -0,0 +1,36 @@
+num_frames = 1
+fps = 1
+image_size = (2560, 1536)
+# image_size = (2048, 2048)
+
+model = dict(
+ type="PixArtMS-1B/2",
+ # from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",
+ space_scale=4,
+ no_temporal_pos_emb=True,
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+ base_size=2048 // 8,
+)
+vae = dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+ subfolder="vae",
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+)
+scheduler = dict(
+ type="dpm-solver",
+ num_sampling_steps=14,
+ cfg_scale=4.5,
+)
+dtype = "bf16"
+
+# Others
+batch_size = 1
+seed = 42
+prompt_path = "./assets/texts/t2i_sigma.txt"
+save_dir = "./samples/samples/"
diff --git a/configs/opensora-v1-2/train/pixart.py b/configs/opensora-v1-2/train/pixart.py
new file mode 100644
index 0000000..e69de29
diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.py
index 2451d91..5fec4d0 100644
--- a/opensora/models/pixart/pixart.py
+++ b/opensora/models/pixart/pixart.py
@@ -393,3 +393,11 @@ def PixArtMS_XL_2(from_pretrained=None, **kwargs):
if from_pretrained is not None:
load_checkpoint(model, from_pretrained)
return model
+
+
[email protected]_module("PixArtMS-1B/2")
+def PixArtMS_1B_2(from_pretrained=None, **kwargs):
+ model = PixArtMS(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/inference.py b/scripts/inference.py
index 7a83ead..179ef94 100644
--- a/scripts/inference.py
+++ b/scripts/inference.py
@@ -70,6 +70,7 @@ def main():
# 3.2. move to device & eval
vae = vae.to(device, dtype).eval()
model = model.to(device, dtype).eval()
+ breakpoint()
# 3.3. build scheduler
scheduler = build_module(cfg.scheduler, SCHEDULERS)