Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

update

周钰坤59591d8afcfe
diff --git a/configs/pixart/inference/1x20481B.py b/configs/pixart/inference/1x20481B.py
new file mode 100644
index 0000000..f3bfe8c
--- /dev/null
+++ b/configs/pixart/inference/1x20481B.py
@@ -0,0 +1,36 @@
+num_frames = 1
+fps = 1
+image_size = (2560, 1536)
+# image_size = (2048, 2048)
+
+model = dict(
+ type="PixArt-1B/2",
+ from_pretrained="PixArt-1B-2.pth",
+ space_scale=4,
+ no_temporal_pos_emb=True,
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+ base_size=2048 // 8,
+)
+vae = dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+ subfolder="vae",
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+)
+scheduler = dict(
+ type="dpm-solver",
+ num_sampling_steps=14,
+ cfg_scale=4.5,
+)
+dtype = "bf16"
+
+# Others
+batch_size = 1
+seed = 42
+prompt_path = "./assets/texts/t2i_sigma.txt"
+save_dir = "./samples/samples/"
diff --git a/configs/pixart/train/1x2048x2048.py b/configs/pixart/train/1x2048x2048.py
new file mode 100644
index 0000000..c0a0881
--- /dev/null
+++ b/configs/pixart/train/1x2048x2048.py
@@ -0,0 +1,54 @@
+# Define dataset
+dataset = dict(
+ type="VideoTextDataset",
+ data_path="/home/zhaowangbo/data/csv/image-v1_1_ext_noempty_rcp_clean_info.csv",
+ num_frames=1,
+ frame_interval=3,
+ image_size=(2048, 2048),
+)
+
+# Define acceleration
+num_workers = 4
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+# Define model
+model = dict(
+ type="PixArt-1B/2",
+ space_scale=4.0,
+ no_temporal_pos_emb=True,
+ from_pretrained="PixArt-1B-2.pth",
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+)
+
+vae = dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+ subfolder="vae",
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+)
+scheduler = dict(
+ type="iddpm",
+ timestep_respacing="",
+)
+
+# Others
+seed = 42
+outputs = "outputs"
+wandb = False
+
+epochs = 1000
+log_every = 10
+ckpt_every = 1000
+load = None
+
+batch_size = 32
+lr = 2e-5
+grad_clip = 1.0
diff --git a/opensora/models/pixart/__init__.py b/opensora/models/pixart/__init__.py
index b83e18e..4c74300 100644
--- a/opensora/models/pixart/__init__.py
+++ b/opensora/models/pixart/__init__.py
@@ -1,2 +1,2 @@
-from .pixart import PixArt, PixArt_XL_2
+from .pixart import PixArt, PixArt_XL_2, PixArt_1B_2
from .pixart_sigma import PixArt_Sigma_XL_2
diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.py
index b5bd9ca..87cf496 100644
--- a/opensora/models/pixart/pixart.py
+++ b/opensora/models/pixart/pixart.py
@@ -402,3 +402,14 @@ def PixArtMS_XL_2(from_pretrained=None, **kwargs):
if from_pretrained is not None:
load_checkpoint(model, from_pretrained)
return model
+<<<<<<< Updated upstream
+=======
+
+
[email protected]_module("PixArt-1B/2")
+def PixArt_1B_2(from_pretrained=None, **kwargs):
+ model = PixArt(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
+>>>>>>> Stashed changes
diff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.py
index 8ed0b9c..b8ecc83 100644
--- a/opensora/utils/ckpt_utils.py
+++ b/opensora/utils/ckpt_utils.py
@@ -39,6 +39,7 @@ pretrained_models = {
"PixArt-Sigma-XL-2-1024-MS.pth": hf_endpoint
+ "/PixArt-alpha/PixArt-Sigma/resolve/main/PixArt-Sigma-XL-2-1024-MS.pth",
"PixArt-Sigma-XL-2-2K-MS.pth": hf_endpoint + "/PixArt-alpha/PixArt-Sigma/resolve/main/PixArt-Sigma-XL-2-2K-MS.pth",
+ # "PixArt-1B-2.pth": "PixArt-1B-2.pth",
}
@@ -68,6 +69,12 @@ def reparameter(ckpt, name=None, model=None):
ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)
if "pos_embed" in ckpt:
del ckpt["pos_embed"]
+
+ if name in ["PixArt-1B-2.pth",]:
+ ckpt = ckpt["state_dict"]
+ if "pos_embed" in ckpt:
+ del ckpt["pos_embed"]
+
# no need pos_embed
if "pos_embed_temporal" in ckpt:
del ckpt["pos_embed_temporal"]
diff --git a/tools/architecture/__init__.py b/tools/architecture/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/architecture/net2net.py b/tools/architecture/net2net.py
new file mode 100644
index 0000000..cb23b77
--- /dev/null
+++ b/tools/architecture/net2net.py
@@ -0,0 +1,70 @@
+"""
+Implementation of Net2Net (http://arxiv.org/abs/1511.05641)
+Numpy modules for Net2Net
+- Net2Wider
+- Net2Deeper
+
+Written by Kyunghyun Paeng
+
+"""
+import numpy as np
+
+def net2net(teach_param, stu_param):
+ # teach param with shape (a, b)
+ # stu param with shape (c, d)
+ # net to net (a, b) -> (c, d) where c >= a and d >= b
+ teach_param_shape = teach_param.shape
+ stu_param_shape = stu_param.shape
+
+ if len(stu_param_shape) > 2:
+ teach_param = teach_param.reshape(teach_param_shape[0], -1)
+ stu_param = stu_param.reshape(stu_param_shape[0], -1)
+
+ assert len(stu_param.shape) == 1 or len(stu_param.shape) == 2, "teach_param and stu_param must be 2-dim array"
+ assert len(teach_param_shape) == len(stu_param_shape), "teach_param and stu_param must have same dimension"
+
+ if len(teach_param_shape) == 1:
+ stu_param[:teach_param_shape[0]] = teach_param
+
+ elif len(teach_param_shape) == 2:
+ stu_param[:teach_param_shape[0], :teach_param_shape[1]] = teach_param
+
+ if stu_param.shape != stu_param_shape:
+ stu_param = stu_param.reshape(stu_param_shape)
+
+ return stu_param
+
+if __name__ == '__main__':
+ """ Net2Net Class Test """
+
+ from opensora.models.pixart import PixArt_Sigma_XL_2, PixArt_1B_2
+ import torch
+
+ model = PixArt_1B_2(no_temporal_pos_emb=True, space_scale=4, enable_flashattn=True, enable_layernorm_kernel=True)
+ print("load model done")
+
+ ckpt = torch.load('/home/zhouyukun/projs/opensora/pretrained_models/PixArt-Sigma-XL-2-2K-MS.pth')
+ print("load ckpt done")
+
+ ckpt = ckpt["state_dict"]
+ ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)
+
+ missing_keys = []
+ for name, module in model.named_parameters():
+ if name in ckpt:
+ teach_param = ckpt[name].data
+ stu_param = module.data
+ stu_param = net2net(teach_param, stu_param)
+
+ module.data = stu_param
+
+ print("processing layer: ", name, "shape: ", module.size())
+
+ else:
+ # print("Missing key: ", name)
+ missing_keys.append(name)
+
+ print(missing_keys)
+
+ breakpoint()
+ torch.save({"state_dict": model.state_dict()}, "PixArt-1B-2.pth")
diff --git a/train_pixart_1B.sh b/train_pixart_1B.sh
new file mode 100644
index 0000000..d3d12ce
--- /dev/null
+++ b/train_pixart_1B.sh
@@ -0,0 +1 @@
+colossalai run --nproc_per_node 8 scripts/train.py configs/pixart/train/1x2048x2048.py --data-path /home/zhaowangbo/data/csv/image-v1_1_ext_noempty_rcp_clean_info.csv