Open-Sora: Democratizing Efficient Video Production for All
Python
update
周钰坤59591d8afcfe
diff --git a/configs/pixart/inference/1x20481B.py b/configs/pixart/inference/1x20481B.pynew file mode 100644index 0000000..f3bfe8c--- /dev/null+++ b/configs/pixart/inference/1x20481B.py@@ -0,0 +1,36 @@+num_frames = 1+fps = 1+image_size = (2560, 1536)+# image_size = (2048, 2048)++model = dict(+ type="PixArt-1B/2",+ from_pretrained="PixArt-1B-2.pth",+ space_scale=4,+ no_temporal_pos_emb=True,+ enable_flashattn=True,+ enable_layernorm_kernel=True,+ base_size=2048 // 8,+)+vae = dict(+ type="VideoAutoencoderKL",+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+)+scheduler = dict(+ type="dpm-solver",+ num_sampling_steps=14,+ cfg_scale=4.5,+)+dtype = "bf16"++# Others+batch_size = 1+seed = 42+prompt_path = "./assets/texts/t2i_sigma.txt"+save_dir = "./samples/samples/"diff --git a/configs/pixart/train/1x2048x2048.py b/configs/pixart/train/1x2048x2048.pynew file mode 100644index 0000000..c0a0881--- /dev/null+++ b/configs/pixart/train/1x2048x2048.py@@ -0,0 +1,54 @@+# Define dataset+dataset = dict(+ type="VideoTextDataset",+ data_path="/home/zhaowangbo/data/csv/image-v1_1_ext_noempty_rcp_clean_info.csv",+ num_frames=1,+ frame_interval=3,+ image_size=(2048, 2048),+)++# Define acceleration+num_workers = 4+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1++# Define model+model = dict(+ type="PixArt-1B/2",+ space_scale=4.0,+ no_temporal_pos_emb=True,+ from_pretrained="PixArt-1B-2.pth",+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)++vae = dict(+ type="VideoAutoencoderKL",+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+)+scheduler = dict(+ type="iddpm",+ timestep_respacing="",+)++# Others+seed = 42+outputs = "outputs"+wandb = False++epochs = 1000+log_every = 10+ckpt_every = 1000+load = None++batch_size = 32+lr = 2e-5+grad_clip = 1.0diff --git a/opensora/models/pixart/__init__.py b/opensora/models/pixart/__init__.pyindex b83e18e..4c74300 100644--- a/opensora/models/pixart/__init__.py+++ b/opensora/models/pixart/__init__.py@@ -1,2 +1,2 @@-from .pixart import PixArt, PixArt_XL_2+from .pixart import PixArt, PixArt_XL_2, PixArt_1B_2from .pixart_sigma import PixArt_Sigma_XL_2diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.pyindex b5bd9ca..87cf496 100644--- a/opensora/models/pixart/pixart.py+++ b/opensora/models/pixart/pixart.py@@ -402,3 +402,14 @@ def PixArtMS_XL_2(from_pretrained=None, **kwargs):if from_pretrained is not None:load_checkpoint(model, from_pretrained)return model+<<<<<<< Updated upstream+=======++[email protected]_module("PixArt-1B/2")+def PixArt_1B_2(from_pretrained=None, **kwargs):+ model = PixArt(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return model+>>>>>>> Stashed changesdiff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.pyindex 8ed0b9c..b8ecc83 100644--- a/opensora/utils/ckpt_utils.py+++ b/opensora/utils/ckpt_utils.py@@ -39,6 +39,7 @@ pretrained_models = {"PixArt-Sigma-XL-2-1024-MS.pth": hf_endpoint+ "/PixArt-alpha/PixArt-Sigma/resolve/main/PixArt-Sigma-XL-2-1024-MS.pth","PixArt-Sigma-XL-2-2K-MS.pth": hf_endpoint + "/PixArt-alpha/PixArt-Sigma/resolve/main/PixArt-Sigma-XL-2-2K-MS.pth",+ # "PixArt-1B-2.pth": "PixArt-1B-2.pth",}@@ -68,6 +69,12 @@ def reparameter(ckpt, name=None, model=None):ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)if "pos_embed" in ckpt:del ckpt["pos_embed"]++ if name in ["PixArt-1B-2.pth",]:+ ckpt = ckpt["state_dict"]+ if "pos_embed" in ckpt:+ del ckpt["pos_embed"]+# no need pos_embedif "pos_embed_temporal" in ckpt:del ckpt["pos_embed_temporal"]diff --git a/tools/architecture/__init__.py b/tools/architecture/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tools/architecture/net2net.py b/tools/architecture/net2net.pynew file mode 100644index 0000000..cb23b77--- /dev/null+++ b/tools/architecture/net2net.py@@ -0,0 +1,70 @@+"""+Implementation of Net2Net (http://arxiv.org/abs/1511.05641)+Numpy modules for Net2Net+- Net2Wider+- Net2Deeper++Written by Kyunghyun Paeng++"""+import numpy as np++def net2net(teach_param, stu_param):+ # teach param with shape (a, b)+ # stu param with shape (c, d)+ # net to net (a, b) -> (c, d) where c >= a and d >= b+ teach_param_shape = teach_param.shape+ stu_param_shape = stu_param.shape++ if len(stu_param_shape) > 2:+ teach_param = teach_param.reshape(teach_param_shape[0], -1)+ stu_param = stu_param.reshape(stu_param_shape[0], -1)++ assert len(stu_param.shape) == 1 or len(stu_param.shape) == 2, "teach_param and stu_param must be 2-dim array"+ assert len(teach_param_shape) == len(stu_param_shape), "teach_param and stu_param must have same dimension"++ if len(teach_param_shape) == 1:+ stu_param[:teach_param_shape[0]] = teach_param++ elif len(teach_param_shape) == 2:+ stu_param[:teach_param_shape[0], :teach_param_shape[1]] = teach_param++ if stu_param.shape != stu_param_shape:+ stu_param = stu_param.reshape(stu_param_shape)++ return stu_param++if __name__ == '__main__':+ """ Net2Net Class Test """++ from opensora.models.pixart import PixArt_Sigma_XL_2, PixArt_1B_2+ import torch++ model = PixArt_1B_2(no_temporal_pos_emb=True, space_scale=4, enable_flashattn=True, enable_layernorm_kernel=True)+ print("load model done")++ ckpt = torch.load('/home/zhouyukun/projs/opensora/pretrained_models/PixArt-Sigma-XL-2-2K-MS.pth')+ print("load ckpt done")++ ckpt = ckpt["state_dict"]+ ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)++ missing_keys = []+ for name, module in model.named_parameters():+ if name in ckpt:+ teach_param = ckpt[name].data+ stu_param = module.data+ stu_param = net2net(teach_param, stu_param)++ module.data = stu_param++ print("processing layer: ", name, "shape: ", module.size())++ else:+ # print("Missing key: ", name)+ missing_keys.append(name)++ print(missing_keys)++ breakpoint()+ torch.save({"state_dict": model.state_dict()}, "PixArt-1B-2.pth")diff --git a/train_pixart_1B.sh b/train_pixart_1B.shnew file mode 100644index 0000000..d3d12ce--- /dev/null+++ b/train_pixart_1B.sh@@ -0,0 +1 @@+colossalai run --nproc_per_node 8 scripts/train.py configs/pixart/train/1x2048x2048.py --data-path /home/zhaowangbo/data/csv/image-v1_1_ext_noempty_rcp_clean_info.csv