Open-Sora: Democratizing Efficient Video Production for All
Python
[exp] update
Zangwei Zhenge00f93d111de
diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.pyindex 669ce14..e0e14c7 100644--- a/configs/opensora-v1-1/train/benchmark.py+++ b/configs/opensora-v1-1/train/benchmark.py@@ -13,7 +13,7 @@ bucket_config = {# "240p": {32: (1.0, 8)}, # 4.6s/it# "240p": {16: (1.0, 16)}, # 4.6s/it# "480p": {16: (1.0, 4)}, # 4.6s/it- "720p": {16: (1.0, 2)},+ "720p": {16: (1.0, 2)}, # 5.89s/it# "256": {1: (1.0, 256)}, # 4.5s/it# "512": {1: (1.0, 96)}, # 4.7s/it# "512": {1: (1.0, 128)}, # 6.3s/it@@ -38,7 +38,7 @@ sp_size = 1# Define modelmodel = dict(type="STDiT2-XL/2",- from_pretrained="PixArt-XL-2-1024-MS.pth",+ from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512enable_flashattn=True,enable_layernorm_kernel=True,@@ -51,7 +51,7 @@ vae = dict(text_encoder = dict(type="t5",from_pretrained="DeepFloyd/t5-v1_1-xxl",- model_max_length=300,+ model_max_length=200,shardformer=True,)scheduler = dict(diff --git a/configs/opensora-v1-1/train/image.py b/configs/opensora-v1-1/train/image.pyindex fe9a671..74ab439 100644--- a/configs/opensora-v1-1/train/image.py+++ b/configs/opensora-v1-1/train/image.py@@ -25,7 +25,7 @@ sp_size = 1# Define modelmodel = dict(type="STDiT2-XL/2",- from_pretrained="PixArt-XL-2-1024-MS.pth",+ from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512enable_flashattn=True,enable_layernorm_kernel=True,@@ -38,7 +38,7 @@ vae = dict(text_encoder = dict(type="t5",from_pretrained="DeepFloyd/t5-v1_1-xxl",- model_max_length=300,+ model_max_length=200,shardformer=True,)scheduler = dict(diff --git a/configs/opensora-v1-1/train/video.py b/configs/opensora-v1-1/train/video.pyindex eb8e843..776d3de 100644--- a/configs/opensora-v1-1/train/video.py+++ b/configs/opensora-v1-1/train/video.py@@ -34,7 +34,7 @@ sp_size = 1# Define modelmodel = dict(type="STDiT2-XL/2",- from_pretrained="PixArt-XL-2-1024-MS.pth",+ from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512enable_flashattn=True,enable_layernorm_kernel=True,diff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.pyindex 051aca3..7d416be 100644--- a/opensora/utils/ckpt_utils.py+++ b/opensora/utils/ckpt_utils.py@@ -32,7 +32,7 @@ pretrained_models = {def reparameter(ckpt, name=None, model=None):- if "DiT" in name:+ if "DiT-XL" in name and "STDiT" not in name:ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)del ckpt["pos_embed"]elif "Latte" in name:@@ -47,12 +47,16 @@ def reparameter(ckpt, name=None, model=None):# different text lengthif "y_embedder.y_embedding" in ckpt:- if ckpt["y_embedder.y_embedding"].shape[0] > model.y_embedder.y_embedding.shape[0]:+ if ckpt["y_embedder.y_embedding"].shape[0] < model.y_embedder.y_embedding.shape[0]:additional_length = model.y_embedder.y_embedding.shape[0] - ckpt["y_embedder.y_embedding"].shape[0]new_y_embedding = torch.randn(additional_length, model.y_embedder.y_embedding.shape[1])ckpt["y_embedder.y_embedding"] = torch.cat([ckpt["y_embedder.y_embedding"], new_y_embedding], dim=0)- elif ckpt["y_embedder.y_embedding"].shape[0] < model.y_embedder.y_embedding.shape[0]:+ elif ckpt["y_embedder.y_embedding"].shape[0] > model.y_embedder.y_embedding.shape[0]:ckpt["y_embedder.y_embedding"] = ckpt["y_embedder.y_embedding"][: model.y_embedder.y_embedding.shape[0]]+ if "pos_embed_temporal" in ckpt:+ del ckpt["pos_embed_temporal"]+ if "pos_embed" in ckpt:+ del ckpt["pos_embed"]return ckpt@@ -63,17 +67,11 @@ def find_model(model_name, model=None):if model_name in pretrained_models: # Find/download our pre-trained DiT checkpointsmodel_ckpt = download_model(model_name)model_ckpt = reparameter(model_ckpt, model_name, model=model)- return model_ckptelse: # Load a custom DiT checkpoint:assert os.path.isfile(model_name), f"Could not find DiT checkpoint at {model_name}"- checkpoint = torch.load(model_name, map_location=lambda storage, loc: storage)- if "pos_embed_temporal" in checkpoint:- del checkpoint["pos_embed_temporal"]- if "pos_embed" in checkpoint:- del checkpoint["pos_embed"]- if "ema" in checkpoint: # supports checkpoints from train.py- checkpoint = checkpoint["ema"]- return checkpoint+ model_ckpt = torch.load(model_name, map_location=lambda storage, loc: storage)+ model_ckpt = reparameter(model_ckpt, model_name, model=model)+ return model_ckptdef download_model(model_name):diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex 2c11f76..16166d7 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -449,18 +449,10 @@ def main(args):# make intersectionif args.intersection is not None:- data_int = pd.read_csv(args.intersection)- print(f"Intersection csv contains {len(data_int)} samples.")- data = data[data["path"].isin(data_int["path"])]- input_name += f"-{os.path.basename(args.intersection).split('.')[0]}"- # get additional data column from intersection- int_new_keys = [k for k in data_int.keys() if not k in data]- for k in int_new_keys:- # get corresponding values- data[k] = apply(data["path"], lambda x: get_key_val_given_path(x, k, data_int))-- print(f"added {len(int_new_keys)} keys from {args.intersection}: {int_new_keys}")- print(f"Filtered number of samples: {len(data)}.")+ data_new = pd.read_csv(args.intersection)+ print(f"Intersection csv contains {len(data_new)} samples.")+ data = pd.merge(data, data_new, on="path", how="inner")+ print(f"Intersection number of samples: {len(data)}.")# get output pathoutput_path = get_output_path(args, input_name)