Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[exp] update

Zangwei Zhenge00f93d111de
diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.py
index 669ce14..e0e14c7 100644
--- a/configs/opensora-v1-1/train/benchmark.py
+++ b/configs/opensora-v1-1/train/benchmark.py
@@ -13,7 +13,7 @@ bucket_config = {
# "240p": {32: (1.0, 8)}, # 4.6s/it
# "240p": {16: (1.0, 16)}, # 4.6s/it
# "480p": {16: (1.0, 4)}, # 4.6s/it
- "720p": {16: (1.0, 2)},
+ "720p": {16: (1.0, 2)}, # 5.89s/it
# "256": {1: (1.0, 256)}, # 4.5s/it
# "512": {1: (1.0, 96)}, # 4.7s/it
# "512": {1: (1.0, 128)}, # 6.3s/it
@@ -38,7 +38,7 @@ sp_size = 1
# Define model
model = dict(
type="STDiT2-XL/2",
- from_pretrained="PixArt-XL-2-1024-MS.pth",
+ from_pretrained=None,
input_sq_size=512, # pretrained model is trained on 512x512
enable_flashattn=True,
enable_layernorm_kernel=True,
@@ -51,7 +51,7 @@ vae = dict(
text_encoder = dict(
type="t5",
from_pretrained="DeepFloyd/t5-v1_1-xxl",
- model_max_length=300,
+ model_max_length=200,
shardformer=True,
)
scheduler = dict(
diff --git a/configs/opensora-v1-1/train/image.py b/configs/opensora-v1-1/train/image.py
index fe9a671..74ab439 100644
--- a/configs/opensora-v1-1/train/image.py
+++ b/configs/opensora-v1-1/train/image.py
@@ -25,7 +25,7 @@ sp_size = 1
# Define model
model = dict(
type="STDiT2-XL/2",
- from_pretrained="PixArt-XL-2-1024-MS.pth",
+ from_pretrained=None,
input_sq_size=512, # pretrained model is trained on 512x512
enable_flashattn=True,
enable_layernorm_kernel=True,
@@ -38,7 +38,7 @@ vae = dict(
text_encoder = dict(
type="t5",
from_pretrained="DeepFloyd/t5-v1_1-xxl",
- model_max_length=300,
+ model_max_length=200,
shardformer=True,
)
scheduler = dict(
diff --git a/configs/opensora-v1-1/train/video.py b/configs/opensora-v1-1/train/video.py
index eb8e843..776d3de 100644
--- a/configs/opensora-v1-1/train/video.py
+++ b/configs/opensora-v1-1/train/video.py
@@ -34,7 +34,7 @@ sp_size = 1
# Define model
model = dict(
type="STDiT2-XL/2",
- from_pretrained="PixArt-XL-2-1024-MS.pth",
+ from_pretrained=None,
input_sq_size=512, # pretrained model is trained on 512x512
enable_flashattn=True,
enable_layernorm_kernel=True,
diff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.py
index 051aca3..7d416be 100644
--- a/opensora/utils/ckpt_utils.py
+++ b/opensora/utils/ckpt_utils.py
@@ -32,7 +32,7 @@ pretrained_models = {
def reparameter(ckpt, name=None, model=None):
- if "DiT" in name:
+ if "DiT-XL" in name and "STDiT" not in name:
ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)
del ckpt["pos_embed"]
elif "Latte" in name:
@@ -47,12 +47,16 @@ def reparameter(ckpt, name=None, model=None):
# different text length
if "y_embedder.y_embedding" in ckpt:
- if ckpt["y_embedder.y_embedding"].shape[0] > model.y_embedder.y_embedding.shape[0]:
+ if ckpt["y_embedder.y_embedding"].shape[0] < model.y_embedder.y_embedding.shape[0]:
additional_length = model.y_embedder.y_embedding.shape[0] - ckpt["y_embedder.y_embedding"].shape[0]
new_y_embedding = torch.randn(additional_length, model.y_embedder.y_embedding.shape[1])
ckpt["y_embedder.y_embedding"] = torch.cat([ckpt["y_embedder.y_embedding"], new_y_embedding], dim=0)
- elif ckpt["y_embedder.y_embedding"].shape[0] < model.y_embedder.y_embedding.shape[0]:
+ elif ckpt["y_embedder.y_embedding"].shape[0] > model.y_embedder.y_embedding.shape[0]:
ckpt["y_embedder.y_embedding"] = ckpt["y_embedder.y_embedding"][: model.y_embedder.y_embedding.shape[0]]
+ if "pos_embed_temporal" in ckpt:
+ del ckpt["pos_embed_temporal"]
+ if "pos_embed" in ckpt:
+ del ckpt["pos_embed"]
return ckpt
@@ -63,17 +67,11 @@ def find_model(model_name, model=None):
if model_name in pretrained_models: # Find/download our pre-trained DiT checkpoints
model_ckpt = download_model(model_name)
model_ckpt = reparameter(model_ckpt, model_name, model=model)
- return model_ckpt
else: # Load a custom DiT checkpoint:
assert os.path.isfile(model_name), f"Could not find DiT checkpoint at {model_name}"
- checkpoint = torch.load(model_name, map_location=lambda storage, loc: storage)
- if "pos_embed_temporal" in checkpoint:
- del checkpoint["pos_embed_temporal"]
- if "pos_embed" in checkpoint:
- del checkpoint["pos_embed"]
- if "ema" in checkpoint: # supports checkpoints from train.py
- checkpoint = checkpoint["ema"]
- return checkpoint
+ model_ckpt = torch.load(model_name, map_location=lambda storage, loc: storage)
+ model_ckpt = reparameter(model_ckpt, model_name, model=model)
+ return model_ckpt
def download_model(model_name):
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.py
index 2c11f76..16166d7 100644
--- a/tools/datasets/csvutil.py
+++ b/tools/datasets/csvutil.py
@@ -449,18 +449,10 @@ def main(args):
# make intersection
if args.intersection is not None:
- data_int = pd.read_csv(args.intersection)
- print(f"Intersection csv contains {len(data_int)} samples.")
- data = data[data["path"].isin(data_int["path"])]
- input_name += f"-{os.path.basename(args.intersection).split('.')[0]}"
- # get additional data column from intersection
- int_new_keys = [k for k in data_int.keys() if not k in data]
- for k in int_new_keys:
- # get corresponding values
- data[k] = apply(data["path"], lambda x: get_key_val_given_path(x, k, data_int))
-
- print(f"added {len(int_new_keys)} keys from {args.intersection}: {int_new_keys}")
- print(f"Filtered number of samples: {len(data)}.")
+ data_new = pd.read_csv(args.intersection)
+ print(f"Intersection csv contains {len(data_new)} samples.")
+ data = pd.merge(data, data_new, on="path", how="inner")
+ print(f"Intersection number of samples: {len(data)}.")
# get output path
output_path = get_output_path(args, input_name)