Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

llava fix and config modification

Zangwei Zhengcdd3b5eb74e8
diff --git a/configs/opensora-v1-1/train/Vx360p.py b/configs/opensora-v1-1/train/image.py
similarity index 84%
rename from configs/opensora-v1-1/train/Vx360p.py
rename to configs/opensora-v1-1/train/image.py
index edf022b..9b6b311 100644
--- a/configs/opensora-v1-1/train/Vx360p.py
+++ b/configs/opensora-v1-1/train/image.py
@@ -8,9 +8,9 @@ dataset = dict(
transform_name="resize_crop",
)
bucket_config = {
- "240p": {1: (1.0, 128), 16: (1.0, 4), 48: (1.0, 4), 72: (1.0, 2)},
- "360p": {1: (1.0, 2), 16: (0.5, 2), 24: (0.5, 2), 72: (0.0, None)},
- "720p": {1: (1.0, 2), 16: (0.5, 1), 72: (0.0, None)},
+ "240p": {1: (1.0, 128)},
+ "360p": {1: (1.0, 2)},
+ "720p": {1: (1.0, 2)},
"1080p": {1: (1.0, 2)},
}
diff --git a/configs/opensora-v1-1/train/test.py b/configs/opensora-v1-1/train/test.py
index e05db5e..1877160 100644
--- a/configs/opensora-v1-1/train/test.py
+++ b/configs/opensora-v1-1/train/test.py
@@ -8,7 +8,7 @@ dataset = dict(
transform_name="resize_crop",
)
bucket_config = {
- "240p": {1: (1.0, 64)},
+ "240p": {1: (1.0, 128)},
}
# Define acceleration
diff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.py
index 87c5ac1..12568bc 100644
--- a/opensora/utils/ckpt_utils.py
+++ b/opensora/utils/ckpt_utils.py
@@ -29,7 +29,7 @@ pretrained_models = {
}
-def reparameter(ckpt, name=None):
+def reparameter(ckpt, name=None, model=None):
if "DiT" in name:
ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)
del ckpt["pos_embed"]
@@ -42,17 +42,23 @@ def reparameter(ckpt, name=None):
ckpt = ckpt["state_dict"]
ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)
del ckpt["pos_embed"]
+
+ # different text length
+ if ckpt["y_embedder.y_embedding"].shape[0] != model.y_embedder.y_embedding.shape[0]:
+ additional_length = model.y_embedder.y_embedding.shape[0] - ckpt["y_embedder.y_embedding"].shape[0]
+ new_y_embedding = torch.randn(additional_length, model.y_embedder.y_embedding.shape[1])
+ ckpt["y_embedder.y_embedding"] = torch.cat([ckpt["y_embedder.y_embedding"], new_y_embedding], dim=0)
return ckpt
-def find_model(model_name):
+def find_model(model_name, model=None):
"""
Finds a pre-trained DiT model, downloading it if necessary. Alternatively, loads a model from a local path.
"""
if model_name in pretrained_models: # Find/download our pre-trained DiT checkpoints
- model = download_model(model_name)
- model = reparameter(model, model_name)
- return model
+ model_ckpt = download_model(model_name)
+ model_ckpt = reparameter(model_ckpt, model_name, model=model)
+ return model_ckpt
else: # Load a custom DiT checkpoint:
assert os.path.isfile(model_name), f"Could not find DiT checkpoint at {model_name}"
checkpoint = torch.load(model_name, map_location=lambda storage, loc: storage)
@@ -223,7 +229,7 @@ def create_logger(logging_dir):
def load_checkpoint(model, ckpt_path, save_as_pt=True):
if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):
- state_dict = find_model(ckpt_path)
+ state_dict = find_model(ckpt_path, model=model)
missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)
print(f"Missing keys: {missing_keys}")
print(f"Unexpected keys: {unexpected_keys}")
diff --git a/tools/caption/caption_gpt4.py b/tools/caption/caption_gpt4.py
index 8618651..c0d1359 100644
--- a/tools/caption/caption_gpt4.py
+++ b/tools/caption/caption_gpt4.py
@@ -50,7 +50,7 @@ def main(args):
output_file = os.path.splitext(args.input)[0] + "_caption.csv"
f = open(output_file, "w")
writer = csv.writer(f)
- writer.writerow(["video", "caption"])
+ writer.writerow(["video", "text"])
# ======================================================
# 2. generate captions
diff --git a/tools/caption/caption_llava.py b/tools/caption/caption_llava.py
index 3a673aa..5d33290 100644
--- a/tools/caption/caption_llava.py
+++ b/tools/caption/caption_llava.py
@@ -145,7 +145,7 @@ def main(args):
output_file_split = output_file.replace(".csv", f"_part{dp_rank}.csv")
dp_file = open(output_file_split, "w")
dp_writer = csv.writer(dp_file)
- dp_writer.writerow(["path", "caption", "num_frames"])
+ dp_writer.writerow(["path", "text", "num_frames"])
# ======================================================
# 5. generate captions
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.py
index d83892e..9d66426 100644
--- a/tools/datasets/csvutil.py
+++ b/tools/datasets/csvutil.py
@@ -71,6 +71,8 @@ LLAVA_PREFIX = [
def remove_caption_prefix(caption):
for prefix in LLAVA_PREFIX:
+ if isinstance(caption, float):
+ breakpoint()
if caption.startswith(prefix):
caption = caption[len(prefix) :].strip()
if caption[0].islower():
@@ -106,10 +108,13 @@ def parse_args():
parser.add_argument("--sort-descending", type=str, default=None)
parser.add_argument("--sort-ascending", type=str, default=None)
parser.add_argument("--difference", type=str, default=None)
+ parser.add_argument("--intersection", type=str, default=None)
# path processing
- parser.add_argument("--abspath", type=str, default=None)
parser.add_argument("--relpath", type=str, default=None)
+ parser.add_argument("--abspath", type=str, default=None)
+ # path filtering
+ parser.add_argument("--ext", action="store_true")
# caption filtering
parser.add_argument("--remove-empty-caption", action="store_true")
parser.add_argument("--lang", type=str, default=None)
@@ -137,10 +142,13 @@ def get_output_path(args, input_name):
dir_path = os.path.dirname(args.input[0])
# path processing
- if args.abspath is not None:
- name += "_abspath"
if args.relpath is not None:
name += "_relpath"
+ if args.abspath is not None:
+ name += "_abspath"
+ # path filtering
+ if args.ext:
+ name += "_ext"
# caption filtering
if args.remove_empty_caption:
name += "_noempty"
@@ -200,6 +208,14 @@ def main(args):
input_name += f"-{os.path.basename(args.difference).split('.')[0]}"
print(f"Filtered number of samples: {len(data)}.")
+ # make intersection
+ if args.intersection is not None:
+ data_int = pd.read_csv(args.intersection)
+ print(f"Intersection csv contains {len(data_int)} samples.")
+ data = data[data["path"].isin(data_int["path"])]
+ input_name += f"-{os.path.basename(args.intersection).split('.')[0]}"
+ print(f"Filtered number of samples: {len(data)}.")
+
# get output path
output_path = get_output_path(args, input_name)
@@ -208,12 +224,10 @@ def main(args):
detect_lang = build_lang_detector(args.lang)
# processing
- if args.abspath is not None:
- assert args.relpath is None
- data["path"] = apply(data["path"], lambda x: os.path.join(args.abspath, x))
if args.relpath is not None:
- assert args.abspath is None
data["path"] = apply(data["path"], lambda x: os.path.relpath(x, args.relpath))
+ if args.abspath is not None:
+ data["path"] = apply(data["path"], lambda x: os.path.join(args.abspath, x))
if args.remove_caption_prefix:
assert "text" in data.columns
data["text"] = apply(data["text"], remove_caption_prefix)
@@ -225,9 +239,13 @@ def main(args):
data["num_frames"], data["height"], data["width"], data["aspect_ratio"], data["fps"] = zip(*info)
# filtering
+ if args.ext:
+ assert "path" in data.columns
+ data = data[apply(data["path"], os.path.exists)]
if args.remove_empty_caption:
assert "text" in data.columns
data = data[data["text"].str.len() > 0]
+ data = data[~data["text"].isna()]
if args.remove_url:
assert "text" in data.columns
data = data[~data["text"].str.contains(r"(?P<url>https?://[^\s]+)", regex=True)]
@@ -258,9 +276,9 @@ def main(args):
if args.shard is not None:
sharded_data = np.array_split(data, args.shard)
for i in range(args.shard):
- output_path = output_path.replace(".csv", f"_{i}.csv")
- sharded_data[i].to_csv(output_path, index=False)
- print(f"Saved {len(sharded_data[i])} samples to {output_path}.")
+ output_path_s = output_path.replace(".csv", f"_{i}.csv")
+ sharded_data[i].to_csv(output_path_s, index=False)
+ print(f"Saved {len(sharded_data[i])} samples to {output_path_s}.")
else:
data.to_csv(output_path, index=False)
print(f"Saved {len(data)} samples to {output_path}.")