Open-Sora: Democratizing Efficient Video Production for All
Python
llava fix and config modification
Zangwei Zhengcdd3b5eb74e8
diff --git a/configs/opensora-v1-1/train/Vx360p.py b/configs/opensora-v1-1/train/image.pysimilarity index 84%rename from configs/opensora-v1-1/train/Vx360p.pyrename to configs/opensora-v1-1/train/image.pyindex edf022b..9b6b311 100644--- a/configs/opensora-v1-1/train/Vx360p.py+++ b/configs/opensora-v1-1/train/image.py@@ -8,9 +8,9 @@ dataset = dict(transform_name="resize_crop",)bucket_config = {- "240p": {1: (1.0, 128), 16: (1.0, 4), 48: (1.0, 4), 72: (1.0, 2)},- "360p": {1: (1.0, 2), 16: (0.5, 2), 24: (0.5, 2), 72: (0.0, None)},- "720p": {1: (1.0, 2), 16: (0.5, 1), 72: (0.0, None)},+ "240p": {1: (1.0, 128)},+ "360p": {1: (1.0, 2)},+ "720p": {1: (1.0, 2)},"1080p": {1: (1.0, 2)},}diff --git a/configs/opensora-v1-1/train/test.py b/configs/opensora-v1-1/train/test.pyindex e05db5e..1877160 100644--- a/configs/opensora-v1-1/train/test.py+++ b/configs/opensora-v1-1/train/test.py@@ -8,7 +8,7 @@ dataset = dict(transform_name="resize_crop",)bucket_config = {- "240p": {1: (1.0, 64)},+ "240p": {1: (1.0, 128)},}# Define accelerationdiff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.pyindex 87c5ac1..12568bc 100644--- a/opensora/utils/ckpt_utils.py+++ b/opensora/utils/ckpt_utils.py@@ -29,7 +29,7 @@ pretrained_models = {}-def reparameter(ckpt, name=None):+def reparameter(ckpt, name=None, model=None):if "DiT" in name:ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)del ckpt["pos_embed"]@@ -42,17 +42,23 @@ def reparameter(ckpt, name=None):ckpt = ckpt["state_dict"]ckpt["x_embedder.proj.weight"] = ckpt["x_embedder.proj.weight"].unsqueeze(2)del ckpt["pos_embed"]++ # different text length+ if ckpt["y_embedder.y_embedding"].shape[0] != model.y_embedder.y_embedding.shape[0]:+ additional_length = model.y_embedder.y_embedding.shape[0] - ckpt["y_embedder.y_embedding"].shape[0]+ new_y_embedding = torch.randn(additional_length, model.y_embedder.y_embedding.shape[1])+ ckpt["y_embedder.y_embedding"] = torch.cat([ckpt["y_embedder.y_embedding"], new_y_embedding], dim=0)return ckpt-def find_model(model_name):+def find_model(model_name, model=None):"""Finds a pre-trained DiT model, downloading it if necessary. Alternatively, loads a model from a local path."""if model_name in pretrained_models: # Find/download our pre-trained DiT checkpoints- model = download_model(model_name)- model = reparameter(model, model_name)- return model+ model_ckpt = download_model(model_name)+ model_ckpt = reparameter(model_ckpt, model_name, model=model)+ return model_ckptelse: # Load a custom DiT checkpoint:assert os.path.isfile(model_name), f"Could not find DiT checkpoint at {model_name}"checkpoint = torch.load(model_name, map_location=lambda storage, loc: storage)@@ -223,7 +229,7 @@ def create_logger(logging_dir):def load_checkpoint(model, ckpt_path, save_as_pt=True):if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):- state_dict = find_model(ckpt_path)+ state_dict = find_model(ckpt_path, model=model)missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)print(f"Missing keys: {missing_keys}")print(f"Unexpected keys: {unexpected_keys}")diff --git a/tools/caption/caption_gpt4.py b/tools/caption/caption_gpt4.pyindex 8618651..c0d1359 100644--- a/tools/caption/caption_gpt4.py+++ b/tools/caption/caption_gpt4.py@@ -50,7 +50,7 @@ def main(args):output_file = os.path.splitext(args.input)[0] + "_caption.csv"f = open(output_file, "w")writer = csv.writer(f)- writer.writerow(["video", "caption"])+ writer.writerow(["video", "text"])# ======================================================# 2. generate captionsdiff --git a/tools/caption/caption_llava.py b/tools/caption/caption_llava.pyindex 3a673aa..5d33290 100644--- a/tools/caption/caption_llava.py+++ b/tools/caption/caption_llava.py@@ -145,7 +145,7 @@ def main(args):output_file_split = output_file.replace(".csv", f"_part{dp_rank}.csv")dp_file = open(output_file_split, "w")dp_writer = csv.writer(dp_file)- dp_writer.writerow(["path", "caption", "num_frames"])+ dp_writer.writerow(["path", "text", "num_frames"])# ======================================================# 5. generate captionsdiff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex d83892e..9d66426 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -71,6 +71,8 @@ LLAVA_PREFIX = [def remove_caption_prefix(caption):for prefix in LLAVA_PREFIX:+ if isinstance(caption, float):+ breakpoint()if caption.startswith(prefix):caption = caption[len(prefix) :].strip()if caption[0].islower():@@ -106,10 +108,13 @@ def parse_args():parser.add_argument("--sort-descending", type=str, default=None)parser.add_argument("--sort-ascending", type=str, default=None)parser.add_argument("--difference", type=str, default=None)+ parser.add_argument("--intersection", type=str, default=None)# path processing- parser.add_argument("--abspath", type=str, default=None)parser.add_argument("--relpath", type=str, default=None)+ parser.add_argument("--abspath", type=str, default=None)+ # path filtering+ parser.add_argument("--ext", action="store_true")# caption filteringparser.add_argument("--remove-empty-caption", action="store_true")parser.add_argument("--lang", type=str, default=None)@@ -137,10 +142,13 @@ def get_output_path(args, input_name):dir_path = os.path.dirname(args.input[0])# path processing- if args.abspath is not None:- name += "_abspath"if args.relpath is not None:name += "_relpath"+ if args.abspath is not None:+ name += "_abspath"+ # path filtering+ if args.ext:+ name += "_ext"# caption filteringif args.remove_empty_caption:name += "_noempty"@@ -200,6 +208,14 @@ def main(args):input_name += f"-{os.path.basename(args.difference).split('.')[0]}"print(f"Filtered number of samples: {len(data)}.")+ # make intersection+ if args.intersection is not None:+ data_int = pd.read_csv(args.intersection)+ print(f"Intersection csv contains {len(data_int)} samples.")+ data = data[data["path"].isin(data_int["path"])]+ input_name += f"-{os.path.basename(args.intersection).split('.')[0]}"+ print(f"Filtered number of samples: {len(data)}.")+# get output pathoutput_path = get_output_path(args, input_name)@@ -208,12 +224,10 @@ def main(args):detect_lang = build_lang_detector(args.lang)# processing- if args.abspath is not None:- assert args.relpath is None- data["path"] = apply(data["path"], lambda x: os.path.join(args.abspath, x))if args.relpath is not None:- assert args.abspath is Nonedata["path"] = apply(data["path"], lambda x: os.path.relpath(x, args.relpath))+ if args.abspath is not None:+ data["path"] = apply(data["path"], lambda x: os.path.join(args.abspath, x))if args.remove_caption_prefix:assert "text" in data.columnsdata["text"] = apply(data["text"], remove_caption_prefix)@@ -225,9 +239,13 @@ def main(args):data["num_frames"], data["height"], data["width"], data["aspect_ratio"], data["fps"] = zip(*info)# filtering+ if args.ext:+ assert "path" in data.columns+ data = data[apply(data["path"], os.path.exists)]if args.remove_empty_caption:assert "text" in data.columnsdata = data[data["text"].str.len() > 0]+ data = data[~data["text"].isna()]if args.remove_url:assert "text" in data.columnsdata = data[~data["text"].str.contains(r"(?P<url>https?://[^\s]+)", regex=True)]@@ -258,9 +276,9 @@ def main(args):if args.shard is not None:sharded_data = np.array_split(data, args.shard)for i in range(args.shard):- output_path = output_path.replace(".csv", f"_{i}.csv")- sharded_data[i].to_csv(output_path, index=False)- print(f"Saved {len(sharded_data[i])} samples to {output_path}.")+ output_path_s = output_path.replace(".csv", f"_{i}.csv")+ sharded_data[i].to_csv(output_path_s, index=False)+ print(f"Saved {len(sharded_data[i])} samples to {output_path_s}.")else:data.to_csv(output_path, index=False)print(f"Saved {len(data)} samples to {output_path}.")