Open-Sora: Democratizing Efficient Video Production for All
Python
update inference
zhengzangwb6b2535c7914
diff --git a/--llm-refine b/--llm-refinenew file mode 100644index 0000000..e69de29diff --git a/configs/opensora-v1-2/inference/sample.py b/configs/opensora-v1-2/inference/sample.pyindex 72cb47b..f07587c 100644--- a/configs/opensora-v1-2/inference/sample.py+++ b/configs/opensora-v1-2/inference/sample.py@@ -5,17 +5,6 @@ fps = 24frame_interval = 1save_fps = 24-prompt_path = "./assets/texts/t2v_sora.txt"-# == Uncomment the following line to use the prompt below ==-# prompt = [-# 'Drone view of waves crashing against the rugged cliffs along Big Sur\'s garay point beach. {"reference_path": "assets/images/condition/cliff.png", "mask_strategy": "0"}',-# 'A breathtaking sunrise scene.{"reference_path": "assets/images/condition/sunset1.png","mask_strategy": "0"}',-# 'A car driving on the ocean.{"reference_path": "https://cdn.openai.com/tmp/s/interp/d0.mp4","mask_strategy": "0,0,-8,0,8"}',-# 'A snowy forest.{"reference_path": "https://cdn.pixabay.com/video/2021/04/25/72171-542991404_large.mp4","mask_strategy": "0,0,0,0,15,0.8"}',-# 'A breathtaking sunrise scene.{"reference_path": "assets/images/condition/sunset1.png;assets/images/condition/sunset2.png","mask_strategy": "0;0,1,0,-1,1"}',-# '|0|a white jeep equipped with a roof rack driving on a dirt road in a coniferous forest.|2|a white jeep equipped with a roof rack driving on a dirt road in the desert.|4|a white jeep equipped with a roof rack driving on a dirt road in a mountain.|6|A white jeep equipped with a roof rack driving on a dirt road in a city.|8|a white jeep equipped with a roof rack driving on a dirt road on the surface of a river.|10|a white jeep equipped with a roof rack driving on a dirt road under the lake.|12|a white jeep equipped with a roof rack flying into the sky.|14|a white jeep equipped with a roof rack driving in the universe. Earth is the background.{"reference_path": "https://cdn.openai.com/tmp/s/interp/d0.mp4", "mask_strategy": "0,0,0,0,15"}',-# ]-save_dir = "./samples/samples/"seed = 42batch_size = 1diff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.pyindex 1f67a76..f854124 100644--- a/opensora/utils/config_utils.py+++ b/opensora/utils/config_utils.py@@ -46,6 +46,7 @@ def parse_args(training=False):parser.add_argument("--prompt-path", default=None, type=str, help="path to prompt txt file")parser.add_argument("--prompt", default=None, type=str, nargs="+", help="prompt list")parser.add_argument("--llm-refine", default=None, type=str2bool, help="enable LLM refine")+ parser.add_argument("--prompt-generator", default=None, type=str, help="prompt generator")# image/videoparser.add_argument("--num-frames", default=None, type=str, help="number of frames")diff --git a/opensora/utils/inference_utils.py b/opensora/utils/inference_utils.pyindex 56575d3..fcf24ca 100644--- a/opensora/utils/inference_utils.py+++ b/opensora/utils/inference_utils.py@@ -115,9 +115,10 @@ def extract_prompts_loop(prompts, num_loop):ret_prompts.append(prompt)return ret_prompts+def split_prompt(prompt_text):if prompt_text.startswith("|0|"):- # this is for prompts which look like+ # this is for prompts which look like# |0| a beautiful day |1| a sunny day |2| a rainy day# we want to parse it into a list of prompts with the loop indexprompt_list = prompt_text.split("|")[1:]@@ -131,7 +132,8 @@ def split_prompt(prompt_text):return text_list, loop_idxelse:return [prompt_text], None-++def merge_prompt(text_list, loop_idx_list=None):if loop_idx_list is None:return text_list[0]@@ -285,9 +287,11 @@ def refine_prompt_by_openai(prompt):response = get_openai_response(REFINE_PROMPTS, prompt)return response+def has_openai_key():return "OPENAI_API_KEY" in os.environ+def refine_prompts_by_openai(prompts):new_prompts = []for prompt in prompts:diff --git a/scripts/inference.py b/scripts/inference.pyindex 1299be6..5e02356 100644--- a/scripts/inference.py+++ b/scripts/inference.py@@ -29,7 +29,7 @@ from opensora.utils.inference_utils import (prepare_multi_resolution_info,refine_prompts_by_openai,split_prompt,- merge_prompt+ merge_prompt,)from opensora.utils.misc import all_exists, create_logger, is_distributed, is_main_process, to_torch_dtype@@ -115,8 +115,10 @@ def main():prompts = cfg.get("prompt", None)start_idx = cfg.get("start_index", 0)if prompts is None:- assert cfg.get("prompt_path", None) is not None, "Prompt or prompt_path must be provided"- prompts = load_prompts(cfg.prompt_path, start_idx, cfg.get("end_index", None))+ if cfg.get("prompt_path", None) is not None:+ prompts = load_prompts(cfg.prompt_path, start_idx, cfg.get("end_index", None))+ else:+ prompts = [cfg.get("prompt_generator", "")] * 1_000_000 # endless loop# == prepare reference ==reference_path = cfg.get("reference_path", [""] * len(prompts))@@ -180,19 +182,19 @@ def main():if prompt_as_path and all_exists(save_paths):continue- # == process prompts step by step ==+ # == process prompts step by step ==# 0. split prompt# each element in the list is [prompt_segment_list, loop_idx_list]batched_prompt_segment_list = []batched_loop_idx_list = []for prompt in batch_prompts:- prompt_segment_list, loop_idx_list = split_prompt(prompt)+ prompt_segment_list, loop_idx_list = split_prompt(prompt)batched_prompt_segment_list.append(prompt_segment_list)batched_loop_idx_list.append(loop_idx_list)-+# 1. refine prompt by openaiif cfg.get("llm_refine", False):- # only call openai API when+ # only call openai API when# 1. seq parallel is not enabled# 2. seq parallel is enabled and the process is rank 0if not enable_sequence_parallelism or (enable_sequence_parallelism and is_main_process()):@@ -202,10 +204,16 @@ def main():# sync the prompt if using seq parallelif enable_sequence_parallelism:coordinator.block_all()- prompt_segment_length = [len(prompt_segment_list) for prompt_segment_list in batched_prompt_segment_list]+ prompt_segment_length = [+ len(prompt_segment_list) for prompt_segment_list in batched_prompt_segment_list+ ]# flatten the prompt segment list- batched_prompt_segment_list = [prompt_segment for prompt_segment_list in batched_prompt_segment_list for prompt_segment in prompt_segment_list]+ batched_prompt_segment_list = [+ prompt_segment+ for prompt_segment_list in batched_prompt_segment_list+ for prompt_segment in prompt_segment_list+ ]# create a list of size equal to world sizebroadcast_obj_list = [batched_prompt_segment_list] * coordinator.world_size@@ -216,9 +224,9 @@ def main():start_idx = 0all_prompts = broadcast_obj_list[0]for num_segment in prompt_segment_length:- batched_prompt_segment_list.append(all_prompts[start_idx:start_idx+num_segment])+ batched_prompt_segment_list.append(all_prompts[start_idx : start_idx + num_segment])start_idx += num_segment-+# 2. append scorefor idx, prompt_segment_list in enumerate(batched_prompt_segment_list):batched_prompt_segment_list[idx] = append_score_to_prompts(@@ -246,13 +254,7 @@ def main():# == add condition frames for loop ==if loop_i > 0:refs, ms = append_generated(- vae,- video_clips[-1],- refs,- ms,- loop_i,- condition_frame_length,- condition_frame_edit+ vae, video_clips[-1], refs, ms, loop_i, condition_frame_length, condition_frame_edit)# == sampling ==