Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

update inference

zhengzangwb6b2535c7914
diff --git a/--llm-refine b/--llm-refine
new file mode 100644
index 0000000..e69de29
diff --git a/configs/opensora-v1-2/inference/sample.py b/configs/opensora-v1-2/inference/sample.py
index 72cb47b..f07587c 100644
--- a/configs/opensora-v1-2/inference/sample.py
+++ b/configs/opensora-v1-2/inference/sample.py
@@ -5,17 +5,6 @@ fps = 24
frame_interval = 1
save_fps = 24
-prompt_path = "./assets/texts/t2v_sora.txt"
-# == Uncomment the following line to use the prompt below ==
-# prompt = [
-# 'Drone view of waves crashing against the rugged cliffs along Big Sur\'s garay point beach. {"reference_path": "assets/images/condition/cliff.png", "mask_strategy": "0"}',
-# 'A breathtaking sunrise scene.{"reference_path": "assets/images/condition/sunset1.png","mask_strategy": "0"}',
-# 'A car driving on the ocean.{"reference_path": "https://cdn.openai.com/tmp/s/interp/d0.mp4","mask_strategy": "0,0,-8,0,8"}',
-# 'A snowy forest.{"reference_path": "https://cdn.pixabay.com/video/2021/04/25/72171-542991404_large.mp4","mask_strategy": "0,0,0,0,15,0.8"}',
-# 'A breathtaking sunrise scene.{"reference_path": "assets/images/condition/sunset1.png;assets/images/condition/sunset2.png","mask_strategy": "0;0,1,0,-1,1"}',
-# '|0|a white jeep equipped with a roof rack driving on a dirt road in a coniferous forest.|2|a white jeep equipped with a roof rack driving on a dirt road in the desert.|4|a white jeep equipped with a roof rack driving on a dirt road in a mountain.|6|A white jeep equipped with a roof rack driving on a dirt road in a city.|8|a white jeep equipped with a roof rack driving on a dirt road on the surface of a river.|10|a white jeep equipped with a roof rack driving on a dirt road under the lake.|12|a white jeep equipped with a roof rack flying into the sky.|14|a white jeep equipped with a roof rack driving in the universe. Earth is the background.{"reference_path": "https://cdn.openai.com/tmp/s/interp/d0.mp4", "mask_strategy": "0,0,0,0,15"}',
-# ]
-
save_dir = "./samples/samples/"
seed = 42
batch_size = 1
diff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.py
index 1f67a76..f854124 100644
--- a/opensora/utils/config_utils.py
+++ b/opensora/utils/config_utils.py
@@ -46,6 +46,7 @@ def parse_args(training=False):
parser.add_argument("--prompt-path", default=None, type=str, help="path to prompt txt file")
parser.add_argument("--prompt", default=None, type=str, nargs="+", help="prompt list")
parser.add_argument("--llm-refine", default=None, type=str2bool, help="enable LLM refine")
+ parser.add_argument("--prompt-generator", default=None, type=str, help="prompt generator")
# image/video
parser.add_argument("--num-frames", default=None, type=str, help="number of frames")
diff --git a/opensora/utils/inference_utils.py b/opensora/utils/inference_utils.py
index 56575d3..fcf24ca 100644
--- a/opensora/utils/inference_utils.py
+++ b/opensora/utils/inference_utils.py
@@ -115,9 +115,10 @@ def extract_prompts_loop(prompts, num_loop):
ret_prompts.append(prompt)
return ret_prompts
+
def split_prompt(prompt_text):
if prompt_text.startswith("|0|"):
- # this is for prompts which look like
+ # this is for prompts which look like
# |0| a beautiful day |1| a sunny day |2| a rainy day
# we want to parse it into a list of prompts with the loop index
prompt_list = prompt_text.split("|")[1:]
@@ -131,7 +132,8 @@ def split_prompt(prompt_text):
return text_list, loop_idx
else:
return [prompt_text], None
-
+
+
def merge_prompt(text_list, loop_idx_list=None):
if loop_idx_list is None:
return text_list[0]
@@ -285,9 +287,11 @@ def refine_prompt_by_openai(prompt):
response = get_openai_response(REFINE_PROMPTS, prompt)
return response
+
def has_openai_key():
return "OPENAI_API_KEY" in os.environ
+
def refine_prompts_by_openai(prompts):
new_prompts = []
for prompt in prompts:
diff --git a/scripts/inference.py b/scripts/inference.py
index 1299be6..5e02356 100644
--- a/scripts/inference.py
+++ b/scripts/inference.py
@@ -29,7 +29,7 @@ from opensora.utils.inference_utils import (
prepare_multi_resolution_info,
refine_prompts_by_openai,
split_prompt,
- merge_prompt
+ merge_prompt,
)
from opensora.utils.misc import all_exists, create_logger, is_distributed, is_main_process, to_torch_dtype
@@ -115,8 +115,10 @@ def main():
prompts = cfg.get("prompt", None)
start_idx = cfg.get("start_index", 0)
if prompts is None:
- assert cfg.get("prompt_path", None) is not None, "Prompt or prompt_path must be provided"
- prompts = load_prompts(cfg.prompt_path, start_idx, cfg.get("end_index", None))
+ if cfg.get("prompt_path", None) is not None:
+ prompts = load_prompts(cfg.prompt_path, start_idx, cfg.get("end_index", None))
+ else:
+ prompts = [cfg.get("prompt_generator", "")] * 1_000_000 # endless loop
# == prepare reference ==
reference_path = cfg.get("reference_path", [""] * len(prompts))
@@ -180,19 +182,19 @@ def main():
if prompt_as_path and all_exists(save_paths):
continue
- # == process prompts step by step ==
+ # == process prompts step by step ==
# 0. split prompt
# each element in the list is [prompt_segment_list, loop_idx_list]
batched_prompt_segment_list = []
batched_loop_idx_list = []
for prompt in batch_prompts:
- prompt_segment_list, loop_idx_list = split_prompt(prompt)
+ prompt_segment_list, loop_idx_list = split_prompt(prompt)
batched_prompt_segment_list.append(prompt_segment_list)
batched_loop_idx_list.append(loop_idx_list)
-
+
# 1. refine prompt by openai
if cfg.get("llm_refine", False):
- # only call openai API when
+ # only call openai API when
# 1. seq parallel is not enabled
# 2. seq parallel is enabled and the process is rank 0
if not enable_sequence_parallelism or (enable_sequence_parallelism and is_main_process()):
@@ -202,10 +204,16 @@ def main():
# sync the prompt if using seq parallel
if enable_sequence_parallelism:
coordinator.block_all()
- prompt_segment_length = [len(prompt_segment_list) for prompt_segment_list in batched_prompt_segment_list]
+ prompt_segment_length = [
+ len(prompt_segment_list) for prompt_segment_list in batched_prompt_segment_list
+ ]
# flatten the prompt segment list
- batched_prompt_segment_list = [prompt_segment for prompt_segment_list in batched_prompt_segment_list for prompt_segment in prompt_segment_list]
+ batched_prompt_segment_list = [
+ prompt_segment
+ for prompt_segment_list in batched_prompt_segment_list
+ for prompt_segment in prompt_segment_list
+ ]
# create a list of size equal to world size
broadcast_obj_list = [batched_prompt_segment_list] * coordinator.world_size
@@ -216,9 +224,9 @@ def main():
start_idx = 0
all_prompts = broadcast_obj_list[0]
for num_segment in prompt_segment_length:
- batched_prompt_segment_list.append(all_prompts[start_idx:start_idx+num_segment])
+ batched_prompt_segment_list.append(all_prompts[start_idx : start_idx + num_segment])
start_idx += num_segment
-
+
# 2. append score
for idx, prompt_segment_list in enumerate(batched_prompt_segment_list):
batched_prompt_segment_list[idx] = append_score_to_prompts(
@@ -246,13 +254,7 @@ def main():
# == add condition frames for loop ==
if loop_i > 0:
refs, ms = append_generated(
- vae,
- video_clips[-1],
- refs,
- ms,
- loop_i,
- condition_frame_length,
- condition_frame_edit
+ vae, video_clips[-1], refs, ms, loop_i, condition_frame_length, condition_frame_edit
)
# == sampling ==