Open-Sora: Democratizing Efficient Video Production for All
Python
[wip] prompt with json
zhengzangw2b42ca82d800
diff --git a/configs/opensora-v1-1/inference/sample-ref-inprompt.py b/configs/opensora-v1-1/inference/sample-ref-inprompt.pynew file mode 100644index 0000000..3b75eb4--- /dev/null+++ b/configs/opensora-v1-1/inference/sample-ref-inprompt.py@@ -0,0 +1,47 @@+num_frames = 16+frame_interval = 3+fps = 24+image_size = (240, 426)+multi_resolution = "STDiT2"++# Condition+loop = 1+condition_frame_length = 4+prompt = [+ # "A car driving on the ocean.",+ 'Drone view of waves crashing against the rugged cliffs along Big Sur\'s garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff\'s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff\'s edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.{"reference_path": "assets/images/condition/cliff.png", "mask_strategy": "0,0,0,1,0"}',+ # "In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, seizing the moment, skillfully navigate the face of the wave.",+]++# Define model+model = dict(+ type="STDiT2-XL/2",+ from_pretrained=None,+ input_sq_size=512,+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+vae = dict(+ type="VideoAutoencoderKL",+ from_pretrained="stabilityai/sd-vae-ft-ema",+ cache_dir=None, # "/mnt/hdd/cached_models",+ micro_batch_size=4,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ cache_dir=None, # "/mnt/hdd/cached_models",+ model_max_length=200,+)+scheduler = dict(+ type="iddpm",+ num_sampling_steps=100,+ cfg_scale=7.0,+ cfg_channel=3, # or None+)+dtype = "bf16"++# Others+batch_size = 1+seed = 42+save_dir = "./samples/samples/"diff --git a/configs/opensora-v1-1/inference/sample-ref.py b/configs/opensora-v1-1/inference/sample-ref.pyindex 6e8e805..2712a6e 100644--- a/configs/opensora-v1-1/inference/sample-ref.py+++ b/configs/opensora-v1-1/inference/sample-ref.py@@ -7,26 +7,23 @@ multi_resolution = "STDiT2"# Conditionprompt_path = Noneprompt = [- "A car driving on a road in the middle of a desert.",- # "A man smiling",- # "Drone view of waves crashing against the rugged cliffs along Big Sur's garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff’s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.",+ # "A car driving on the ocean.",+ "Drone view of waves crashing against the rugged cliffs along Big Sur's garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff’s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.",# "In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, seizing the moment, skillfully navigate the face of the wave.",]loop = 1condition_frame_length = 4reference_path = [- "https://cdn.openai.com/tmp/s/interp/d0.mp4",- # "https://www.comp.nus.edu.sg/~youy/index_files/yangyou3.png",- # "assets/images/condition/cliff.png",+ # "https://cdn.openai.com/tmp/s/interp/d0.mp4",+ "assets/images/condition/cliff.png",# "assets/images/condition/wave.png",]# valid when reference_path is not None# (loop id, ref id, ref start, length, target start)mask_strategy = [- "0,0,0,8,0,0.5",- # "0,0,0,1,0",- # "0,0,0,1,0",+ # "0,0,0,8,0,0.3",+ "0,0,0,1,0",# "0,0,0,1,0",]diff --git a/eval/launch_sample.sh b/eval/launch_sample.shnew file mode 100644index 0000000..346312c--- /dev/null+++ b/eval/launch_sample.sh@@ -0,0 +1,25 @@+#!/bin/bash++set -x+set -e++CKPT=$1+if [[ $CKPT == *"ema"* ]]; then+ parentdir=$(dirname $CKPT)+ CKPT_BASE=$(basename $parentdir)_ema+else+ CKPT_BASE=$(basename $CKPT)+fi+LOG_BASE=logs/sample/CKPT_BASE+echo "Logging to $LOG_BASE"++CUDA_VISIBLE_DEVICES=0 bash eval/sample.sh $CKPT -1 >${LOG_BASE}_1.log 2>&1 &+CUDA_VISIBLE_DEVICES=1 bash eval/sample.sh $CKPT -3 >${LOG_BASE}_3.log 2>&1 &+CUDA_VISIBLE_DEVICES=2 bash eval/sample.sh $CKPT -2a >${LOG_BASE}_2a.log 2>&1 &+CUDA_VISIBLE_DEVICES=3 bash eval/sample.sh $CKPT -2b >${LOG_BASE}_2b.log 2>&1 &+CUDA_VISIBLE_DEVICES=4 bash eval/sample.sh $CKPT -2c >${LOG_BASE}_2c.log 2>&1 &+CUDA_VISIBLE_DEVICES=5 bash eval/sample.sh $CKPT -2d >${LOG_BASE}_2d.log 2>&1 &+CUDA_VISIBLE_DEVICES=6 bash eval/sample.sh $CKPT -2e >${LOG_BASE}_2e.log 2>&1 &+CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT -2f >${LOG_BASE}_2f.log 2>&1 &++# kill all by: pkill -f "inference"diff --git a/eval/sample.sh b/eval/sample.shindex 37320b0..675bf87 100644--- a/eval/sample.sh+++ b/eval/sample.sh@@ -51,6 +51,8 @@ function run_image() { # 10minfunction run_video_a() { # 20min# 2.1.1 16x240x426+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 16 --image-size 144 256 --sample-name sample_16x144x256+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 16 --image-size 256 256 --sample-name sample_16x256x256eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name sample_16x240x426# 2.1.2 16x720p multi-resolution@@ -129,49 +131,49 @@ function run_video_edit() { # 23min# vbench has 950 samples-function run_vbenck_a() {+function run_vbenck_a() { # 2heval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \--prompt-path assets/texts/VBench/all_dimension.txt \--start-index 0 --end-index 120}-function run_vbenck_b() {+function run_vbenck_b() { # 2heval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \--prompt-path assets/texts/VBench/all_dimension.txt \--start-index 120 --end-index 240}-function run_vbenck_c() {+function run_vbenck_c() { # 2heval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \--prompt-path assets/texts/VBench/all_dimension.txt \--start-index 240 --end-index 360}-function run_vbenck_d() {+function run_vbenck_d() { # 2heval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \--prompt-path assets/texts/VBench/all_dimension.txt \--start-index 360 --end-index 480}-function run_vbenck_e() {+function run_vbenck_e() { # 2heval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \--prompt-path assets/texts/VBench/all_dimension.txt \--start-index 480 --end-index 600}-function run_vbenck_f() {+function run_vbenck_f() { # 2heval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \--prompt-path assets/texts/VBench/all_dimension.txt \--start-index 600 --end-index 720}-function run_vbenck_g() {+function run_vbenck_g() { # 2heval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \--prompt-path assets/texts/VBench/all_dimension.txt \--start-index 720 --end-index 840}-function run_vbenck_h() {+function run_vbenck_h() { # 2heval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \--prompt-path assets/texts/VBench/all_dimension.txt \--start-index 840@@ -260,4 +262,5 @@ doneend=$(date +%s)runtime=$((end - start))+echo "Runtime: $runtime seconds"diff --git a/eval/vbench.sh b/eval/vbench.shindex 864cd9b..7065765 100644--- a/eval/vbench.sh+++ b/eval/vbench.sh@@ -1,11 +1,32 @@#!/bin/bash+set -e+# Base path for videos-videos_path='./samples/samples_stage1_41k_2_17k_vbench'-json_path='./eval/VBench_full_info.json'+videos_path=$1+videos_base=$(basename $videos_path)+json_path=./eval/VBench_full_info.json+output_path=./evaluation_results/$videos_base# Define the dimension list-dimensions=("subject_consistency" "background_consistency" "aesthetic_quality" "imaging_quality" "object_class" "multiple_objects" "color" "spatial_relationship" "scene" "temporal_style" "overall_consistency" "human_action" "temporal_flickering" "motion_smoothness" "dynamic_degree" "appearance_style")+dimensions=(+ "subject_consistency"+ "background_consistency"+ "aesthetic_quality"+ "imaging_quality"+ "object_class"+ "multiple_objects"+ "color"+ "spatial_relationship"+ "scene"+ "temporal_style"+ "overall_consistency"+ "human_action"+ "temporal_flickering"+ "motion_smoothness"+ "dynamic_degree"+ "appearance_style"+)# Loop over each dimensionfor i in "${!dimensions[@]}"; do@@ -16,5 +37,5 @@ for i in "${!dimensions[@]}"; doecho "$dimension $videos_path"# Run the evaluation script- vbench evaluate --videos_path $videos_path --dimension $dimension --full_json_dir $json_path+ vbench evaluate --videos_path $videos_path --dimension $dimension --full_json_dir $json_path --output_path $output_pathdonediff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.pyindex 78ee638..b5edb93 100644--- a/opensora/utils/config_utils.py+++ b/opensora/utils/config_utils.py@@ -71,6 +71,12 @@ def merge_args(cfg, args, training=False):if args.ckpt_path is not None:cfg.model["from_pretrained"] = args.ckpt_pathargs.ckpt_path = None+ if args.cfg_scale is not None:+ cfg.scheduler["cfg_scale"] = args.cfg_scale+ args.cfg_scale = None+ if args.num_sampling_steps is not None:+ cfg.scheduler["num_sampling_steps"] = args.num_sampling_steps+ args.num_sampling_steps = Noneif training and args.data_path is not None:cfg.dataset["data_path"] = args.data_pathargs.data_path = Nonediff --git a/scripts/inference-long.py b/scripts/inference-long.pyindex 69060dc..257755b 100644--- a/scripts/inference-long.py+++ b/scripts/inference-long.py@@ -1,4 +1,6 @@+import jsonimport os+import reimport colossalaiimport torch@@ -80,6 +82,20 @@ def process_prompts(prompts, num_loop):return ret_prompts+def extract_json_from_prompts(prompts):+ additional_infos = []+ ret_prompts = []+ for prompt in prompts:+ parts = re.split(r"(?=[{\[])", prompt)+ assert len(parts) <= 2, f"Invalid prompt: {prompt}"+ ret_prompts.append(parts[0])+ if len(parts) == 1:+ additional_infos.append({})+ else:+ additional_infos.append(json.loads(parts[1]))+ return ret_prompts, additional_infos++def main():# ======================================================# 1. cfg and init distributed env@@ -186,7 +202,9 @@ def main():# 4.1. batch generationfor i in range(0, len(prompts), cfg.batch_size):- batch_prompts_loops = process_prompts(prompts[i : i + cfg.batch_size], cfg.loop)+ batch_prompts_raw = prompts[i : i + cfg.batch_size]+ batch_prompts_raw, additional_infos = extract_json_from_prompts(batch_prompts_raw)+ batch_prompts_loops = process_prompts(batch_prompts_raw, cfg.loop)video_clips = []# 4.2. load reference videos & images@@ -238,9 +256,9 @@ def main():video_clips[i][idx][:, cfg.condition_frame_length :] for i in range(1, cfg.loop)]video = torch.cat(video_clips_i, dim=1)- print(f"Prompt: {prompts[i + idx]}")+ print(f"Prompt: {batch_prompts_raw[idx]}")if cfg.prompt_as_path:- sample_name_suffix = batch_prompts[idx]+ sample_name_suffix = batch_prompts_raw[idx]else:sample_name_suffix = f"_{sample_idx}"save_path = os.path.join(save_dir, f"{sample_name}{sample_name_suffix}")diff --git a/scripts/inference.py b/scripts/inference.pyindex 733118f..ccf1417 100644--- a/scripts/inference.py+++ b/scripts/inference.py@@ -112,8 +112,8 @@ def main():# 4.1. batch generationfor i in range(0, len(prompts), cfg.batch_size):# 4.2 sample in hidden space- batch_prompts = prompts[i : i + cfg.batch_size]- batch_prompts = [text_preprocessing(prompt) for prompt in batch_prompts]+ batch_prompts_raw = prompts[i : i + cfg.batch_size]+ batch_prompts = [text_preprocessing(prompt) for prompt in batch_prompts_raw]z = torch.randn(len(batch_prompts), vae.out_channels, *latent_size, device=device, dtype=dtype)# 4.3. diffusion sampling@@ -122,6 +122,22 @@ def main():for k in range(cfg.num_sample):sample_idx = old_sample_idx+ # Skip if the sample already exists+ # This is useful for resuming sampling VBench+ if cfg.prompt_as_path:+ skip = True+ for batch_prompt in batch_prompts_raw:+ path = os.path.join(save_dir, f"{sample_name}{batch_prompt}")+ if cfg.num_sample != 1:+ path = f"{path}-{k}"+ path = f"{path}.mp4"+ if not os.path.exists(path):+ skip = False+ break+ if skip:+ continue++ # samplingsamples = scheduler.sample(model,text_encoder,@@ -132,11 +148,12 @@ def main():)samples = vae.decode(samples.to(dtype))+ # 4.4. save samplesif not use_dist or coordinator.is_master():for idx, sample in enumerate(samples):- print(f"Prompt: {batch_prompts[idx]}")+ print(f"Prompt: {batch_prompts_raw[idx]}")if cfg.prompt_as_path:- sample_name_suffix = batch_prompts[idx]+ sample_name_suffix = batch_prompts_raw[idx]else:sample_name_suffix = f"_{sample_idx}"save_path = os.path.join(save_dir, f"{sample_name}{sample_name_suffix}")