Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[wip] prompt with json

zhengzangw2b42ca82d800
diff --git a/configs/opensora-v1-1/inference/sample-ref-inprompt.py b/configs/opensora-v1-1/inference/sample-ref-inprompt.py
new file mode 100644
index 0000000..3b75eb4
--- /dev/null
+++ b/configs/opensora-v1-1/inference/sample-ref-inprompt.py
@@ -0,0 +1,47 @@
+num_frames = 16
+frame_interval = 3
+fps = 24
+image_size = (240, 426)
+multi_resolution = "STDiT2"
+
+# Condition
+loop = 1
+condition_frame_length = 4
+prompt = [
+ # "A car driving on the ocean.",
+ 'Drone view of waves crashing against the rugged cliffs along Big Sur\'s garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff\'s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff\'s edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.{"reference_path": "assets/images/condition/cliff.png", "mask_strategy": "0,0,0,1,0"}',
+ # "In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, seizing the moment, skillfully navigate the face of the wave.",
+]
+
+# Define model
+model = dict(
+ type="STDiT2-XL/2",
+ from_pretrained=None,
+ input_sq_size=512,
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+)
+vae = dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="stabilityai/sd-vae-ft-ema",
+ cache_dir=None, # "/mnt/hdd/cached_models",
+ micro_batch_size=4,
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ cache_dir=None, # "/mnt/hdd/cached_models",
+ model_max_length=200,
+)
+scheduler = dict(
+ type="iddpm",
+ num_sampling_steps=100,
+ cfg_scale=7.0,
+ cfg_channel=3, # or None
+)
+dtype = "bf16"
+
+# Others
+batch_size = 1
+seed = 42
+save_dir = "./samples/samples/"
diff --git a/configs/opensora-v1-1/inference/sample-ref.py b/configs/opensora-v1-1/inference/sample-ref.py
index 6e8e805..2712a6e 100644
--- a/configs/opensora-v1-1/inference/sample-ref.py
+++ b/configs/opensora-v1-1/inference/sample-ref.py
@@ -7,26 +7,23 @@ multi_resolution = "STDiT2"
# Condition
prompt_path = None
prompt = [
- "A car driving on a road in the middle of a desert.",
- # "A man smiling",
- # "Drone view of waves crashing against the rugged cliffs along Big Sur's garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff’s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.",
+ # "A car driving on the ocean.",
+ "Drone view of waves crashing against the rugged cliffs along Big Sur's garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff’s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.",
# "In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, seizing the moment, skillfully navigate the face of the wave.",
]
loop = 1
condition_frame_length = 4
reference_path = [
- "https://cdn.openai.com/tmp/s/interp/d0.mp4",
- # "https://www.comp.nus.edu.sg/~youy/index_files/yangyou3.png",
- # "assets/images/condition/cliff.png",
+ # "https://cdn.openai.com/tmp/s/interp/d0.mp4",
+ "assets/images/condition/cliff.png",
# "assets/images/condition/wave.png",
]
# valid when reference_path is not None
# (loop id, ref id, ref start, length, target start)
mask_strategy = [
- "0,0,0,8,0,0.5",
- # "0,0,0,1,0",
- # "0,0,0,1,0",
+ # "0,0,0,8,0,0.3",
+ "0,0,0,1,0",
# "0,0,0,1,0",
]
diff --git a/eval/launch_sample.sh b/eval/launch_sample.sh
new file mode 100644
index 0000000..346312c
--- /dev/null
+++ b/eval/launch_sample.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -x
+set -e
+
+CKPT=$1
+if [[ $CKPT == *"ema"* ]]; then
+ parentdir=$(dirname $CKPT)
+ CKPT_BASE=$(basename $parentdir)_ema
+else
+ CKPT_BASE=$(basename $CKPT)
+fi
+LOG_BASE=logs/sample/CKPT_BASE
+echo "Logging to $LOG_BASE"
+
+CUDA_VISIBLE_DEVICES=0 bash eval/sample.sh $CKPT -1 >${LOG_BASE}_1.log 2>&1 &
+CUDA_VISIBLE_DEVICES=1 bash eval/sample.sh $CKPT -3 >${LOG_BASE}_3.log 2>&1 &
+CUDA_VISIBLE_DEVICES=2 bash eval/sample.sh $CKPT -2a >${LOG_BASE}_2a.log 2>&1 &
+CUDA_VISIBLE_DEVICES=3 bash eval/sample.sh $CKPT -2b >${LOG_BASE}_2b.log 2>&1 &
+CUDA_VISIBLE_DEVICES=4 bash eval/sample.sh $CKPT -2c >${LOG_BASE}_2c.log 2>&1 &
+CUDA_VISIBLE_DEVICES=5 bash eval/sample.sh $CKPT -2d >${LOG_BASE}_2d.log 2>&1 &
+CUDA_VISIBLE_DEVICES=6 bash eval/sample.sh $CKPT -2e >${LOG_BASE}_2e.log 2>&1 &
+CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT -2f >${LOG_BASE}_2f.log 2>&1 &
+
+# kill all by: pkill -f "inference"
diff --git a/eval/sample.sh b/eval/sample.sh
index 37320b0..675bf87 100644
--- a/eval/sample.sh
+++ b/eval/sample.sh
@@ -51,6 +51,8 @@ function run_image() { # 10min
function run_video_a() { # 20min
# 2.1.1 16x240x426
+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 16 --image-size 144 256 --sample-name sample_16x144x256
+ eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 16 --image-size 256 256 --sample-name sample_16x256x256
eval $CMD --ckpt-path $CKPT --prompt-path assets/texts/t2v_samples.txt --save-dir $OUTPUT --num-frames 16 --image-size 240 426 --sample-name sample_16x240x426
# 2.1.2 16x720p multi-resolution
@@ -129,49 +131,49 @@ function run_video_edit() { # 23min
# vbench has 950 samples
-function run_vbenck_a() {
+function run_vbenck_a() { # 2h
eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
--prompt-path assets/texts/VBench/all_dimension.txt \
--start-index 0 --end-index 120
}
-function run_vbenck_b() {
+function run_vbenck_b() { # 2h
eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
--prompt-path assets/texts/VBench/all_dimension.txt \
--start-index 120 --end-index 240
}
-function run_vbenck_c() {
+function run_vbenck_c() { # 2h
eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
--prompt-path assets/texts/VBench/all_dimension.txt \
--start-index 240 --end-index 360
}
-function run_vbenck_d() {
+function run_vbenck_d() { # 2h
eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
--prompt-path assets/texts/VBench/all_dimension.txt \
--start-index 360 --end-index 480
}
-function run_vbenck_e() {
+function run_vbenck_e() { # 2h
eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
--prompt-path assets/texts/VBench/all_dimension.txt \
--start-index 480 --end-index 600
}
-function run_vbenck_f() {
+function run_vbenck_f() { # 2h
eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
--prompt-path assets/texts/VBench/all_dimension.txt \
--start-index 600 --end-index 720
}
-function run_vbenck_g() {
+function run_vbenck_g() { # 2h
eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
--prompt-path assets/texts/VBench/all_dimension.txt \
--start-index 720 --end-index 840
}
-function run_vbenck_h() {
+function run_vbenck_h() { # 2h
eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
--prompt-path assets/texts/VBench/all_dimension.txt \
--start-index 840
@@ -260,4 +262,5 @@ done
end=$(date +%s)
runtime=$((end - start))
+
echo "Runtime: $runtime seconds"
diff --git a/eval/vbench.sh b/eval/vbench.sh
index 864cd9b..7065765 100644
--- a/eval/vbench.sh
+++ b/eval/vbench.sh
@@ -1,11 +1,32 @@
#!/bin/bash
+set -e
+
# Base path for videos
-videos_path='./samples/samples_stage1_41k_2_17k_vbench'
-json_path='./eval/VBench_full_info.json'
+videos_path=$1
+videos_base=$(basename $videos_path)
+json_path=./eval/VBench_full_info.json
+output_path=./evaluation_results/$videos_base
# Define the dimension list
-dimensions=("subject_consistency" "background_consistency" "aesthetic_quality" "imaging_quality" "object_class" "multiple_objects" "color" "spatial_relationship" "scene" "temporal_style" "overall_consistency" "human_action" "temporal_flickering" "motion_smoothness" "dynamic_degree" "appearance_style")
+dimensions=(
+ "subject_consistency"
+ "background_consistency"
+ "aesthetic_quality"
+ "imaging_quality"
+ "object_class"
+ "multiple_objects"
+ "color"
+ "spatial_relationship"
+ "scene"
+ "temporal_style"
+ "overall_consistency"
+ "human_action"
+ "temporal_flickering"
+ "motion_smoothness"
+ "dynamic_degree"
+ "appearance_style"
+)
# Loop over each dimension
for i in "${!dimensions[@]}"; do
@@ -16,5 +37,5 @@ for i in "${!dimensions[@]}"; do
echo "$dimension $videos_path"
# Run the evaluation script
- vbench evaluate --videos_path $videos_path --dimension $dimension --full_json_dir $json_path
+ vbench evaluate --videos_path $videos_path --dimension $dimension --full_json_dir $json_path --output_path $output_path
done
diff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.py
index 78ee638..b5edb93 100644
--- a/opensora/utils/config_utils.py
+++ b/opensora/utils/config_utils.py
@@ -71,6 +71,12 @@ def merge_args(cfg, args, training=False):
if args.ckpt_path is not None:
cfg.model["from_pretrained"] = args.ckpt_path
args.ckpt_path = None
+ if args.cfg_scale is not None:
+ cfg.scheduler["cfg_scale"] = args.cfg_scale
+ args.cfg_scale = None
+ if args.num_sampling_steps is not None:
+ cfg.scheduler["num_sampling_steps"] = args.num_sampling_steps
+ args.num_sampling_steps = None
if training and args.data_path is not None:
cfg.dataset["data_path"] = args.data_path
args.data_path = None
diff --git a/scripts/inference-long.py b/scripts/inference-long.py
index 69060dc..257755b 100644
--- a/scripts/inference-long.py
+++ b/scripts/inference-long.py
@@ -1,4 +1,6 @@
+import json
import os
+import re
import colossalai
import torch
@@ -80,6 +82,20 @@ def process_prompts(prompts, num_loop):
return ret_prompts
+def extract_json_from_prompts(prompts):
+ additional_infos = []
+ ret_prompts = []
+ for prompt in prompts:
+ parts = re.split(r"(?=[{\[])", prompt)
+ assert len(parts) <= 2, f"Invalid prompt: {prompt}"
+ ret_prompts.append(parts[0])
+ if len(parts) == 1:
+ additional_infos.append({})
+ else:
+ additional_infos.append(json.loads(parts[1]))
+ return ret_prompts, additional_infos
+
+
def main():
# ======================================================
# 1. cfg and init distributed env
@@ -186,7 +202,9 @@ def main():
# 4.1. batch generation
for i in range(0, len(prompts), cfg.batch_size):
- batch_prompts_loops = process_prompts(prompts[i : i + cfg.batch_size], cfg.loop)
+ batch_prompts_raw = prompts[i : i + cfg.batch_size]
+ batch_prompts_raw, additional_infos = extract_json_from_prompts(batch_prompts_raw)
+ batch_prompts_loops = process_prompts(batch_prompts_raw, cfg.loop)
video_clips = []
# 4.2. load reference videos & images
@@ -238,9 +256,9 @@ def main():
video_clips[i][idx][:, cfg.condition_frame_length :] for i in range(1, cfg.loop)
]
video = torch.cat(video_clips_i, dim=1)
- print(f"Prompt: {prompts[i + idx]}")
+ print(f"Prompt: {batch_prompts_raw[idx]}")
if cfg.prompt_as_path:
- sample_name_suffix = batch_prompts[idx]
+ sample_name_suffix = batch_prompts_raw[idx]
else:
sample_name_suffix = f"_{sample_idx}"
save_path = os.path.join(save_dir, f"{sample_name}{sample_name_suffix}")
diff --git a/scripts/inference.py b/scripts/inference.py
index 733118f..ccf1417 100644
--- a/scripts/inference.py
+++ b/scripts/inference.py
@@ -112,8 +112,8 @@ def main():
# 4.1. batch generation
for i in range(0, len(prompts), cfg.batch_size):
# 4.2 sample in hidden space
- batch_prompts = prompts[i : i + cfg.batch_size]
- batch_prompts = [text_preprocessing(prompt) for prompt in batch_prompts]
+ batch_prompts_raw = prompts[i : i + cfg.batch_size]
+ batch_prompts = [text_preprocessing(prompt) for prompt in batch_prompts_raw]
z = torch.randn(len(batch_prompts), vae.out_channels, *latent_size, device=device, dtype=dtype)
# 4.3. diffusion sampling
@@ -122,6 +122,22 @@ def main():
for k in range(cfg.num_sample):
sample_idx = old_sample_idx
+ # Skip if the sample already exists
+ # This is useful for resuming sampling VBench
+ if cfg.prompt_as_path:
+ skip = True
+ for batch_prompt in batch_prompts_raw:
+ path = os.path.join(save_dir, f"{sample_name}{batch_prompt}")
+ if cfg.num_sample != 1:
+ path = f"{path}-{k}"
+ path = f"{path}.mp4"
+ if not os.path.exists(path):
+ skip = False
+ break
+ if skip:
+ continue
+
+ # sampling
samples = scheduler.sample(
model,
text_encoder,
@@ -132,11 +148,12 @@ def main():
)
samples = vae.decode(samples.to(dtype))
+ # 4.4. save samples
if not use_dist or coordinator.is_master():
for idx, sample in enumerate(samples):
- print(f"Prompt: {batch_prompts[idx]}")
+ print(f"Prompt: {batch_prompts_raw[idx]}")
if cfg.prompt_as_path:
- sample_name_suffix = batch_prompts[idx]
+ sample_name_suffix = batch_prompts_raw[idx]
else:
sample_name_suffix = f"_{sample_idx}"
save_path = os.path.join(save_dir, f"{sample_name}{sample_name_suffix}")