Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

format

Shen-Chenhui1573dbbc0175
diff --git a/eval/sample.sh b/eval/sample.sh
index 1c52506..eff1368 100644
--- a/eval/sample.sh
+++ b/eval/sample.sh
@@ -260,10 +260,17 @@ function run_vbench() {
--resolution $VBENCH_RES --aspect-ratio $VBENCH_ASP_RATIO --num-sampling-steps ${NUM_SAMPLING_STEPS} --flow ${FLOW} \
--batch-size $VBENCH_BS --num-frames $NUM_FRAMES --start-index $1 --end-index $2
else
- eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
- --prompt-path assets/texts/VBench/all_dimension.txt \
- --resolution $VBENCH_RES --aspect-ratio $VBENCH_ASP_RATIO --num-sampling-steps ${NUM_SAMPLING_STEPS} --flow ${FLOW} --llm-refine ${LLM_REFINE} \
- --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --start-index $1 --end-index $2
+ if [ "${FLOW}" = "None" ]; then
+ eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
+ --prompt-path assets/texts/VBench/all_dimension.txt \
+ --resolution $VBENCH_RES --aspect-ratio $VBENCH_ASP_RATIO --num-sampling-steps ${NUM_SAMPLING_STEPS} --llm-refine ${LLM_REFINE} \
+ --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --start-index $1 --end-index $2
+ else
+ eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \
+ --prompt-path assets/texts/VBench/all_dimension.txt \
+ --resolution $VBENCH_RES --aspect-ratio $VBENCH_ASP_RATIO --num-sampling-steps ${NUM_SAMPLING_STEPS} --flow ${FLOW} --llm-refine ${LLM_REFINE} \
+ --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --start-index $1 --end-index $2
+ fi
fi
fi
fi
diff --git a/eval/vbench/calc_vbench.py b/eval/vbench/calc_vbench.py
index b2505bf..e5570a1 100644
--- a/eval/vbench/calc_vbench.py
+++ b/eval/vbench/calc_vbench.py
@@ -1,37 +1,46 @@
import argparse
import os
-from vbench import VBench
-import torch
import time
+import torch
+
+from vbench import VBench
+
full_info_path = "eval/vbench/VBench_full_info.json"
dimensions = [
- # Quality Score
- "subject_consistency",
- "background_consistency",
- "motion_smoothness",
- "dynamic_degree",
- "aesthetic_quality",
- "imaging_quality",
- "temporal_flickering",
- # Semantic Score
- "object_class",
- "multiple_objects",
- "color",
- "spatial_relationship",
- "scene",
- "temporal_style",
- "overall_consistency",
- "human_action",
- "appearance_style",
+ # a: 10min
+ "subject_consistency", # 4min
+ "imaging_quality", # 6min
+ # b: 12min
+ "background_consistency", # 2min
+ "motion_smoothness", # 5min
+ "overall_consistency", # 2min
+ "human_action", # 3min
+ # c: 14min
+ "multiple_objects", # 14min
+ # d: 14min
+ "spatial_relationship", # 14min
+ # e: 12min
+ "object_class", # 12min
+ # f: 12min
+ "color", # 12min
+ # g: 10.5min
+ "aesthetic_quality", # 2.5min
+ "appearance_style", # 6min
+ "temporal_flickering", # 2min
+ # h: 9min
+ "scene", # 3min
+ "temporal_style", # 2min
+ "dynamic_degree", # 4min
]
+
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("video_folder", type=str) # samples/samples..._vbench/eval
parser.add_argument("model_ckpt", type=str)
- parser.add_argument("--start", type=int, default=0) # start index of dimension to be evaluated
- parser.add_argument("--end", type=int, default=-1) # start index of dimension to be evaluated
+ parser.add_argument("--start", type=int, default=0) # start index of dimension to be evaluated
+ parser.add_argument("--end", type=int, default=-1) # start index of dimension to be evaluated
args = parser.parse_args()
return args
@@ -44,23 +53,23 @@ if __name__ == "__main__":
video_path = args.video_folder
kwargs = {}
- kwargs['imaging_quality_preprocessing_mode'] = 'longer' # use VBench/evaluate.py default
+ kwargs["imaging_quality_preprocessing_mode"] = "longer" # use VBench/evaluate.py default
start_time = time.time()
# NOTE: important to use torch.device("cuda"), else will have issue with object_class third_party module
my_VBench = VBench(torch.device("cuda"), full_info_path, output_dir)
- if args.end == -1: # adjust end accordingly
+ if args.end == -1: # adjust end accordingly
args.end = len(dimensions)
- for dim in dimensions[args.start:args.end]:
+ for dim in dimensions[args.start : args.end]:
my_VBench.evaluate(
videos_path=video_path,
name=dim,
local=False,
read_frame=False,
dimension_list=[dim],
- mode='vbench_standard',
- **kwargs
+ mode="vbench_standard",
+ **kwargs,
)
- print("Runtime: %s seconds " % (time.time() - start_time))
\ No newline at end of file
+ print("Runtime: %s seconds " % (time.time() - start_time))
diff --git a/eval/vbench/launch_calc.sh b/eval/vbench/launch_calc.sh
index 53114b9..9f14ce5 100644
--- a/eval/vbench/launch_calc.sh
+++ b/eval/vbench/launch_calc.sh
@@ -7,11 +7,10 @@ mkdir -p $LOG_BASE
echo "Logging to $LOG_BASE"
GPUS=(0 1 2 3 4 5 6 7)
-START_INDEX_LIST=(0 2 4 6 8 10 12 14)
-END_INDEX_LIST=(2 4 6 8 10 12 14 16)
+START_INDEX_LIST=(0 2 6 7 8 9 10 13)
+END_INDEX_LIST=(2 6 7 8 9 10 13 16)
TASK_ID_LIST=(calc_vbench_a calc_vbench_b calc_vbench_c calc_vbench_d calc_vbench_e calc_vbench_f calc_vbench_g calc_vbench_h) # for log records only
-
for i in "${!GPUS[@]}"; do
CUDA_VISIBLE_DEVICES=${GPUS[i]} python eval/vbench/calc_vbench.py $VIDEO_DIR $CKPT_DIR --start ${START_INDEX_LIST[i]} --end ${END_INDEX_LIST[i]} > ${LOG_BASE}/${TASK_ID_LIST[i]}.log 2>&1 &
done