Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

format

Shen-Chenhui01fee497f3bc
diff --git a/.gitignore b/.gitignore
index ebf9f97..b4237e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -196,3 +196,7 @@ package.json
# PLLaVA
tools/caption/pllava_dir/PLLaVA/
+
+# vbench
+vbench
+vbench2_beta_i2v
diff --git a/eval/README.md b/eval/README.md
index d30d594..15852bf 100644
--- a/eval/README.md
+++ b/eval/README.md
@@ -79,6 +79,8 @@ bash eval/vbench_i2v/launch.sh /path/to/ckpt num_frames model_name
python eval/vbench_i2v/vbench_i2v.py
# Step 3: obtain the scaled scores
+def load_i2v_dimension_info(json_dir, dimension, lang, resolution):
+# Note that if you need to go to `your_conda_env_path/lib/python3.x/site-packages/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your appropriate image folder.
python eval/vbench_i2v/tabulate_vbench_i2v_scores.py --score_dir path/to/evaluation_results/dir
```
diff --git a/eval/human_eval/launch.sh b/eval/human_eval/launch.sh
index 79095f1..c980267 100644
--- a/eval/human_eval/launch.sh
+++ b/eval/human_eval/launch.sh
@@ -16,6 +16,7 @@ fi
# LOG_BASE=logs/samples/${MODEL_NAME}_${CKPT_BASE}
# mkdir -p logs/samples
LOG_BASE=$(dirname $CKPT)/eval
+mkdir -p ${LOG_BASE}
echo "Logging to $LOG_BASE"
GPUS=(0 1 2 3 4 5 6 7)
diff --git a/eval/loss/launch.sh b/eval/loss/launch.sh
index bf28b02..4eb5435 100644
--- a/eval/loss/launch.sh
+++ b/eval/loss/launch.sh
@@ -17,7 +17,7 @@ else
fi
# LOG_BASE=logs/loss/${MODEL_NAME}_${CKPT_BASE}
# mkdir -p logs/loss
-LOG_BASE=$(dirname $CKPT)/eval
+LOG_BASE=$(dirname $CKPT_PATH)/eval
echo "Logging to $LOG_BASE"
diff --git a/eval/vbench/launch.sh b/eval/vbench/launch.sh
index 436a937..c295fb5 100644
--- a/eval/vbench/launch.sh
+++ b/eval/vbench/launch.sh
@@ -4,8 +4,9 @@ set -x
set -e
CKPT=$1
-MODEL_NAME=$2
-NUM_FRAMES=$3
+NUM_FRAMES=$2
+MODEL_NAME=$3
+
if [[ $CKPT == *"ema"* ]]; then
parentdir=$(dirname $CKPT)
@@ -13,12 +14,12 @@ if [[ $CKPT == *"ema"* ]]; then
else
CKPT_BASE=$(basename $CKPT)
fi
-LOG_BASE=logs/sample/${MODEL_NAME}_${CKPT_BASE}
+LOG_BASE=$(dirname $CKPT)/eval
echo "Logging to $LOG_BASE"
GPUS=(0 1 2 3 4 5 6 7)
TASK_ID_LIST=(4a 4b 4c 4d 4e 4f 4g 4h)
for i in "${!GPUS[@]}"; do
- CUDA_VISIBLE_DEVICES=${GPUS[i]} bash eval/sample.sh $CKPT ${NUM_FRAMES} ${MODEL_NAME} -${TASK_ID_LIST[i]} >${LOG_BASE}_${TASK_ID_LIST[i]}.log 2>&1 &
+ CUDA_VISIBLE_DEVICES=${GPUS[i]} bash eval/sample.sh $CKPT ${NUM_FRAMES} ${MODEL_NAME} -${TASK_ID_LIST[i]} >${LOG_BASE}/${TASK_ID_LIST[i]}.log 2>&1 &
done
diff --git a/eval/vbench_i2v/vbench_i2v.py b/eval/vbench_i2v/vbench_i2v.py
index c5ab048..ac82ab9 100644
--- a/eval/vbench_i2v/vbench_i2v.py
+++ b/eval/vbench_i2v/vbench_i2v.py
@@ -1,10 +1,9 @@
import argparse
import os
-from vbench import VBench
from vbench2_beta_i2v import VBenchI2V
-FULL_INFO_PATH = "vbench2_beta_i2v/vbench2_i2v_full_info.json"
+FULL_INFO_PATH = "eval/vbench_i2v/vbench2_i2v_full_info.json"
VIDEO_QUALITY_DIMENSIONS = [
"subject_consistency",
"background_consistency",
@@ -31,12 +30,12 @@ if __name__ == "__main__":
os.makedirs(output_dir, exist_ok=True)
video_path = args.video_folder
- my_VBench_I2V = VBenchI2V("cuda", FULL_INFO_PATH, "evaluation_results")
+ my_VBench_I2V = VBenchI2V("cuda", FULL_INFO_PATH, output_dir)
my_VBench_I2V.evaluate(videos_path=video_path, name="vbench_i2v", dimension_list=I2V_DIMENSIONS, resolution="1-1")
- my_VBench = VBench("cuda", FULL_INFO_PATH, output_dir)
- my_VBench.evaluate(
- videos_path=video_path,
- name="vbench_video_quality",
- dimension_list=VIDEO_QUALITY_DIMENSIONS,
- )
+ # my_VBench = VBench("cuda", FULL_INFO_PATH, output_dir)
+ # my_VBench.evaluate(
+ # videos_path=video_path,
+ # name="vbench_video_quality",
+ # dimension_list=VIDEO_QUALITY_DIMENSIONS,
+ # )