Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

Dev/zangwei (#10)

Zheng Zangwei (Alex Zheng)a289bced0938
diff --git a/tools/aesthetic/inference.py b/tools/aesthetic/inference.py
index e3b588e..76aec8a 100644
--- a/tools/aesthetic/inference.py
+++ b/tools/aesthetic/inference.py
@@ -100,6 +100,7 @@ class AestheticScorer(nn.Module):
return self.mlp(image_features)
def main(args):
output_file = args.input.replace(".csv", "_aes.csv")
diff --git a/tools/caption/README.md b/tools/caption/README.md
index 3c4c056..ec35acb 100644
--- a/tools/caption/README.md
+++ b/tools/caption/README.md
@@ -4,20 +4,11 @@ Human labeling of videos is expensive and time-consuming. We adopt powerful imag
![Caption](https://i0.imgs.ovh/2024/03/16/eXdvC.png)
-We extract three frames from the video for captioning. With batch inference, we can achieve 10 times speedup, with 2.4 videos/s on 8 GPUs.
-
-## GPT-4V Captioning
-
-Run the following command to generate captions for videos with GPT-4V:
-
-```bash
-python -m tools.caption.caption_gpt4 FOLDER_WITH_VIDEOS output.csv --key $OPENAI_API_KEY
-```
-
-The cost is approximately $0.01 per video (3 frames per video). The output is a CSV file with path and caption.
## LLaVA Captioning
+We extract three frames from the video for captioning. With batch inference, we can achieve 10 times speedup, with 2.4 videos/s on 8 GPUs.
+
### Requirement
```bash
@@ -25,27 +16,44 @@ The cost is approximately $0.01 per video (3 frames per video). The output is a
conda create -n llava python=3.10 -y
conda activate llava
-# install llava
+# install torch
+pip install torch torchvision
+
+# clone llava
git clone https://github.com/haotian-liu/LLaVA.git
cd LLaVA
+# CAUTION: This line is to remove torch dependency in pyproject.toml, which is:
+# "torch==2.1.2", "torchvision==0.16.2",
+# It is better manually remove it in your local pyproject.toml
+sed -i '16d' pyproject.toml
+
+# install llava
pip install --upgrade pip # enable PEP 660 support
pip install -e .
-# 如果你在英博机器上,需要重新安装torch
-pip uninstall torch torchvision torchaudio
-conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia
-
# install flash attention
pip install flash-attn --no-build-isolation
+# install colossalai
+pip install colossalai
```
-First, install LLaVA according to their [official instructions](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#install). We use the `liuhaotian/llava-v1.6-34b` model for captioning, which can be download [here](https://huggingface.co/liuhaotian/llava-v1.6-vicuna-7b).
+Since only the 34B model's performance is comparable to GPT-4V, we only provide the usage of the 34B model. The 34B model is available [here](https://huggingface.co/liuhaotian/llava-v1.6-vicuna-7b), or run our script and it will be downloaded automatically.
### Usage
-Then, run the following command to generate captions for videos with LLaVA:
+Prepare a csv file for processing. The csv file can be generated by `convert_dataset.py` according to its [documentation](/tools/datasets/README.md). Then, run the following command to generate captions for videos/images with LLaVA:
```bash
# we run this on 8xH800 GPUs
torchrun --nproc_per_node 8 --standalone -m tools.caption.caption_llava samples output.csv --tp-size 2 --dp-size 4 --bs 16
```
+
+## GPT-4V Captioning
+
+Run the following command to generate captions for videos with GPT-4V:
+
+```bash
+python -m tools.caption.caption_gpt4 FOLDER_WITH_VIDEOS output.csv --key $OPENAI_API_KEY
+```
+
+The cost is approximately $0.01 per video (3 frames per video). The output is a CSV file with path and caption.
diff --git a/tools/caption/caption_llava.py b/tools/caption/caption_llava.py
index 1c5df73..b6333c7 100644
--- a/tools/caption/caption_llava.py
+++ b/tools/caption/caption_llava.py
@@ -7,6 +7,7 @@ import time
import warnings
import colossalai
+import pandas as pd
import torch
import torch.distributed as dist
from colossalai.cluster import DistCoordinator, ProcessGroupMesh
@@ -21,7 +22,7 @@ from llava.utils import disable_torch_init
from tqdm import tqdm
from .acceleration.llava.policy import LlavaForCausalLMPolicy
-from .utils import Timer, extract_frames, prompts, read_video_list
+from .utils import Timer, extract_frames, prompts
disable_torch_init()
@@ -269,11 +270,9 @@ def main(args):
# ======================================================
# 2. read video list
# ======================================================
- videos = read_video_list(args.video_folder, args.output_file)
-
- if len(videos) == 0:
- print("No videos are found or all videos have been processed.")
- return
+ output_file = args.input.replace(".csv", "_caption.csv")
+ data_info = pd.read_csv(args.input)
+ videos = data_info["path"].tolist()
# shard by DP
dp_size = dist.get_world_size(dp_group)
@@ -288,13 +287,13 @@ def main(args):
if has_main_writer:
# we keep track of the processed videos in main file
# so we use append mode
- main_file = open(args.output_file, "a")
+ main_file = open(output_file, "a")
main_writer = csv.writer(main_file)
if has_dp_writter:
# the dp writer takes care of the files processed on the current dp rank
# so we use write mode
- output_file_split = f"{args.output_file}.part{dp_rank}"
+ output_file_split = f"{output_file}.part{dp_rank}"
dp_file = open(output_file_split, "w")
dp_writer = csv.writer(dp_file)
@@ -458,7 +457,7 @@ def main(args):
# merge files
if has_main_writer:
for i in range(dp_size):
- output_file_split = f"{args.output_file}.part{i}"
+ output_file_split = f"{output_file}.part{i}"
with open(output_file_split, "r") as f:
reader = csv.reader(f)
for row in reader:
@@ -472,8 +471,8 @@ def main(args):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
- parser.add_argument("video_folder", type=str)
- parser.add_argument("output_file", type=str)
+ parser.add_argument("input", type=str, help="Path to the input CSV file")
+
parser.add_argument("--bs", type=int, default=32)
parser.add_argument("--prompt", type=str, default="three_frames")
parser.add_argument("--tp-size", type=int, default=1)