Open-Sora: Democratizing Efficient Video Production for All
Python
Dev/zangwei (#10)
Zheng Zangwei (Alex Zheng)a289bced0938
diff --git a/tools/aesthetic/inference.py b/tools/aesthetic/inference.pyindex e3b588e..76aec8a 100644--- a/tools/aesthetic/inference.py+++ b/tools/aesthetic/inference.py@@ -100,6 +100,7 @@ class AestheticScorer(nn.Module):return self.mlp(image_features)[email protected]_mode()def main(args):output_file = args.input.replace(".csv", "_aes.csv")diff --git a/tools/caption/README.md b/tools/caption/README.mdindex 3c4c056..ec35acb 100644--- a/tools/caption/README.md+++ b/tools/caption/README.md@@ -4,20 +4,11 @@ Human labeling of videos is expensive and time-consuming. We adopt powerful imag-We extract three frames from the video for captioning. With batch inference, we can achieve 10 times speedup, with 2.4 videos/s on 8 GPUs.--## GPT-4V Captioning--Run the following command to generate captions for videos with GPT-4V:--```bash-python -m tools.caption.caption_gpt4 FOLDER_WITH_VIDEOS output.csv --key $OPENAI_API_KEY-```--The cost is approximately $0.01 per video (3 frames per video). The output is a CSV file with path and caption.## LLaVA Captioning+We extract three frames from the video for captioning. With batch inference, we can achieve 10 times speedup, with 2.4 videos/s on 8 GPUs.+### Requirement```bash@@ -25,27 +16,44 @@ The cost is approximately $0.01 per video (3 frames per video). The output is aconda create -n llava python=3.10 -yconda activate llava-# install llava+# install torch+pip install torch torchvision++# clone llavagit clone https://github.com/haotian-liu/LLaVA.gitcd LLaVA+# CAUTION: This line is to remove torch dependency in pyproject.toml, which is:+# "torch==2.1.2", "torchvision==0.16.2",+# It is better manually remove it in your local pyproject.toml+sed -i '16d' pyproject.toml++# install llavapip install --upgrade pip # enable PEP 660 supportpip install -e .-# 如果你在英博机器上,需要重新安装torch-pip uninstall torch torchvision torchaudio-conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia-# install flash attentionpip install flash-attn --no-build-isolation+# install colossalai+pip install colossalai```-First, install LLaVA according to their [official instructions](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#install). We use the `liuhaotian/llava-v1.6-34b` model for captioning, which can be download [here](https://huggingface.co/liuhaotian/llava-v1.6-vicuna-7b).+Since only the 34B model's performance is comparable to GPT-4V, we only provide the usage of the 34B model. The 34B model is available [here](https://huggingface.co/liuhaotian/llava-v1.6-vicuna-7b), or run our script and it will be downloaded automatically.### Usage-Then, run the following command to generate captions for videos with LLaVA:+Prepare a csv file for processing. The csv file can be generated by `convert_dataset.py` according to its [documentation](/tools/datasets/README.md). Then, run the following command to generate captions for videos/images with LLaVA:```bash# we run this on 8xH800 GPUstorchrun --nproc_per_node 8 --standalone -m tools.caption.caption_llava samples output.csv --tp-size 2 --dp-size 4 --bs 16```++## GPT-4V Captioning++Run the following command to generate captions for videos with GPT-4V:++```bash+python -m tools.caption.caption_gpt4 FOLDER_WITH_VIDEOS output.csv --key $OPENAI_API_KEY+```++The cost is approximately $0.01 per video (3 frames per video). The output is a CSV file with path and caption.diff --git a/tools/caption/caption_llava.py b/tools/caption/caption_llava.pyindex 1c5df73..b6333c7 100644--- a/tools/caption/caption_llava.py+++ b/tools/caption/caption_llava.py@@ -7,6 +7,7 @@ import timeimport warningsimport colossalai+import pandas as pdimport torchimport torch.distributed as distfrom colossalai.cluster import DistCoordinator, ProcessGroupMesh@@ -21,7 +22,7 @@ from llava.utils import disable_torch_initfrom tqdm import tqdmfrom .acceleration.llava.policy import LlavaForCausalLMPolicy-from .utils import Timer, extract_frames, prompts, read_video_list+from .utils import Timer, extract_frames, promptsdisable_torch_init()@@ -269,11 +270,9 @@ def main(args):# ======================================================# 2. read video list# ======================================================- videos = read_video_list(args.video_folder, args.output_file)-- if len(videos) == 0:- print("No videos are found or all videos have been processed.")- return+ output_file = args.input.replace(".csv", "_caption.csv")+ data_info = pd.read_csv(args.input)+ videos = data_info["path"].tolist()# shard by DPdp_size = dist.get_world_size(dp_group)@@ -288,13 +287,13 @@ def main(args):if has_main_writer:# we keep track of the processed videos in main file# so we use append mode- main_file = open(args.output_file, "a")+ main_file = open(output_file, "a")main_writer = csv.writer(main_file)if has_dp_writter:# the dp writer takes care of the files processed on the current dp rank# so we use write mode- output_file_split = f"{args.output_file}.part{dp_rank}"+ output_file_split = f"{output_file}.part{dp_rank}"dp_file = open(output_file_split, "w")dp_writer = csv.writer(dp_file)@@ -458,7 +457,7 @@ def main(args):# merge filesif has_main_writer:for i in range(dp_size):- output_file_split = f"{args.output_file}.part{i}"+ output_file_split = f"{output_file}.part{i}"with open(output_file_split, "r") as f:reader = csv.reader(f)for row in reader:@@ -472,8 +471,8 @@ def main(args):if __name__ == "__main__":parser = argparse.ArgumentParser()- parser.add_argument("video_folder", type=str)- parser.add_argument("output_file", type=str)+ parser.add_argument("input", type=str, help="Path to the input CSV file")+parser.add_argument("--bs", type=int, default=32)parser.add_argument("--prompt", type=str, default="three_frames")parser.add_argument("--tp-size", type=int, default=1)