Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

faster pllava

Tom Young880d380a211d
diff --git a/tools/caption/README.md b/tools/caption/README.md
index fbaf3c9..763b04c 100644
--- a/tools/caption/README.md
+++ b/tools/caption/README.md
@@ -122,7 +122,7 @@ python python_scripts/hf.py # download the weights
``` -->
### Usage
-Since PLLaVA is not fashioned as a package, we will use PYTHONPATH to use it as a package.
+Since PLLaVA is not fashioned as a package, we will use PYTHONPATH to use it.
```bash
@@ -130,7 +130,7 @@ cd .. # step back to pllava_dir
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
PYTHONPATH='$PYTHONPATH:OPEN_SORA_HOME/tools/caption/pllava_dir/PLLaVA' \
-nohup /mnt/nfs-207/envs/pllava/bin/python caption_pllava.py \
+nohup python caption_pllava.py \
--pretrained_model_name_or_path PLLaVA/MODELS/pllava-13b \
--use_lora \
--lora_alpha 4 \
diff --git a/tools/caption/pllava_dir/caption_pllava.py b/tools/caption/pllava_dir/caption_pllava.py
index f27643f..5390ca0 100644
--- a/tools/caption/pllava_dir/caption_pllava.py
+++ b/tools/caption/pllava_dir/caption_pllava.py
@@ -1,3 +1,5 @@
+from multiprocessing import Process, Queue
+import os
import pandas as pd
import functools
import itertools
@@ -14,12 +16,12 @@ import torchvision
from decord import VideoReader, cpu
import transformers
import pandas as pd
-# import sys
-# sys.path.append('/home/tom/Open-Sora-dev/tools/caption/pllava_dir/PLLaVA')
-from tasks.eval.model_utils import load_pllava, pllava_answer
+import itertools
+from transformers.feature_extraction_utils import BatchFeature
+from tofiletimer import ToFileTimer
+from tasks.eval.model_utils import load_pllava
from tasks.eval.eval_utils import Conversation
-
conv_template = Conversation(
system="Describe this video. Pay attention to all objects in the video. The description should be useful for AI to re-generate the video. The description should be no more than six sentences. Here are some examples of good descriptions: 1. A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage. She wears a black leather jacket, a long red dress, and black boots, and carries a black purse. She wears sunglasses and red lipstick. She walks confidently and casually. The street is damp and reflective, creating a mirror effect of the colorful lights. Many pedestrians walk about. 2. Several giant wooly mammoths approach treading through a snowy meadow, their long wooly fur lightly blows in the wind as they walk, snow covered trees and dramatic snow capped mountains in the distance, mid afternoon light with wispy clouds and a sun high in the distance creates a warm glow, the low camera view is stunning capturing the large furry mammal with beautiful photography, depth of field. 3. Drone view of waves crashing against the rugged cliffs along Big Sur's garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff's edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff’s edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.",
roles=("USER:", "ASSISTANT:"),
@@ -28,13 +30,42 @@ conv_template = Conversation(
mm_token='<image>'
)
-
logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
RESOLUTION = 672 #
+def pllava_answer(conv: Conversation, model, processor, video_list, do_sample=True, max_new_tokens=200, num_beams=1, min_length=1, top_p=0.9, repetition_penalty=1.0, length_penalty=1, temperature=1.0, stop_criteria_keywords=None, print_res=False):
+ # torch.cuda.empty_cache()
+ prompt = conv.get_prompt()
+ with ToFileTimer('processing inputs', os.getenv('TIME_LOG')):
+ inputs_list = [processor(text=prompt, images=video, return_tensors="pt") for video in video_list]
+ inputs_batched = dict() # add batch dimension by cat
+ for input_type in list(inputs_list[0].keys()):
+ inputs_batched[input_type] = torch.cat([inputs[input_type] for inputs in inputs_list])
+ inputs_batched = BatchFeature(inputs_batched, tensor_type="pt").to(model.device)
+
+ with torch.no_grad():
+ with ToFileTimer('generate', os.getenv('TIME_LOG')):
+ output_texts = model.generate(**inputs_batched, media_type='video',
+ do_sample=do_sample, max_new_tokens=max_new_tokens, num_beams=num_beams, min_length=min_length,
+ top_p=top_p, repetition_penalty=repetition_penalty, length_penalty=length_penalty, temperature=temperature)
+ output_texts = processor.batch_decode(output_texts, skip_special_tokens=True, clean_up_tokenization_spaces=False)
+ for i in range(len(output_texts)):
+ if print_res: # debug usage
+ print('### PROMPTING LM WITH: ', prompt)
+ print('### LM OUTPUT TEXT: ', output_texts[i])
+ if conv.roles[-1] == "<|im_start|>assistant\n":
+ split_tag = "<|im_start|> assistant\n"
+ else:
+ split_tag = conv.roles[-1]
+ output_texts[i] = output_texts[i].split(split_tag)[-1]
+ ending = conv.sep if isinstance(conv.sep, str) else conv.sep[1]
+ output_texts[i] = output_texts[i].removesuffix(ending).strip()
+ conv.messages[-1][1] = output_texts[i]
+ return output_texts, conv
+
def get_index(num_frames, num_segments):
seg_size = float(num_frames - 1) / num_segments
start = int(seg_size / 2)
@@ -61,10 +92,14 @@ def load_video(video_path, num_frames, return_msg=False, resolution=336):
else:
return images_group
+def collate_fn(batch):
+ return batch
+
class CSVDataset(Dataset):
- def __init__(self, csv_path):
+ def __init__(self, csv_path, num_frames):
self.df = pd.read_csv(csv_path)
self.data_list = self.df.path.tolist()
+ self.num_frames = num_frames
def __len__(self):
return len(self.data_list)
@@ -72,8 +107,8 @@ class CSVDataset(Dataset):
def __getitem__(self, idx):
if idx < 0 or idx >= len(self.data_list):
raise IndexError
- # print(len(self.data_list))
- return self.data_list[idx]
+ video = load_video(self.data_list[idx], self.num_frames, resolution=RESOLUTION)
+ return video
def set_rank_and_world_size(self, rank, world_size):
self.rank = rank
@@ -91,6 +126,12 @@ def parse_args():
required=True,
default='llava-hf/llava-1.5-7b-hf'
)
+ parser.add_argument(
+ "--batch_size",
+ type=int,
+ required=False,
+ default=1,
+ )
parser.add_argument(
"--csv_path",
type=str,
@@ -148,37 +189,37 @@ def load_model_and_dataset(rank, world_size, pretrained_model_name_or_path, num_
model = model.to(torch.device(rank))
model = model.eval()
- dataset = CSVDataset(csv_path)
+ dataset = CSVDataset(csv_path, num_frames)
dataset.set_rank_and_world_size(rank, world_size)
return model, processor, dataset
def infer(\
model,
processor,
- path,
+ video_list,
conv_mode,
- num_frames,
print_res=True,
):
- video_list = load_video(path, num_frames, resolution=RESOLUTION)
conv = conv_template.copy()
conv.user_query("Describe the video in details.", is_mm=True)
- llm_response, conv = pllava_answer(
+ llm_responses, conv = pllava_answer(
conv=conv,
model=model,
processor=processor,
- img_list=video_list,
+ video_list=video_list,
max_new_tokens=256,
do_sample=False,
print_res=print_res
)
- return llm_response
+ return llm_responses
-def run(rank, args, world_size):
+def run(rank, args, world_size, output_queue):
if rank == 0:
- # import ptvsd; ptvsd.enable_attach(address=('localhost', 1025), redirect_output=True); ptvsd.wait_for_attach(); print('waiting for debugger attachment')
+ import os;
+ if os.getenv('DEBUG_ADDRESS') != None:
+ import ptvsd; ptvsd.enable_attach(address=('localhost', int(os.getenv('DEBUG_ADDRESS'))), redirect_output=True); ptvsd.wait_for_attach(); print('waiting for debugger attachment')
pass
if rank != 0:
transformers.utils.logging.set_verbosity_error()
@@ -202,29 +243,32 @@ def run(rank, args, world_size):
logger.info(f'done model and dataset...')
logger.info('constructing dataset...')
logger.info('single test...')
+ dataloader = torch.utils.data.DataLoader(
+ dataset,
+ num_workers=2,
+ batch_size=args.batch_size,
+ collate_fn=collate_fn,
+ shuffle=False,
+ )
- tbar = tqdm(total=len(dataset))
total = 0
result_list = []
- done_count = 0
print(len(dataset))
- for example in dataset:
+ for batch in tqdm(dataloader):
total += 1
try:
- pred = infer(
+ preds = infer(
model,
processor,
- example,
+ batch,
conv_mode=conv_mode,
- num_frames=args.num_frames,
print_res=print_res,
)
except Exception as e:
- logger.error(f'error in {example}: {str(e)}')
- pred = args.error_message
- result_list.append(pred)
- tbar.update(len(result_list) - done_count)
- done_count = len(result_list)
+ logger.error(f'error in {batch}: {str(e)}')
+ preds = args.error_message
+ result_list.extend(preds)
+ output_queue.put((rank, result_list))
return result_list
def main():
@@ -236,17 +280,43 @@ def main():
n_gpus = torch.cuda.device_count()
world_size = n_gpus
print(f'world_size: {world_size}')
- with Pool(world_size) as pool:
- func = functools.partial(run, args=args, world_size=world_size)
- result_lists = pool.map(func, range(world_size))
+ # Create a queue to collect results from each process
+ output_queue = Queue()
+
+ # with Pool(world_size) as pool:
+ # func = functools.partial(run, args=args, world_size=world_size)
+ # result_lists = pool.map(func, range(world_size))
+ processes = []
+ for i in range(world_size):
+ # Each process will now also take the output queue as an argument
+ p = Process(target=run, args=(i, args, world_size, output_queue))
+ p.daemon = False
+ processes.append(p)
+ p.start()
+ results_by_rank = {}
+ for _ in range(world_size):
+ rank, results = output_queue.get() # Retrieve results as they finish
+ results_by_rank[rank] = results
+ print(f'Results received from rank {rank}')
+ # ORDER THE RESULTS BY RANK
logger.info('finished running')
- result_list = [res for res in itertools.chain(*result_lists)]
+ for p in processes:
+ p.join()
+
+ results_list = list(itertools.chain.from_iterable(results_by_rank[i] for i in range(world_size)))
+ # results_list = list(itertools.chain([results_by_rank[i] for i in range(world_size)]))
+ # (data[key] for key in sorted_keys)
+ # results_list = [item for sublist in results_by_rank.values() for item in sublist]
+
else:
- result_list = run(0, world_size=1, args=args) # debug
+ results_list = run(0, world_size=1, args=args) # debug
+
+ print(results_list)
+
df = pd.read_csv(args.csv_path)
# add a new column to the dataframe
- df['text'] = result_list
+ df['text'] = results_list
drop_failed = True
if drop_failed:
# iterate through the dataframe and delete the entire row if captioning failed