Open-Sora: Democratizing Efficient Video Production for All
Python
faster pllava
Tom Young880d380a211d
diff --git a/tools/caption/README.md b/tools/caption/README.mdindex fbaf3c9..763b04c 100644--- a/tools/caption/README.md+++ b/tools/caption/README.md@@ -122,7 +122,7 @@ python python_scripts/hf.py # download the weights``` -->### Usage-Since PLLaVA is not fashioned as a package, we will use PYTHONPATH to use it as a package.+Since PLLaVA is not fashioned as a package, we will use PYTHONPATH to use it.```bash@@ -130,7 +130,7 @@ cd .. # step back to pllava_dirCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \PYTHONPATH='$PYTHONPATH:OPEN_SORA_HOME/tools/caption/pllava_dir/PLLaVA' \-nohup /mnt/nfs-207/envs/pllava/bin/python caption_pllava.py \+nohup python caption_pllava.py \--pretrained_model_name_or_path PLLaVA/MODELS/pllava-13b \--use_lora \--lora_alpha 4 \diff --git a/tools/caption/pllava_dir/caption_pllava.py b/tools/caption/pllava_dir/caption_pllava.pyindex f27643f..5390ca0 100644--- a/tools/caption/pllava_dir/caption_pllava.py+++ b/tools/caption/pllava_dir/caption_pllava.py@@ -1,3 +1,5 @@+from multiprocessing import Process, Queue+import osimport pandas as pdimport functoolsimport itertools@@ -14,12 +16,12 @@ import torchvisionfrom decord import VideoReader, cpuimport transformersimport pandas as pd-# import sys-# sys.path.append('/home/tom/Open-Sora-dev/tools/caption/pllava_dir/PLLaVA')-from tasks.eval.model_utils import load_pllava, pllava_answer+import itertools+from transformers.feature_extraction_utils import BatchFeature+from tofiletimer import ToFileTimer+from tasks.eval.model_utils import load_pllavafrom tasks.eval.eval_utils import Conversation-conv_template = Conversation(system="Describe this video. Pay attention to all objects in the video. The description should be useful for AI to re-generate the video. The description should be no more than six sentences. Here are some examples of good descriptions: 1. A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage. She wears a black leather jacket, a long red dress, and black boots, and carries a black purse. She wears sunglasses and red lipstick. She walks confidently and casually. The street is damp and reflective, creating a mirror effect of the colorful lights. Many pedestrians walk about. 2. Several giant wooly mammoths approach treading through a snowy meadow, their long wooly fur lightly blows in the wind as they walk, snow covered trees and dramatic snow capped mountains in the distance, mid afternoon light with wispy clouds and a sun high in the distance creates a warm glow, the low camera view is stunning capturing the large furry mammal with beautiful photography, depth of field. 3. Drone view of waves crashing against the rugged cliffs along Big Sur's garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff's edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff’s edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.",roles=("USER:", "ASSISTANT:"),@@ -28,13 +30,42 @@ conv_template = Conversation(mm_token='<image>')-logging.basicConfig()logger = logging.getLogger(__name__)logger.setLevel(logging.INFO)RESOLUTION = 672 #+def pllava_answer(conv: Conversation, model, processor, video_list, do_sample=True, max_new_tokens=200, num_beams=1, min_length=1, top_p=0.9, repetition_penalty=1.0, length_penalty=1, temperature=1.0, stop_criteria_keywords=None, print_res=False):+ # torch.cuda.empty_cache()+ prompt = conv.get_prompt()+ with ToFileTimer('processing inputs', os.getenv('TIME_LOG')):+ inputs_list = [processor(text=prompt, images=video, return_tensors="pt") for video in video_list]+ inputs_batched = dict() # add batch dimension by cat+ for input_type in list(inputs_list[0].keys()):+ inputs_batched[input_type] = torch.cat([inputs[input_type] for inputs in inputs_list])+ inputs_batched = BatchFeature(inputs_batched, tensor_type="pt").to(model.device)++ with torch.no_grad():+ with ToFileTimer('generate', os.getenv('TIME_LOG')):+ output_texts = model.generate(**inputs_batched, media_type='video',+ do_sample=do_sample, max_new_tokens=max_new_tokens, num_beams=num_beams, min_length=min_length,+ top_p=top_p, repetition_penalty=repetition_penalty, length_penalty=length_penalty, temperature=temperature)+ output_texts = processor.batch_decode(output_texts, skip_special_tokens=True, clean_up_tokenization_spaces=False)+ for i in range(len(output_texts)):+ if print_res: # debug usage+ print('### PROMPTING LM WITH: ', prompt)+ print('### LM OUTPUT TEXT: ', output_texts[i])+ if conv.roles[-1] == "<|im_start|>assistant\n":+ split_tag = "<|im_start|> assistant\n"+ else:+ split_tag = conv.roles[-1]+ output_texts[i] = output_texts[i].split(split_tag)[-1]+ ending = conv.sep if isinstance(conv.sep, str) else conv.sep[1]+ output_texts[i] = output_texts[i].removesuffix(ending).strip()+ conv.messages[-1][1] = output_texts[i]+ return output_texts, conv+def get_index(num_frames, num_segments):seg_size = float(num_frames - 1) / num_segmentsstart = int(seg_size / 2)@@ -61,10 +92,14 @@ def load_video(video_path, num_frames, return_msg=False, resolution=336):else:return images_group+def collate_fn(batch):+ return batch+class CSVDataset(Dataset):- def __init__(self, csv_path):+ def __init__(self, csv_path, num_frames):self.df = pd.read_csv(csv_path)self.data_list = self.df.path.tolist()+ self.num_frames = num_framesdef __len__(self):return len(self.data_list)@@ -72,8 +107,8 @@ class CSVDataset(Dataset):def __getitem__(self, idx):if idx < 0 or idx >= len(self.data_list):raise IndexError- # print(len(self.data_list))- return self.data_list[idx]+ video = load_video(self.data_list[idx], self.num_frames, resolution=RESOLUTION)+ return videodef set_rank_and_world_size(self, rank, world_size):self.rank = rank@@ -91,6 +126,12 @@ def parse_args():required=True,default='llava-hf/llava-1.5-7b-hf')+ parser.add_argument(+ "--batch_size",+ type=int,+ required=False,+ default=1,+ )parser.add_argument("--csv_path",type=str,@@ -148,37 +189,37 @@ def load_model_and_dataset(rank, world_size, pretrained_model_name_or_path, num_model = model.to(torch.device(rank))model = model.eval()- dataset = CSVDataset(csv_path)+ dataset = CSVDataset(csv_path, num_frames)dataset.set_rank_and_world_size(rank, world_size)return model, processor, datasetdef infer(\model,processor,- path,+ video_list,conv_mode,- num_frames,print_res=True,):- video_list = load_video(path, num_frames, resolution=RESOLUTION)conv = conv_template.copy()conv.user_query("Describe the video in details.", is_mm=True)- llm_response, conv = pllava_answer(+ llm_responses, conv = pllava_answer(conv=conv,model=model,processor=processor,- img_list=video_list,+ video_list=video_list,max_new_tokens=256,do_sample=False,print_res=print_res)- return llm_response+ return llm_responses-def run(rank, args, world_size):+def run(rank, args, world_size, output_queue):if rank == 0:- # import ptvsd; ptvsd.enable_attach(address=('localhost', 1025), redirect_output=True); ptvsd.wait_for_attach(); print('waiting for debugger attachment')+ import os;+ if os.getenv('DEBUG_ADDRESS') != None:+ import ptvsd; ptvsd.enable_attach(address=('localhost', int(os.getenv('DEBUG_ADDRESS'))), redirect_output=True); ptvsd.wait_for_attach(); print('waiting for debugger attachment')passif rank != 0:transformers.utils.logging.set_verbosity_error()@@ -202,29 +243,32 @@ def run(rank, args, world_size):logger.info(f'done model and dataset...')logger.info('constructing dataset...')logger.info('single test...')+ dataloader = torch.utils.data.DataLoader(+ dataset,+ num_workers=2,+ batch_size=args.batch_size,+ collate_fn=collate_fn,+ shuffle=False,+ )- tbar = tqdm(total=len(dataset))total = 0result_list = []- done_count = 0print(len(dataset))- for example in dataset:+ for batch in tqdm(dataloader):total += 1try:- pred = infer(+ preds = infer(model,processor,- example,+ batch,conv_mode=conv_mode,- num_frames=args.num_frames,print_res=print_res,)except Exception as e:- logger.error(f'error in {example}: {str(e)}')- pred = args.error_message- result_list.append(pred)- tbar.update(len(result_list) - done_count)- done_count = len(result_list)+ logger.error(f'error in {batch}: {str(e)}')+ preds = args.error_message+ result_list.extend(preds)+ output_queue.put((rank, result_list))return result_listdef main():@@ -236,17 +280,43 @@ def main():n_gpus = torch.cuda.device_count()world_size = n_gpusprint(f'world_size: {world_size}')- with Pool(world_size) as pool:- func = functools.partial(run, args=args, world_size=world_size)- result_lists = pool.map(func, range(world_size))+ # Create a queue to collect results from each process+ output_queue = Queue()++ # with Pool(world_size) as pool:+ # func = functools.partial(run, args=args, world_size=world_size)+ # result_lists = pool.map(func, range(world_size))+ processes = []+ for i in range(world_size):+ # Each process will now also take the output queue as an argument+ p = Process(target=run, args=(i, args, world_size, output_queue))+ p.daemon = False+ processes.append(p)+ p.start()+ results_by_rank = {}+ for _ in range(world_size):+ rank, results = output_queue.get() # Retrieve results as they finish+ results_by_rank[rank] = results+ print(f'Results received from rank {rank}')+ # ORDER THE RESULTS BY RANKlogger.info('finished running')- result_list = [res for res in itertools.chain(*result_lists)]+ for p in processes:+ p.join()++ results_list = list(itertools.chain.from_iterable(results_by_rank[i] for i in range(world_size)))+ # results_list = list(itertools.chain([results_by_rank[i] for i in range(world_size)]))+ # (data[key] for key in sorted_keys)+ # results_list = [item for sublist in results_by_rank.values() for item in sublist]+else:- result_list = run(0, world_size=1, args=args) # debug+ results_list = run(0, world_size=1, args=args) # debug++ print(results_list)+df = pd.read_csv(args.csv_path)# add a new column to the dataframe- df['text'] = result_list+ df['text'] = results_listdrop_failed = Trueif drop_failed:# iterate through the dataframe and delete the entire row if captioning failed