Open-Sora: Democratizing Efficient Video Production for All
Python
llama3 dp torchrun
Tom Younga3971731045f
diff --git a/tools/caption/caption_llama3.py b/tools/caption/caption_llama3.pyindex e3dc75a..6e79596 100644--- a/tools/caption/caption_llama3.py+++ b/tools/caption/caption_llama3.py@@ -1,127 +1,223 @@+from datetime import timedeltaimport argparse-import base64import csvimport os-from io import BytesIO+import torch.distributed as dist+import argparse+import csv+import os+from torch.utils.data import Dataset-import requests-import tqdm+from tqdm import tqdmfrom transformers import AutoTokenizer, AutoModelForCausalLMimport torch-from .utils import PROMPTS, VideoTextDataset, read_file+from .utils import read_fileimport warnings+import pandas as pdwarnings.filterwarnings("ignore")os.environ["TOKENIZERS_PARALLELISM"] = "false"+class CSVTextDataset(Dataset):+ def __init__(self, csv_path):+ self.df = pd.read_csv(csv_path)+ # assert text is in the columns+ assert 'text' in self.df.columns, "text column not found in the csv file"++ def __len__(self):+ return len(self.df)++ def __getitem__(self, idx):+ if idx < 0 or idx >= len(self.df):+ raise IndexError+ return self.df.iloc[idx]++ def set_rank_and_world_size(self, rank, world_size):+ self.rank = rank+ self.world_size = world_size+ self.data_per_gpu = len(self) // world_size+ self.start_index = rank * self.data_per_gpu+ self.end_index = (rank + 1) * self.data_per_gpu if rank != world_size - 1 else len(self)+ self.df = self.df.iloc[self.start_index:self.end_index]++ def write_to_csv(self, output_file, data, new_key):+ ''' write the part of the df to a csv file corresponding to the rank and write self.data_list as a new column '''+ writer = csv.writer(open(output_file, "w"))+ columns = self.df.columns + [new_key]+ writer.writerow(columns)+ for index, row in self.df.iterrows():+ if index < self.start_index or index >= self.end_index:+ continue+ writer.writerow([*row, data[index - self.start_index]])+ writer.close()++def pad_left(sequences, padding_value=0):+ # Determine the maximum length of the sequences+ max_len = max([s.size(0) for s in sequences])+ # Create a list to hold the padded sequences+ padded_sequences = []+ for sequence in sequences:+ # Calculate the number of padding elements needed for this sequence+ num_padding = max_len - sequence.size(0)+ # Create a tensor of padding values+ padding = torch.full((num_padding,), padding_value, dtype=sequence.dtype).to(sequence.device)+ # Concatenate the padding and the sequence to pad on the left+ padded_sequence = torch.cat([padding, sequence], dim=0)+ padded_sequences.append(padded_sequence)+ # Stack the padded sequences into a batch+ batch = torch.stack(padded_sequences)+ return batch+def main(args):# ======================================================- # 1. read video list+ # 1. init environment+ # ======================================================+ dist.init_process_group(backend="nccl", timeout=timedelta(hours=24))+ torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count())++ # ======================================================+ # 2. Prep rank-wise dataloader# ======================================================dataframe = read_file(args.input)print("read data from {}".format(args.input))-- output_file = os.path.splitext(args.input)[0] + "_llama3.csv"- f = open(output_file, "w")- writer = csv.writer(f)-- columns = dataframe.columns + [args.key]+ dataset = CSVTextDataset(args.input)+ dataset.set_rank_and_world_size(+ dist.get_rank(),+ dist.get_world_size()+ )++ output_file = os.path.splitext(args.input)[0] + f"_rank{dist.get_rank()}" + "_llama3.csv"++ writer = csv.writer(open(output_file, "w"))+ columns = list(dataframe.columns) + [args.key]+writer.writerow(columns)# add a new key named summary, write in csv file- print("the processed data saved to {}".format(output_file))+ print("the processed data saved on this rank will be saved to {}".format(output_file))+ def collate_fn(batch):+ return batch++ dataloader = torch.utils.data.DataLoader(+ dataset,+ num_workers=2,+ batch_size=args.batch_size,+ collate_fn=collate_fn,+ shuffle=False,+ )+# ======================================================# 2. process using llama3 and prompt# ======================================================print("Using model with the id {}".format(args.model_id))model_id = args.model_id- tokenizer = AutoTokenizer.from_pretrained(model_id)+ tokenizer = AutoTokenizer.from_pretrained(model_id, padding_side="left")model = AutoModelForCausalLM.from_pretrained(model_id,torch_dtype=torch.bfloat16,- device_map="auto",+ device_map=dist.get_rank() % torch.cuda.device_count(),)+ # .to(dist.get_rank() % torch.cuda.device_count())+ dist.barrier()print("======== Process data using LLAMA3 ========")- def extract(text, prompt):- messages = [- {"role": "system", "content": prompt},- {"role": "user", "content": text},+ def extract_batch(texts, prompt):+ input_ids_list = [+ tokenizer.apply_chat_template(+ [+ {"role": "system", "content": prompt},+ {"role": "user", "content": text}+ ],+ add_generation_prompt=True,+ return_tensors="pt"+ ).to(model.device)[0]+ for text in texts]++ attention_mask_list = [+ torch.ones(input_ids.shape, dtype=torch.long, device=model.device)+ for input_ids in input_ids_list+ ]++ # input_ids_batch = pad_left(+ # input_ids_list, padding_value=tokenizer.eos_token_id+ # )++ input_ids_batch = torch.nn.utils.rnn.pad_sequence(+ input_ids_list, batch_first=True, padding_value=tokenizer.eos_token_id+ )++ attention_mask_batch = torch.nn.utils.rnn.pad_sequence(+ attention_mask_list, batch_first=True, padding_value=0+ )++ # attention_mask_batch = pad_left(+ # attention_mask_list, padding_value=0+ # )- input_ids = tokenizer.apply_chat_template(- messages,- add_generation_prompt=True,- return_tensors="pt"- ).to(model.device)-- # The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.- # Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.terminators = [tokenizer.eos_token_id,tokenizer.convert_tokens_to_ids("<|eot_id|>"),]-- attention_mask = torch.ones(input_ids.shape, dtype=torch.long, device=model.device)-outputs = model.generate(- input_ids,+ input_ids_batch,max_new_tokens=512,- attention_mask=attention_mask,+ attention_mask=attention_mask_batch,pad_token_id=tokenizer.eos_token_id,eos_token_id=terminators,- do_sample=True,- temperature=0.6,- top_p=0.9,+ # do_sample=True,+ # temperature=0.6,+ # top_p=0.9,)- response = outputs[0][input_ids.shape[-1]:]- response = tokenizer.decode(response, skip_special_tokens=True)- return response++ responses = []+ for i in range(len(texts)):+ response = outputs[i][input_ids_list[i].shape[-1]:]+ response = tokenizer.decode(response, skip_special_tokens=True)+ responses.append(response)++ return responsesprint("Processing starting...")if args.prompt == "":- prompt = ("You are a AI assistant to extract user's text into keywords. "- "For example: user: 'a woman on a news station talking about traffic.', you just need a list of keywords separate by , and covered by '[' and ']': '[woman, news station, traffic]' ")+ prompt = ("You are a AI assistant to extract objects from user's text. "+ "For example: user: 'In this video a dog is running around. In addition, a person is laughing at the dog.', you produce a list of objects separated by ',' and wrapped by '[' and ']': '[dog, person]' ")else:prompt = args.promptprint("Prompt: {}".format(prompt))- key = args.key- # prompt to process text, first input prompt to AI assistant, second input is the text to process- # batch process, row['text'] to key extraction, for example: a woman on a news station talking about traffic. -> woman, news station, traffic-- length = len(dataframe)- for index, row in dataframe.iterrows():- row_text = row['text']-- # process text- keywords = extract(row_text, prompt)-- # process keywords- keywords_start = keywords.find("[")- keywords_end = keywords.find("]")- keywords = keywords[keywords_start+1:keywords_end]- if index % 100 == 0:- print("{}/{}".format(index, length))- print(f"text: {row_text} "- f"keywords: {keywords}")+ batch_size = args.batch_size+ # for i in tqdm(range(0, len(dataframe), batch_size)):+ for _, batch in enumerate(tqdm(dataloader)):+ # get the text column from the batch+ texts = [batch[i]['text'] for i in range(len(batch))]+ list_keywords = extract_batch(texts, prompt)- # add a new key named summary, write in csv file- writer.writerow([*row, keywords])+ for idx, keywords in enumerate(list_keywords):+ try:+ keywords_start = keywords.find("[")+ keywords_end = keywords.find("]")+ keywords = keywords[keywords_start+1:keywords_end]+ except:+ keywords = "NONE_FOUND"- f.close()+ row = batch[idx]+ writer.writerow([*row, keywords])+ dist.barrier()+ # terminate distributed env+ dist.destroy_process_group()if __name__ == "__main__":parser = argparse.ArgumentParser()- parser.add_argument("--model-id", default="/home/data/models/meta-llama/Meta-Llama-3-8B-Instruct")+ parser.add_argument("--model-id", default="meta-llama/Meta-Llama-3-8B-Instruct")parser.add_argument("input", type=str, help="Path to the input CSV file")parser.add_argument("--prompt", type=str, default="")- parser.add_argument("--batch", type=int, default=16)+ parser.add_argument("--batch_size", type=int, default=64)parser.add_argument('--key', default='summary', type=str)args = parser.parse_args()- main(args)+ main(args)\ No newline at end of file