Open-Sora: Democratizing Efficient Video Production for All
Python
specify output file
Tom Young811739dbba3a
diff --git a/tools/caption/caption_llama3.py b/tools/caption/caption_llama3.pyindex f463c52..8ccd992 100644--- a/tools/caption/caption_llama3.py+++ b/tools/caption/caption_llama3.py@@ -87,9 +87,9 @@ def main(args):dist.get_world_size())- output_file = os.path.splitext(args.input)[0] + f"_rank{dist.get_rank()}" + "_llama3.csv"-- writer = csv.writer(open(output_file, "w"))+ output_file = args.output_prefix + f"_rank{dist.get_rank()}" + "_llama3.csv"+ output_file_handle = open(output_file, "w")+ writer = csv.writer(output_file_handle)columns = list(dataframe.columns) + [args.key]writer.writerow(columns)@@ -206,12 +206,15 @@ def main(args):row = batch[idx]writer.writerow([*row, keywords])++ output_file_handle.close()dist.barrier()if dist.get_rank() == 0:- print("All ranks are finished. Collating the processed data to {}".format(output_file))+ collated_file = args.output_prefix + "_llama3.csv"+ print("All ranks are finished. Collating the processed data to {}".format(collated_file))import pandas as pd- csv_files = [os.path.splitext(args.input)[0] + f"_rank{i}" + "_llama3.csv" for i in range(dist.get_world_size())]+ csv_files = [args.output_prefix + f"_rank{i}" + "_llama3.csv" for i in range(dist.get_world_size())]# List to hold DataFramesdataframes = []# Read each CSV into a DataFrame and append to list@@ -221,7 +224,6 @@ def main(args):# Concatenate all DataFramescombined_df = pd.concat(dataframes, ignore_index=True)- collated_file = os.path.splitext(args.input)[0] + "_llama3.csv"# Save the combined DataFrame to a new CSV filecombined_df.to_csv(collated_file, index=False)print("Collated data saved to {}".format(collated_file))@@ -233,6 +235,7 @@ if __name__ == "__main__":parser = argparse.ArgumentParser()parser.add_argument("--model-id", default="meta-llama/Meta-Llama-3-8B-Instruct")parser.add_argument("input", type=str, help="Path to the input CSV file")+ parser.add_argument("--output_prefix", type=str, help="Path to the output CSV file")parser.add_argument("--prompt", type=str, default="")parser.add_argument("--batch_size", type=int, default=64)parser.add_argument('--key', default='summary', type=str)