Open-Sora: Democratizing Efficient Video Production for All
Python
fix nans and enable error propagation
Tom Youngd30752bc925e
diff --git a/tools/caption/caption_llama3.py b/tools/caption/caption_llama3.pyindex 273105c..8328f9b 100644--- a/tools/caption/caption_llama3.py+++ b/tools/caption/caption_llama3.py@@ -17,7 +17,7 @@ import pandas as pdos.system(f'cp {__file__} ~/backup/') # optionally backup the scriptwarnings.filterwarnings("ignore")os.environ["TOKENIZERS_PARALLELISM"] = "false"-+from torch.distributed.elastic.multiprocessing.errors import recordclass CSVTextDataset(Dataset):def __init__(self, csv_path):@@ -69,6 +69,7 @@ def pad_left(sequences, padding_value=0):batch = torch.stack(padded_sequences)return batch+@recorddef main(args):# ======================================================# 1. init environment@@ -204,7 +205,7 @@ def main(args):keywords_start = keywords.find("[")keywords_end = keywords.find("]")keywords = keywords[keywords_start+1:keywords_end]- if "\n" in keywords: # we empirically observe that it produces newlines when no keywords are found+ if "\n" in keywords or len(keywords) == 0: # we empirically observe that it produces newlines when no keywords are foundkeywords = "NONE_FOUND"except:keywords = "NONE_FOUND"