Open-Sora: Democratizing Efficient Video Production for All
Python
[data] added error handling to dataset
FrankLeeeee6a72b8910b29
diff --git a/opensora/datasets/dataloader.py b/opensora/datasets/dataloader.pyindex 15058ac..60d0b24 100644--- a/opensora/datasets/dataloader.py+++ b/opensora/datasets/dataloader.py@@ -111,6 +111,9 @@ def prepare_dataloader(def collate_fn_default(batch):+ # filter out None+ batch = [x for x in batch if x is not None]+# HACK: for loading text featuresuse_mask = Falseif "mask" in batch[0] and isinstance(batch[0]["mask"], int):@@ -132,6 +135,9 @@ def collate_fn_batch(batch):"""Used only with BatchDistributedSampler"""+ # filter out None+ batch = [x for x in batch if x is not None]+res = torch.utils.data.default_collate(batch)# squeeze the first dimension, which is due to torch.stack() in default_collate()diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.pyindex 8b5fdd6..b148268 100644--- a/opensora/datasets/datasets.py+++ b/opensora/datasets/datasets.py@@ -190,7 +190,10 @@ class VariableVideoTextDataset(VideoTextDataset):return retdef __getitem__(self, index):- return self.getitem(index)+ try:+ return self.getitem(index)+ except:+ return None@DATASETS.register_module()