Open-Sora: Democratizing Efficient Video Production for All
Python
fixed multiprocessing dataloader (#14)
Frank Lee46f9af34fc71
diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.pyindex f9cbf83..ff7dff5 100644--- a/opensora/datasets/datasets.py+++ b/opensora/datasets/datasets.py@@ -113,10 +113,6 @@ class VariableVideoTextDataset(VideoTextDataset):):super().__init__(data_path, num_frames, frame_interval, image_size, transform_name=None)self.transform_name = transform_name- self.data_info = self.data[["num_frames", "height", "width"]].to_numpy().tolist()-- def set_data_info(self, idx, T, H, W):- self.data_info[idx] = [T, H, W]def get_data_info(self, index):T = self.data.iloc[index]["num_frames"]@@ -125,11 +121,13 @@ class VariableVideoTextDataset(VideoTextDataset):return T, H, Wdef getitem(self, index):+ # a hack to pass in the (time, height, width) info from sampler+ index, num_frames, height, width = [int(val) for val in index.split("-")]+sample = self.data.iloc[index]path = sample["path"]text = sample["text"]file_type = self.get_type(path)- num_frames, height, width = self.data_info[index]ar = width / heightif file_type == "video":@@ -158,10 +156,4 @@ class VariableVideoTextDataset(VideoTextDataset):return {"video": video, "text": text, "num_frames": num_frames, "height": height, "width": width, "ar": ar}def __getitem__(self, index):- for _ in range(10):- try:- return self.getitem(index)- except Exception as e:- print(e)- index = np.random.randint(len(self))- raise RuntimeError("Too many bad data.")+ return self.getitem(index)diff --git a/opensora/datasets/sampler.py b/opensora/datasets/sampler.pyindex df7ccd7..a82869e 100644--- a/opensora/datasets/sampler.py+++ b/opensora/datasets/sampler.py@@ -47,8 +47,6 @@ class DistributedVariableVideoSampler(DistributedSampler):bucket_id = self.bucket.get_bucket_id(t, h, w, self.dataset.frame_interval, g)if bucket_id is None:continue- real_t, real_h, real_w = self.bucket.get_thw(bucket_id)- self.dataset.set_data_info(i, real_t, real_h, real_w)if bucket_id not in bucket_sample_dict:bucket_sample_dict[bucket_id] = []bucket_sample_dict[bucket_id].append(i)@@ -94,9 +92,14 @@ class DistributedVariableVideoSampler(DistributedSampler):if not found_last_bucket:continueself.last_bucket_id = k++ real_t, real_h, real_w = self.bucket.get_thw(bucket_id)for sample_idx in v:self.last_bucket_comsumed_samples += self.num_replicas- yield k, sample_idx++ # we return the (t, h, w) dimenisons for data processing+ # in the dataset's getitem method+ yield k, sample_idx, real_t, real_h, real_wself.last_bucket_comsumed_samples = 0self._reset()@@ -156,13 +159,16 @@ class VariableVideoBatchSampler(Sampler[List[int]]):sampler_iter = iter(self.sampler)# init cur buckettry:- cur_bucket_id, sample_idx = next(sampler_iter)+ cur_bucket_id, sample_idx, real_t, real_h, real_w = next(sampler_iter)except StopIteration:return++ # we hack the getitem method to pass in the (time, height, width) infocur_batch_size = self.bucket.get_batch_size(cur_bucket_id)- cur_sample_indices = [sample_idx]+ cur_sample_indices = [f"{sample_idx}-{real_t}-{real_h}-{real_w}"]+# iterate the rest- for bucket_id, sample_idx in sampler_iter:+ for bucket_id, sample_idx, real_t, real_h, real_w in sampler_iter:if len(cur_sample_indices) == cur_batch_size:yield cur_sample_indicescur_sample_indices = []@@ -171,9 +177,9 @@ class VariableVideoBatchSampler(Sampler[List[int]]):yield cur_sample_indicescur_bucket_id = bucket_idcur_batch_size = self.bucket.get_batch_size(cur_bucket_id)- cur_sample_indices = [sample_idx]+ cur_sample_indices = [f"{sample_idx}-{real_t}-{real_h}-{real_w}"]else:- cur_sample_indices.append(sample_idx)+ cur_sample_indices.append(f"{sample_idx}-{real_t}-{real_h}-{real_w}")if len(cur_sample_indices) > 0 and (not self.drop_last or len(cur_sample_indices) == cur_batch_size):yield cur_sample_indices