Open-Sora: Democratizing Efficient Video Production for All
Python
enabled uneven data sharding for captioning (#41)
Frank Lee11978080f4c6
diff --git a/tools/caption/caption_llava.py b/tools/caption/caption_llava.pyindex 3ee21b6..b93b76f 100644--- a/tools/caption/caption_llava.py+++ b/tools/caption/caption_llava.py@@ -14,6 +14,7 @@ from llava.conversation import conv_templatesfrom llava.mm_utils import get_model_name_from_path, process_images, tokenizer_image_tokenfrom llava.model.builder import load_pretrained_modelfrom llava.utils import disable_torch_init+from torch.utils.data.distributed import DistributedSamplerfrom tqdm import tqdmfrom .acceleration.llava.policy import LlavaForCausalLMPolicy@@ -22,6 +23,38 @@ from .utils import IMG_EXTENSIONS, PROMPTS, VID_EXTENSIONS, Timer, VideoTextDatadisable_torch_init()+class NoPaddingDistributedSampler(DistributedSampler):+ def __init__(self, dataset, num_replicas=None, rank=None, shuffle=True, seed=0, drop_last=False):+ super().__init__(+ dataset=dataset, num_replicas=num_replicas, rank=rank, seed=seed, shuffle=False, drop_last=False+ )+ remainder = len(self.dataset) % self.num_replicas+ if remainder > 0 and (self.rank + 1) - remainder <= 0:+ # if the dataset is not divisible by num_replicas+ # the remaining items will be allocated to the first n ranks+ self.num_samples = len(self.dataset) // self.num_replicas + 1+ else:+ self.num_samples = len(self.dataset) // self.num_replicas+ self.total_size = len(dataset)++ def __iter__(self):+ if self.shuffle:+ # deterministically shuffle based on epoch and seed+ g = torch.Generator()+ g.manual_seed(self.seed + self.epoch)+ indices = torch.randperm(len(self.dataset), generator=g).tolist() # type: ignore[arg-type]+ else:+ indices = list(range(len(self.dataset))) # type: ignore[arg-type]++ # remove tail of data to make it evenly divisible.+ indices = indices[: self.total_size]++ # subsample+ indices = indices[self.rank : self.total_size : self.num_replicas]+ assert len(indices) == self.num_samples+ return iter(indices)++@torch.inference_mode()def main(args):# ======================================================@@ -138,9 +171,7 @@ def main(args):# build samplerdp_rank = dist.get_rank(dp_group)dp_size = dist.get_world_size(dp_group)- sampler = torch.utils.data.distributed.DistributedSampler(- dataset, rank=dp_rank, num_replicas=dp_size, shuffle=False- )+ sampler = NoPaddingDistributedSampler(dataset, rank=dp_rank, num_replicas=dp_size)# build dataloaderdataloader = torch.utils.data.DataLoader(