Open-Sora: Democratizing Efficient Video Production for All
Python
[sp] updated precision test
FrankLeeeee3552145f847b
diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.pyindex bb71d04..b0c046a 100644--- a/opensora/models/stdit/stdit3.py+++ b/opensora/models/stdit/stdit3.py@@ -368,12 +368,17 @@ class STDiT3(PreTrainedModel):# for simplicity, we can adjust the height to make it divisibleif self.enable_sequence_parallelism:sp_size = dist.get_world_size(get_sequence_parallel_group())- h_pad_size = sp_size - H % sp_size- hx_pad_size = h_pad_size * self.patch_size[1]+ if H % sp_size != 0:+ h_pad_size = sp_size - H % sp_size+ else:+ h_pad_size = 0- # pad x along the H dimension- H += h_pad_size- x = F.pad(x, (0, 0, 0, hx_pad_size))+ if h_pad_size > 0:+ hx_pad_size = h_pad_size * self.patch_size[1]++ # pad x along the H dimension+ H += h_pad_size+ x = F.pad(x, (0, 0, 0, hx_pad_size))S = H * Wbase_size = round(S**0.5)diff --git a/tests/test_stdit3_sequence_parallelism.py b/tests/test_stdit3_sequence_parallelism.pyindex ba715b7..70786f4 100644--- a/tests/test_stdit3_sequence_parallelism.py+++ b/tests/test_stdit3_sequence_parallelism.py@@ -9,7 +9,7 @@ from opensora.models.stdit.stdit3 import STDiT3, STDiT3Configdef get_sample_data():- x = torch.rand([1, 4, 15, 20, 27], dtype=torch.bfloat16) # (B, C, T, H, W)+ x = torch.rand([1, 4, 15, 20, 28], dtype=torch.bfloat16) # (B, C, T, H, W)timestep = torch.Tensor([924.0]).to(torch.bfloat16)y = torch.rand(1, 1, 300, 4096, dtype=torch.bfloat16)mask = torch.ones([1, 300], dtype=torch.int32)@@ -66,6 +66,17 @@ def run_model(rank, world_size, port):set_seed(1024)dist_model_cfg = get_stdit3_config(enable_sequence_parallelism=True)dist_model = STDiT3(dist_model_cfg).cuda().to(torch.bfloat16)++ # ensure model weights are equal+ for p1, p2 in zip(non_dist_model.parameters(), dist_model.parameters()):+ assert torch.equal(p1, p2)++ # ensure model weights are equal across all ranks+ for p in dist_model.parameters():+ p_list = [torch.zeros_like(p) for _ in range(world_size)]+ dist.all_gather(p_list, p, group=dist.group.WORLD)+ assert torch.equal(*p_list)+dist_out = dist_model(**data)dist_out.mean().backward()@@ -84,9 +95,8 @@ def run_model(rank, world_size, port):for (n1, p1), (n2, p2) in zip(non_dist_model.named_parameters(), dist_model.named_parameters()):assert n1 == n2if p1.grad is not None and p2.grad is not None:- if not torch.allclose(p1.grad, p2.grad, rtol=1e-2, atol=1e-4):- if dist.get_rank() == 0:- print(f"gradient of {n1} is not equal, {p1.grad} vs {p2.grad}")+ if not torch.allclose(p1.grad, p2.grad, rtol=1e-2, atol=1e-4) and dist.get_rank() == 0:+ print(f"gradient of {n1} is not equal, {p1.grad} vs {p2.grad}")else:assert p1.grad is None and p2.grad is None