Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[sp] updated precision test

FrankLeeeee3552145f847b
diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.py
index bb71d04..b0c046a 100644
--- a/opensora/models/stdit/stdit3.py
+++ b/opensora/models/stdit/stdit3.py
@@ -368,12 +368,17 @@ class STDiT3(PreTrainedModel):
# for simplicity, we can adjust the height to make it divisible
if self.enable_sequence_parallelism:
sp_size = dist.get_world_size(get_sequence_parallel_group())
- h_pad_size = sp_size - H % sp_size
- hx_pad_size = h_pad_size * self.patch_size[1]
+ if H % sp_size != 0:
+ h_pad_size = sp_size - H % sp_size
+ else:
+ h_pad_size = 0
- # pad x along the H dimension
- H += h_pad_size
- x = F.pad(x, (0, 0, 0, hx_pad_size))
+ if h_pad_size > 0:
+ hx_pad_size = h_pad_size * self.patch_size[1]
+
+ # pad x along the H dimension
+ H += h_pad_size
+ x = F.pad(x, (0, 0, 0, hx_pad_size))
S = H * W
base_size = round(S**0.5)
diff --git a/tests/test_stdit3_sequence_parallelism.py b/tests/test_stdit3_sequence_parallelism.py
index ba715b7..70786f4 100644
--- a/tests/test_stdit3_sequence_parallelism.py
+++ b/tests/test_stdit3_sequence_parallelism.py
@@ -9,7 +9,7 @@ from opensora.models.stdit.stdit3 import STDiT3, STDiT3Config
def get_sample_data():
- x = torch.rand([1, 4, 15, 20, 27], dtype=torch.bfloat16) # (B, C, T, H, W)
+ x = torch.rand([1, 4, 15, 20, 28], dtype=torch.bfloat16) # (B, C, T, H, W)
timestep = torch.Tensor([924.0]).to(torch.bfloat16)
y = torch.rand(1, 1, 300, 4096, dtype=torch.bfloat16)
mask = torch.ones([1, 300], dtype=torch.int32)
@@ -66,6 +66,17 @@ def run_model(rank, world_size, port):
set_seed(1024)
dist_model_cfg = get_stdit3_config(enable_sequence_parallelism=True)
dist_model = STDiT3(dist_model_cfg).cuda().to(torch.bfloat16)
+
+ # ensure model weights are equal
+ for p1, p2 in zip(non_dist_model.parameters(), dist_model.parameters()):
+ assert torch.equal(p1, p2)
+
+ # ensure model weights are equal across all ranks
+ for p in dist_model.parameters():
+ p_list = [torch.zeros_like(p) for _ in range(world_size)]
+ dist.all_gather(p_list, p, group=dist.group.WORLD)
+ assert torch.equal(*p_list)
+
dist_out = dist_model(**data)
dist_out.mean().backward()
@@ -84,9 +95,8 @@ def run_model(rank, world_size, port):
for (n1, p1), (n2, p2) in zip(non_dist_model.named_parameters(), dist_model.named_parameters()):
assert n1 == n2
if p1.grad is not None and p2.grad is not None:
- if not torch.allclose(p1.grad, p2.grad, rtol=1e-2, atol=1e-4):
- if dist.get_rank() == 0:
- print(f"gradient of {n1} is not equal, {p1.grad} vs {p2.grad}")
+ if not torch.allclose(p1.grad, p2.grad, rtol=1e-2, atol=1e-4) and dist.get_rank() == 0:
+ print(f"gradient of {n1} is not equal, {p1.grad} vs {p2.grad}")
else:
assert p1.grad is None and p2.grad is None