Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Tests: flush each DDP rank's result before teardown in the plain Trainer test (#16357)

Daniel Han0455cae42c1a
diff --git a/tests/python/_plain_trainer_ddp_gc_worker.py b/tests/python/_plain_trainer_ddp_gc_worker.py
index ea8d27c6143..6112f3d7a61 100644
--- a/tests/python/_plain_trainer_ddp_gc_worker.py
+++ b/tests/python/_plain_trainer_ddp_gc_worker.py
@@ -9,6 +9,14 @@ import runpy
import tempfile
+def _report(queue, item):
+ """Deliver the result before teardown. put() hands it to a feeder thread that needs the GIL, and a
+ teardown blocking in C while holding it left both ranks' results unsent after they had trained."""
+ queue.put(item)
+ queue.close()
+ queue.join_thread()
+
+
def run_rank(rank, port, cases, queue):
os.environ.update(
MASTER_ADDR = "127.0.0.1",
@@ -101,10 +109,10 @@ def run_rank(rank, port, cases, queue):
if error is not None:
# A rank that raised leaves the group unusable for the next case.
break
- queue.put((rank, out))
+ _report(queue, (rank, out))
except BaseException:
import traceback
- queue.put((rank, {"error": traceback.format_exc()}))
+ _report(queue, (rank, {"error": traceback.format_exc()}))
raise
if not any(case.get("error") for case in out.values()):
import torch.distributed as dist
diff --git a/tests/python/test_plain_trainer_ddp_gc.py b/tests/python/test_plain_trainer_ddp_gc.py
index ef48acca7b3..2da83716341 100644
--- a/tests/python/test_plain_trainer_ddp_gc.py
+++ b/tests/python/test_plain_trainer_ddp_gc.py
@@ -47,13 +47,14 @@ def _run(cases):
try:
while len(results) < len(procs):
try:
- rank, out = queue.get(timeout = 60 if _failed(results) else 600)
+ # Below the 330 s pytest timeout, so a hang fails here with what did arrive.
+ rank, out = queue.get(timeout = 60 if _failed(results) else 240)
except queue_module.Empty:
break
results[rank] = out
finally:
for p in procs:
- p.join(5 if len(results) < len(procs) else 120)
+ p.join(5 if len(results) < len(procs) else 30)
if p.is_alive():
p.kill()
assert results, "no rank reported"