Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Tests: flush each DDP rank's result before teardown in the plain Trainer test (#16357)
Daniel Han0455cae42c1a
diff --git a/tests/python/_plain_trainer_ddp_gc_worker.py b/tests/python/_plain_trainer_ddp_gc_worker.pyindex ea8d27c6143..6112f3d7a61 100644--- a/tests/python/_plain_trainer_ddp_gc_worker.py+++ b/tests/python/_plain_trainer_ddp_gc_worker.py@@ -9,6 +9,14 @@ import runpyimport tempfile+def _report(queue, item):+ """Deliver the result before teardown. put() hands it to a feeder thread that needs the GIL, and a+ teardown blocking in C while holding it left both ranks' results unsent after they had trained."""+ queue.put(item)+ queue.close()+ queue.join_thread()++def run_rank(rank, port, cases, queue):os.environ.update(MASTER_ADDR = "127.0.0.1",@@ -101,10 +109,10 @@ def run_rank(rank, port, cases, queue):if error is not None:# A rank that raised leaves the group unusable for the next case.break- queue.put((rank, out))+ _report(queue, (rank, out))except BaseException:import traceback- queue.put((rank, {"error": traceback.format_exc()}))+ _report(queue, (rank, {"error": traceback.format_exc()}))raiseif not any(case.get("error") for case in out.values()):import torch.distributed as distdiff --git a/tests/python/test_plain_trainer_ddp_gc.py b/tests/python/test_plain_trainer_ddp_gc.pyindex ef48acca7b3..2da83716341 100644--- a/tests/python/test_plain_trainer_ddp_gc.py+++ b/tests/python/test_plain_trainer_ddp_gc.py@@ -47,13 +47,14 @@ def _run(cases):try:while len(results) < len(procs):try:- rank, out = queue.get(timeout = 60 if _failed(results) else 600)+ # Below the 330 s pytest timeout, so a hang fails here with what did arrive.+ rank, out = queue.get(timeout = 60 if _failed(results) else 240)except queue_module.Empty:breakresults[rank] = outfinally:for p in procs:- p.join(5 if len(results) < len(procs) else 120)+ p.join(5 if len(results) < len(procs) else 30)if p.is_alive():p.kill()assert results, "no rank reported"