Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Distributed tests: pick MASTER_PORT outside the ephemeral range, write vision results atomically (#14334)
Daniel Han846cc5e600c3
diff --git a/tests/_rendezvous_port.py b/tests/_rendezvous_port.pynew file mode 100644index 00000000000..52f5c1d7713--- /dev/null+++ b/tests/_rendezvous_port.py@@ -0,0 +1,43 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A free port for a test's torch.distributed rendezvous (MASTER_PORT).++``bind(0)`` hands out a port from the kernel's ephemeral range and the test releases it before rank 0+listens on it. Gloo's own peer connections, in this test and in every other xdist worker, draw their+local ports from that same range, so one of them can take the port first and rank 0 fails with+EADDRINUSE (seen in Repo tests (CPU, fsdp2): "port: 33653 ... address already in use"). Ports here come+from below every OS's default ephemeral range, and each xdist worker draws from its own slice, so+neither an outgoing connection nor a test running alongside can pick the same one.+"""++import os+import random+import socket++# Linux allocates ephemeral ports from 32768 and macOS / Windows from 49152, so nothing below 32768 is+# handed out by the kernel on its own.+LOW, HIGH = 20000, 32000+SLICE = 1000+++def _worker_index() -> int:+ worker = os.environ.get("PYTEST_XDIST_WORKER", "gw0")+ return int(worker[2:]) if worker[2:].isdigit() else 0+++def free_port() -> int:+ base = LOW + (_worker_index() * SLICE) % (HIGH - LOW)+ rng = random.Random()+ for _ in range(64):+ port = base + rng.randrange(SLICE)+ with socket.socket() as s:+ try:+ s.bind(("127.0.0.1", port))+ except OSError: # taken, or still in TIME_WAIT from an earlier test in this worker+ continue+ return port+ # Every candidate busy: the kernel's choice still works, just without the guarantee.+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]diff --git a/tests/test_context_parallel_fsdp2_gloo.py b/tests/test_context_parallel_fsdp2_gloo.pyindex 3956f1e99a6..9eacb00ef04 100644--- a/tests/test_context_parallel_fsdp2_gloo.py+++ b/tests/test_context_parallel_fsdp2_gloo.py@@ -12,7 +12,6 @@ whole batch. The model is token-wise, so attention exchange is not exercised herfrom __future__ import annotationsimport os-import socketimport typesimport pytest@@ -32,9 +31,8 @@ SEQ = 13def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()class Block(torch.nn.Module):diff --git a/tests/test_context_parallel_fsdp2_save_gloo.py b/tests/test_context_parallel_fsdp2_save_gloo.pyindex d45a7c844f5..18670ab3945 100644--- a/tests/test_context_parallel_fsdp2_save_gloo.py+++ b/tests/test_context_parallel_fsdp2_save_gloo.py@@ -12,7 +12,6 @@ and leave the installed config in place afterwards.from __future__ import annotationsimport os-import socketimport typesimport pytest@@ -28,9 +27,8 @@ if not dist.is_available() or not dist.is_gloo_available():def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _accelerator_less_import():diff --git a/tests/test_fsdp2_detection.py b/tests/test_fsdp2_detection.pyindex f89952e7e36..cabfd185d01 100644--- a/tests/test_fsdp2_detection.py+++ b/tests/test_fsdp2_detection.py@@ -9,7 +9,6 @@ from __future__ import annotationsimport astimport importlib.utilimport os-import socketfrom pathlib import Pathimport pytest@@ -260,9 +259,8 @@ def test_a_built_accelerator_is_authoritative(monkeypatch):def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _rank_worker(rank, port, out_dir):diff --git a/tests/test_fsdp2_frozen_dtypes_gloo.py b/tests/test_fsdp2_frozen_dtypes_gloo.pyindex 1408e22d4f7..8b98900d89b 100644--- a/tests/test_fsdp2_frozen_dtypes_gloo.py+++ b/tests/test_fsdp2_frozen_dtypes_gloo.py@@ -11,7 +11,6 @@ from __future__ import annotationsimport importlib.utilimport osimport pathlib-import socketimport typesimport pytest@@ -36,9 +35,8 @@ def _fsdp2_module():def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()class _Block(torch.nn.Module):diff --git a/tests/test_fsdp2_native_layout_agree_gloo.py b/tests/test_fsdp2_native_layout_agree_gloo.pyindex e28059af2af..f3a4955494a 100644--- a/tests/test_fsdp2_native_layout_agree_gloo.py+++ b/tests/test_fsdp2_native_layout_agree_gloo.py@@ -16,7 +16,6 @@ if multiprocessing.parent_process() is not None:os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hostsimport importlib # noqa: E402-import socket # noqa: E402import sys # noqa: E402import pytest # noqa: E402@@ -29,9 +28,8 @@ REPO_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _init(diff --git a/tests/test_fsdp2_native_quant_gloo.py b/tests/test_fsdp2_native_quant_gloo.pyindex 3219ec79d5b..e83d9b3d16c 100644--- a/tests/test_fsdp2_native_quant_gloo.py+++ b/tests/test_fsdp2_native_quant_gloo.py@@ -13,7 +13,6 @@ plain Parameter with no quant_state. A 16-bit model is left untouched. The modulimport importlib.utilimport os-import socketimport sysimport typesfrom pathlib import Path@@ -56,9 +55,8 @@ def _load_models_package():def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()N_BLOCKS = 3diff --git a/tests/test_fsdp2_native_resume_gloo.py b/tests/test_fsdp2_native_resume_gloo.pyindex d054cd603a2..2b81d06f4f7 100644--- a/tests/test_fsdp2_native_resume_gloo.py+++ b/tests/test_fsdp2_native_resume_gloo.py@@ -8,7 +8,6 @@ native_load_optimizer, then 2 more steps on both: losses and parameters must becontrol: rank 0's plain `optimizer.state_dict()` (the Trainer's own save) holds only rank 0's shards."""import os-import socketimport sysimport pytest@@ -26,9 +25,8 @@ REPO_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _model(lora, seed):diff --git a/tests/test_fsdp2_native_trainer_ckpt_gloo.py b/tests/test_fsdp2_native_trainer_ckpt_gloo.pyindex 788b7521df5..8bdfec28bda 100644--- a/tests/test_fsdp2_native_trainer_ckpt_gloo.py+++ b/tests/test_fsdp2_native_trainer_ckpt_gloo.py@@ -16,7 +16,6 @@ if multiprocessing.parent_process() is not None:os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hostsimport ast # noqa: E402-import socket # noqa: E402import sys # noqa: E402import types # noqa: E402@@ -30,9 +29,8 @@ REPO_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _stub_unsloth_save():diff --git a/tests/test_fsdp2_native_units.py b/tests/test_fsdp2_native_units.pyindex 801d6391734..fc0a434af4a 100644--- a/tests/test_fsdp2_native_units.py+++ b/tests/test_fsdp2_native_units.py@@ -8,7 +8,6 @@ it runs without a GPU."""import importlib.utilimport os-import socketimport sysimport types@@ -104,9 +103,8 @@ def _expected_blocks(model):def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _batch(model, seed = 1):diff --git a/tests/test_fsdp2_native_vision_gloo.py b/tests/test_fsdp2_native_vision_gloo.pyindex 80da3f6fcd5..613ef6dc70e 100644--- a/tests/test_fsdp2_native_vision_gloo.py+++ b/tests/test_fsdp2_native_vision_gloo.py@@ -12,7 +12,6 @@ import datetimeimport importlib.utilimport jsonimport os-import socketimport sysimport types@@ -48,9 +47,8 @@ def _load_native():def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _build(lora):@@ -162,8 +160,10 @@ def _worker(rank, world, port, lora, root_only, dummy, out):result.update(ok = True, worst_rel_grad = worst, compared = compared, losses = losses)except Exception as exc:result.update(ok = False, error = f"{type(exc).__name__}: {str(exc)[:300]}")- with open(f"{out}.{rank}", "w", encoding = "utf-8") as f:+ # Written whole or not at all: a surviving rank is killed once its peer dies, possibly mid-write.+ with open(f"{out}.{rank}.tmp", "w", encoding = "utf-8") as f:json.dump(result, f)+ os.replace(f"{out}.{rank}.tmp", f"{out}.{rank}")try:dist.destroy_process_group()except Exception:@@ -209,7 +209,7 @@ def _run(lora, root_only, dummy, tmp_path):try:with open(f"{out}.{rank}", encoding = "utf-8") as f:results.append(json.load(f))- except FileNotFoundError:+ except (FileNotFoundError, json.JSONDecodeError):results.append({"rank": rank, "ok": False, "error": "no result (hung or killed)"})return resultsdiff --git a/tests/test_fsdp2_peft_checkpoint_gloo.py b/tests/test_fsdp2_peft_checkpoint_gloo.pyindex b4613d0919a..4504bea1017 100644--- a/tests/test_fsdp2_peft_checkpoint_gloo.py+++ b/tests/test_fsdp2_peft_checkpoint_gloo.py@@ -12,7 +12,6 @@ and loading it back must restore every rank's shard.from __future__ import annotationsimport os-import socketimport typesimport pytest@@ -28,9 +27,8 @@ if not dist.is_available() or not dist.is_gloo_available():def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _build():diff --git a/tests/test_fsdp2_perf_layout_gloo.py b/tests/test_fsdp2_perf_layout_gloo.pyindex cce2a15d458..ef52aa4971e 100644--- a/tests/test_fsdp2_perf_layout_gloo.py+++ b/tests/test_fsdp2_perf_layout_gloo.py@@ -12,7 +12,6 @@ plain sharded layout when the model does not fit or CPU offload is asked for.from __future__ import annotationsimport os-import socketimport pytest@@ -40,9 +39,8 @@ _spec.loader.exec_module(fsdp2_perf)def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()D = 16diff --git a/tests/test_fsdp2_qlora.py b/tests/test_fsdp2_qlora.pyindex 79f13613c43..c60cf9f9557 100644--- a/tests/test_fsdp2_qlora.py+++ b/tests/test_fsdp2_qlora.py@@ -9,7 +9,6 @@ bitsandbytes Linear4bit layers (CPU backend) across 2 gloo ranks with FSDP2's fuimport importlib.utilimport os-import socketimport sysfrom pathlib import Path@@ -217,9 +216,8 @@ def test_nonfloat_guard_restores_parameter_new():def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _worker(rank, world_size, port, case, queue):diff --git a/tests/test_fsdp2_qlora_moe_gloo.py b/tests/test_fsdp2_qlora_moe_gloo.pyindex 09780ab8588..ae259fa760f 100644--- a/tests/test_fsdp2_qlora_moe_gloo.py+++ b/tests/test_fsdp2_qlora_moe_gloo.py@@ -14,7 +14,6 @@ GPU-free: real bitsandbytes 4-bit weights (CPU backend) sharded over 2 gloo rankimport importlib.utilimport os-import socketimport sysfrom pathlib import Path@@ -46,9 +45,8 @@ def _load_module():def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _stack_params4bit(shape, seed, module):diff --git a/tests/test_fsdp2_save_gloo.py b/tests/test_fsdp2_save_gloo.pyindex a22c479fe62..2255ca22be2 100644--- a/tests/test_fsdp2_save_gloo.py+++ b/tests/test_fsdp2_save_gloo.py@@ -16,7 +16,6 @@ The modules are loaded from their files (importing unsloth needs a GPU).import importlib.utilimport os-import socketimport sysfrom pathlib import Path@@ -48,9 +47,8 @@ N_LAYERS = 3def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()class _Block(torch.nn.Module):diff --git a/tests/test_fsdp2_vision_gate_gloo.py b/tests/test_fsdp2_vision_gate_gloo.pyindex cfcec2d91a9..64884f0c437 100644--- a/tests/test_fsdp2_vision_gate_gloo.py+++ b/tests/test_fsdp2_vision_gate_gloo.py@@ -18,7 +18,6 @@ import importlib.utilimport inspectimport jsonimport os-import socketimport sysimport types@@ -48,9 +47,8 @@ def _load_native():def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _config():diff --git a/tests/test_fsdp2_wrap_gloo.py b/tests/test_fsdp2_wrap_gloo.pyindex 3c0564d68c9..2b424db68b8 100644--- a/tests/test_fsdp2_wrap_gloo.py+++ b/tests/test_fsdp2_wrap_gloo.py@@ -16,7 +16,6 @@ What Unsloth adds after accelerate shards the model (unsloth/models/fsdp2.py):from __future__ import annotationsimport os-import socketimport typesimport pytest@@ -34,9 +33,8 @@ N_LAYERS = 3def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()def _spy_mlp(self, X):diff --git a/tests/test_grpo_fsdp2_scoring_head.py b/tests/test_grpo_fsdp2_scoring_head.pyindex 0461fdf1bfd..4a4f85578a0 100644--- a/tests/test_grpo_fsdp2_scoring_head.py+++ b/tests/test_grpo_fsdp2_scoring_head.py@@ -14,7 +14,6 @@ import astimport collectionsimport copyimport os-import socketimport sysimport textwrapfrom pathlib import Path@@ -205,9 +204,8 @@ def _worker(rank, port, separate_head, setup_source, loop_source, errors):def _free_port():- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]+ from tests._rendezvous_port import free_port+ return free_port()@pytest.mark.parametrize("separate_head", [False, True], ids = ["root_head", "separate_head"])diff --git a/tests/test_rendezvous_port.py b/tests/test_rendezvous_port.pynew file mode 100644index 00000000000..73f0df7bd21--- /dev/null+++ b/tests/test_rendezvous_port.py@@ -0,0 +1,48 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The distributed tests' MASTER_PORT comes from outside the ephemeral range, in a per-worker slice."""++import socket++from tests import _rendezvous_port as rp+++def test_the_port_is_below_every_default_ephemeral_range_and_in_this_workers_slice(monkeypatch):+ for worker, base in (("gw0", 20000), ("gw2", 22000), ("gw11", 31000)):+ monkeypatch.setenv("PYTEST_XDIST_WORKER", worker)+ for _ in range(20):+ port = rp.free_port()+ assert base <= port < base + rp.SLICE < 32768, (worker, port)+++def test_no_xdist_worker_uses_the_first_slice(monkeypatch):+ monkeypatch.delenv("PYTEST_XDIST_WORKER", raising = False)+ assert 20000 <= rp.free_port() < 21000+++def test_a_taken_port_is_skipped(monkeypatch):+ monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw5")+ held = socket.socket()+ try:+ taken = next(p for p in range(25000, 26000) if _try_listen(held, p))+ other = 25000 + (taken - 25000 + 1) % rp.SLICE+ offsets = iter([taken - 25000, other - 25000])++ class _Rng:+ def randrange(self, n):+ return next(offsets)++ monkeypatch.setattr(rp.random, "Random", _Rng)+ assert rp.free_port() == other+ finally:+ held.close()+++def _try_listen(sock, port):+ try:+ sock.bind(("127.0.0.1", port))+ except OSError:+ return False+ sock.listen()+ return True