Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Distributed tests: pick MASTER_PORT outside the ephemeral range, write vision results atomically (#14334)

Daniel Han846cc5e600c3
diff --git a/tests/_rendezvous_port.py b/tests/_rendezvous_port.py
new file mode 100644
index 00000000000..52f5c1d7713
--- /dev/null
+++ b/tests/_rendezvous_port.py
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A free port for a test's torch.distributed rendezvous (MASTER_PORT).
+
+``bind(0)`` hands out a port from the kernel's ephemeral range and the test releases it before rank 0
+listens on it. Gloo's own peer connections, in this test and in every other xdist worker, draw their
+local ports from that same range, so one of them can take the port first and rank 0 fails with
+EADDRINUSE (seen in Repo tests (CPU, fsdp2): "port: 33653 ... address already in use"). Ports here come
+from below every OS's default ephemeral range, and each xdist worker draws from its own slice, so
+neither an outgoing connection nor a test running alongside can pick the same one.
+"""
+
+import os
+import random
+import socket
+
+# Linux allocates ephemeral ports from 32768 and macOS / Windows from 49152, so nothing below 32768 is
+# handed out by the kernel on its own.
+LOW, HIGH = 20000, 32000
+SLICE = 1000
+
+
+def _worker_index() -> int:
+ worker = os.environ.get("PYTEST_XDIST_WORKER", "gw0")
+ return int(worker[2:]) if worker[2:].isdigit() else 0
+
+
+def free_port() -> int:
+ base = LOW + (_worker_index() * SLICE) % (HIGH - LOW)
+ rng = random.Random()
+ for _ in range(64):
+ port = base + rng.randrange(SLICE)
+ with socket.socket() as s:
+ try:
+ s.bind(("127.0.0.1", port))
+ except OSError: # taken, or still in TIME_WAIT from an earlier test in this worker
+ continue
+ return port
+ # Every candidate busy: the kernel's choice still works, just without the guarantee.
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
diff --git a/tests/test_context_parallel_fsdp2_gloo.py b/tests/test_context_parallel_fsdp2_gloo.py
index 3956f1e99a6..9eacb00ef04 100644
--- a/tests/test_context_parallel_fsdp2_gloo.py
+++ b/tests/test_context_parallel_fsdp2_gloo.py
@@ -12,7 +12,6 @@ whole batch. The model is token-wise, so attention exchange is not exercised her
from __future__ import annotations
import os
-import socket
import types
import pytest
@@ -32,9 +31,8 @@ SEQ = 13
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
class Block(torch.nn.Module):
diff --git a/tests/test_context_parallel_fsdp2_save_gloo.py b/tests/test_context_parallel_fsdp2_save_gloo.py
index d45a7c844f5..18670ab3945 100644
--- a/tests/test_context_parallel_fsdp2_save_gloo.py
+++ b/tests/test_context_parallel_fsdp2_save_gloo.py
@@ -12,7 +12,6 @@ and leave the installed config in place afterwards.
from __future__ import annotations
import os
-import socket
import types
import pytest
@@ -28,9 +27,8 @@ if not dist.is_available() or not dist.is_gloo_available():
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _accelerator_less_import():
diff --git a/tests/test_fsdp2_detection.py b/tests/test_fsdp2_detection.py
index f89952e7e36..cabfd185d01 100644
--- a/tests/test_fsdp2_detection.py
+++ b/tests/test_fsdp2_detection.py
@@ -9,7 +9,6 @@ from __future__ import annotations
import ast
import importlib.util
import os
-import socket
from pathlib import Path
import pytest
@@ -260,9 +259,8 @@ def test_a_built_accelerator_is_authoritative(monkeypatch):
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _rank_worker(rank, port, out_dir):
diff --git a/tests/test_fsdp2_frozen_dtypes_gloo.py b/tests/test_fsdp2_frozen_dtypes_gloo.py
index 1408e22d4f7..8b98900d89b 100644
--- a/tests/test_fsdp2_frozen_dtypes_gloo.py
+++ b/tests/test_fsdp2_frozen_dtypes_gloo.py
@@ -11,7 +11,6 @@ from __future__ import annotations
import importlib.util
import os
import pathlib
-import socket
import types
import pytest
@@ -36,9 +35,8 @@ def _fsdp2_module():
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
class _Block(torch.nn.Module):
diff --git a/tests/test_fsdp2_native_layout_agree_gloo.py b/tests/test_fsdp2_native_layout_agree_gloo.py
index e28059af2af..f3a4955494a 100644
--- a/tests/test_fsdp2_native_layout_agree_gloo.py
+++ b/tests/test_fsdp2_native_layout_agree_gloo.py
@@ -16,7 +16,6 @@ if multiprocessing.parent_process() is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
import importlib # noqa: E402
-import socket # noqa: E402
import sys # noqa: E402
import pytest # noqa: E402
@@ -29,9 +28,8 @@ REPO_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _init(
diff --git a/tests/test_fsdp2_native_quant_gloo.py b/tests/test_fsdp2_native_quant_gloo.py
index 3219ec79d5b..e83d9b3d16c 100644
--- a/tests/test_fsdp2_native_quant_gloo.py
+++ b/tests/test_fsdp2_native_quant_gloo.py
@@ -13,7 +13,6 @@ plain Parameter with no quant_state. A 16-bit model is left untouched. The modul
import importlib.util
import os
-import socket
import sys
import types
from pathlib import Path
@@ -56,9 +55,8 @@ def _load_models_package():
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
N_BLOCKS = 3
diff --git a/tests/test_fsdp2_native_resume_gloo.py b/tests/test_fsdp2_native_resume_gloo.py
index d054cd603a2..2b81d06f4f7 100644
--- a/tests/test_fsdp2_native_resume_gloo.py
+++ b/tests/test_fsdp2_native_resume_gloo.py
@@ -8,7 +8,6 @@ native_load_optimizer, then 2 more steps on both: losses and parameters must be
control: rank 0's plain `optimizer.state_dict()` (the Trainer's own save) holds only rank 0's shards."""
import os
-import socket
import sys
import pytest
@@ -26,9 +25,8 @@ REPO_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _model(lora, seed):
diff --git a/tests/test_fsdp2_native_trainer_ckpt_gloo.py b/tests/test_fsdp2_native_trainer_ckpt_gloo.py
index 788b7521df5..8bdfec28bda 100644
--- a/tests/test_fsdp2_native_trainer_ckpt_gloo.py
+++ b/tests/test_fsdp2_native_trainer_ckpt_gloo.py
@@ -16,7 +16,6 @@ if multiprocessing.parent_process() is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
import ast # noqa: E402
-import socket # noqa: E402
import sys # noqa: E402
import types # noqa: E402
@@ -30,9 +29,8 @@ REPO_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _stub_unsloth_save():
diff --git a/tests/test_fsdp2_native_units.py b/tests/test_fsdp2_native_units.py
index 801d6391734..fc0a434af4a 100644
--- a/tests/test_fsdp2_native_units.py
+++ b/tests/test_fsdp2_native_units.py
@@ -8,7 +8,6 @@ it runs without a GPU."""
import importlib.util
import os
-import socket
import sys
import types
@@ -104,9 +103,8 @@ def _expected_blocks(model):
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _batch(model, seed = 1):
diff --git a/tests/test_fsdp2_native_vision_gloo.py b/tests/test_fsdp2_native_vision_gloo.py
index 80da3f6fcd5..613ef6dc70e 100644
--- a/tests/test_fsdp2_native_vision_gloo.py
+++ b/tests/test_fsdp2_native_vision_gloo.py
@@ -12,7 +12,6 @@ import datetime
import importlib.util
import json
import os
-import socket
import sys
import types
@@ -48,9 +47,8 @@ def _load_native():
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _build(lora):
@@ -162,8 +160,10 @@ def _worker(rank, world, port, lora, root_only, dummy, out):
result.update(ok = True, worst_rel_grad = worst, compared = compared, losses = losses)
except Exception as exc:
result.update(ok = False, error = f"{type(exc).__name__}: {str(exc)[:300]}")
- with open(f"{out}.{rank}", "w", encoding = "utf-8") as f:
+ # Written whole or not at all: a surviving rank is killed once its peer dies, possibly mid-write.
+ with open(f"{out}.{rank}.tmp", "w", encoding = "utf-8") as f:
json.dump(result, f)
+ os.replace(f"{out}.{rank}.tmp", f"{out}.{rank}")
try:
dist.destroy_process_group()
except Exception:
@@ -209,7 +209,7 @@ def _run(lora, root_only, dummy, tmp_path):
try:
with open(f"{out}.{rank}", encoding = "utf-8") as f:
results.append(json.load(f))
- except FileNotFoundError:
+ except (FileNotFoundError, json.JSONDecodeError):
results.append({"rank": rank, "ok": False, "error": "no result (hung or killed)"})
return results
diff --git a/tests/test_fsdp2_peft_checkpoint_gloo.py b/tests/test_fsdp2_peft_checkpoint_gloo.py
index b4613d0919a..4504bea1017 100644
--- a/tests/test_fsdp2_peft_checkpoint_gloo.py
+++ b/tests/test_fsdp2_peft_checkpoint_gloo.py
@@ -12,7 +12,6 @@ and loading it back must restore every rank's shard.
from __future__ import annotations
import os
-import socket
import types
import pytest
@@ -28,9 +27,8 @@ if not dist.is_available() or not dist.is_gloo_available():
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _build():
diff --git a/tests/test_fsdp2_perf_layout_gloo.py b/tests/test_fsdp2_perf_layout_gloo.py
index cce2a15d458..ef52aa4971e 100644
--- a/tests/test_fsdp2_perf_layout_gloo.py
+++ b/tests/test_fsdp2_perf_layout_gloo.py
@@ -12,7 +12,6 @@ plain sharded layout when the model does not fit or CPU offload is asked for.
from __future__ import annotations
import os
-import socket
import pytest
@@ -40,9 +39,8 @@ _spec.loader.exec_module(fsdp2_perf)
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
D = 16
diff --git a/tests/test_fsdp2_qlora.py b/tests/test_fsdp2_qlora.py
index 79f13613c43..c60cf9f9557 100644
--- a/tests/test_fsdp2_qlora.py
+++ b/tests/test_fsdp2_qlora.py
@@ -9,7 +9,6 @@ bitsandbytes Linear4bit layers (CPU backend) across 2 gloo ranks with FSDP2's fu
import importlib.util
import os
-import socket
import sys
from pathlib import Path
@@ -217,9 +216,8 @@ def test_nonfloat_guard_restores_parameter_new():
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _worker(rank, world_size, port, case, queue):
diff --git a/tests/test_fsdp2_qlora_moe_gloo.py b/tests/test_fsdp2_qlora_moe_gloo.py
index 09780ab8588..ae259fa760f 100644
--- a/tests/test_fsdp2_qlora_moe_gloo.py
+++ b/tests/test_fsdp2_qlora_moe_gloo.py
@@ -14,7 +14,6 @@ GPU-free: real bitsandbytes 4-bit weights (CPU backend) sharded over 2 gloo rank
import importlib.util
import os
-import socket
import sys
from pathlib import Path
@@ -46,9 +45,8 @@ def _load_module():
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _stack_params4bit(shape, seed, module):
diff --git a/tests/test_fsdp2_save_gloo.py b/tests/test_fsdp2_save_gloo.py
index a22c479fe62..2255ca22be2 100644
--- a/tests/test_fsdp2_save_gloo.py
+++ b/tests/test_fsdp2_save_gloo.py
@@ -16,7 +16,6 @@ The modules are loaded from their files (importing unsloth needs a GPU).
import importlib.util
import os
-import socket
import sys
from pathlib import Path
@@ -48,9 +47,8 @@ N_LAYERS = 3
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
class _Block(torch.nn.Module):
diff --git a/tests/test_fsdp2_vision_gate_gloo.py b/tests/test_fsdp2_vision_gate_gloo.py
index cfcec2d91a9..64884f0c437 100644
--- a/tests/test_fsdp2_vision_gate_gloo.py
+++ b/tests/test_fsdp2_vision_gate_gloo.py
@@ -18,7 +18,6 @@ import importlib.util
import inspect
import json
import os
-import socket
import sys
import types
@@ -48,9 +47,8 @@ def _load_native():
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _config():
diff --git a/tests/test_fsdp2_wrap_gloo.py b/tests/test_fsdp2_wrap_gloo.py
index 3c0564d68c9..2b424db68b8 100644
--- a/tests/test_fsdp2_wrap_gloo.py
+++ b/tests/test_fsdp2_wrap_gloo.py
@@ -16,7 +16,6 @@ What Unsloth adds after accelerate shards the model (unsloth/models/fsdp2.py):
from __future__ import annotations
import os
-import socket
import types
import pytest
@@ -34,9 +33,8 @@ N_LAYERS = 3
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
def _spy_mlp(self, X):
diff --git a/tests/test_grpo_fsdp2_scoring_head.py b/tests/test_grpo_fsdp2_scoring_head.py
index 0461fdf1bfd..4a4f85578a0 100644
--- a/tests/test_grpo_fsdp2_scoring_head.py
+++ b/tests/test_grpo_fsdp2_scoring_head.py
@@ -14,7 +14,6 @@ import ast
import collections
import copy
import os
-import socket
import sys
import textwrap
from pathlib import Path
@@ -205,9 +204,8 @@ def _worker(rank, port, separate_head, setup_source, loop_source, errors):
def _free_port():
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
+ from tests._rendezvous_port import free_port
+ return free_port()
@pytest.mark.parametrize("separate_head", [False, True], ids = ["root_head", "separate_head"])
diff --git a/tests/test_rendezvous_port.py b/tests/test_rendezvous_port.py
new file mode 100644
index 00000000000..73f0df7bd21
--- /dev/null
+++ b/tests/test_rendezvous_port.py
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The distributed tests' MASTER_PORT comes from outside the ephemeral range, in a per-worker slice."""
+
+import socket
+
+from tests import _rendezvous_port as rp
+
+
+def test_the_port_is_below_every_default_ephemeral_range_and_in_this_workers_slice(monkeypatch):
+ for worker, base in (("gw0", 20000), ("gw2", 22000), ("gw11", 31000)):
+ monkeypatch.setenv("PYTEST_XDIST_WORKER", worker)
+ for _ in range(20):
+ port = rp.free_port()
+ assert base <= port < base + rp.SLICE < 32768, (worker, port)
+
+
+def test_no_xdist_worker_uses_the_first_slice(monkeypatch):
+ monkeypatch.delenv("PYTEST_XDIST_WORKER", raising = False)
+ assert 20000 <= rp.free_port() < 21000
+
+
+def test_a_taken_port_is_skipped(monkeypatch):
+ monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw5")
+ held = socket.socket()
+ try:
+ taken = next(p for p in range(25000, 26000) if _try_listen(held, p))
+ other = 25000 + (taken - 25000 + 1) % rp.SLICE
+ offsets = iter([taken - 25000, other - 25000])
+
+ class _Rng:
+ def randrange(self, n):
+ return next(offsets)
+
+ monkeypatch.setattr(rp.random, "Random", _Rng)
+ assert rp.free_port() == other
+ finally:
+ held.close()
+
+
+def _try_listen(sock, port):
+ try:
+ sock.bind(("127.0.0.1", port))
+ except OSError:
+ return False
+ sock.listen()
+ return True