Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fast_inference under torchrun: every rank takes the same vLLM-or-HF path (#15395)

Daniel Han5204566640dc
diff --git a/tests/test_fast_inference_torchrun_agreement.py b/tests/test_fast_inference_torchrun_agreement.py
new file mode 100644
index 00000000000..6d668dcd83d
--- /dev/null
+++ b/tests/test_fast_inference_torchrun_agreement.py
@@ -0,0 +1,200 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""fast_inference under torchrun: the vLLM-or-HF decision is the same on every rank.
+
+The capability gates read each rank's own card. On a mixed host one rank kept vLLM (an
+external_launcher engine in the job's group) while the other loaded HF and went on to the
+Trainer's collectives, and both waited out the group timeout.
+"""
+
+import ast
+import json
+import os
+import socket
+import subprocess
+import sys
+import textwrap
+import types
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+LOADER = ROOT / "unsloth" / "models" / "loader.py"
+_NAMES = (
+ "_agree_fast_inference_across_ranks",
+ "_drop_vllm_engine_kwargs",
+ "_vllm_needs_newer_gpu",
+ "_vllm_engine_device",
+)
+_ASSIGNS = ("_VLLM_ENGINE_KWARGS", "VLLM_NEEDS_SM80_MODEL_TYPES")
+
+
+def _extract_source():
+ tree = ast.parse(LOADER.read_text(encoding = "utf-8"))
+ wanted = [
+ node
+ for node in tree.body
+ if (isinstance(node, ast.FunctionDef) and node.name in _NAMES)
+ or (
+ isinstance(node, ast.Assign)
+ and any(getattr(t, "id", None) in _ASSIGNS for t in node.targets)
+ )
+ ]
+ found = {getattr(n, "name", None) for n in wanted}
+ missing = [n for n in _NAMES if n not in found]
+ assert not missing, f"loader.py lacks {missing}"
+ return ast.unparse(ast.Module(body = wanted, type_ignores = []))
+
+
+def _helpers(monkeypatch, agree):
+ zoo_vllm = types.ModuleType("unsloth_zoo.vllm_utils")
+ calls = []
+ if agree is not None:
+
+ def torchrun_ranks_agree(local_ok):
+ calls.append(local_ok)
+ return agree(local_ok)
+
+ zoo_vllm.torchrun_ranks_agree = torchrun_ranks_agree
+ monkeypatch.setitem(sys.modules, "unsloth_zoo", types.ModuleType("unsloth_zoo"))
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.vllm_utils", zoo_vllm)
+ namespace = {"DEVICE_TYPE": "cuda"}
+ exec(compile(_extract_source(), "loader.py", "exec"), namespace)
+ return namespace["_agree_fast_inference_across_ranks"], calls
+
+
+def test_a_peer_without_vllm_turns_it_off_here(monkeypatch, capsys):
+ agree, calls = _helpers(monkeypatch, lambda ok: False)
+ kwargs = {"max_num_batched_tokens": 4096, "token": "x"}
+ assert agree(True, True, kwargs) is False
+ assert kwargs == {"token": "x"}
+ assert calls == [True]
+ assert "another torchrun rank" in capsys.readouterr().out
+
+
+def test_a_rank_that_dropped_vllm_still_takes_part(monkeypatch):
+ agree, calls = _helpers(monkeypatch, lambda ok: False)
+ assert agree(True, False, {}) is False
+ assert calls == [False]
+
+
+def test_every_rank_keeping_vllm_keeps_it(monkeypatch):
+ agree, calls = _helpers(monkeypatch, lambda ok: ok)
+ kwargs = {"max_num_batched_tokens": 4096}
+ assert agree(True, True, kwargs) is True
+ assert kwargs == {"max_num_batched_tokens": 4096}
+
+
+def test_not_requested_never_communicates(monkeypatch):
+ agree, calls = _helpers(
+ monkeypatch, lambda ok: pytest.fail("no collective without fast_inference")
+ )
+ assert agree(False, False, {}) is False
+ assert calls == []
+
+
+def test_older_zoo_keeps_the_local_answer(monkeypatch):
+ agree, _ = _helpers(monkeypatch, None)
+ assert agree(True, True, {}) is True
+ assert agree(True, False, {}) is False
+
+
+def _body(source, start, end):
+ i = source.index(start)
+ return source[i : source.index(end, i)]
+
+
+def test_agreement_follows_the_last_per_card_gate():
+ src = LOADER.read_text(encoding = "utf-8")
+ # Prefix only: the formatter wraps the call's arguments onto their own lines.
+ call = "fast_inference = _agree_fast_inference_across_ranks("
+ flm = _body(src, "class FastLanguageModel(", "class FastModel(")
+ i_req = flm.index("fast_inference_requested = fast_inference")
+ i_cc = flm.index("torch.cuda.get_device_capability()[0] < 7")
+ i_gb10 = flm.index('"NVIDIA GB10"')
+ i_call = flm.index(call)
+ assert (
+ i_req
+ < i_cc
+ < i_gb10
+ < i_call
+ < flm.index("raise_if_fast_inference_under_fsdp2(fast_inference)")
+ )
+ fm = src[src.index("class FastModel(") :]
+ i_req = fm.index("fast_inference_requested = fast_inference")
+ i_cc = fm.index("torch.cuda.get_device_capability()[0] < 7")
+ i_old = fm.index("_vllm_old_gpu = _vllm_needs_newer_gpu(model_types)")
+ i_call = fm.index(call)
+ assert i_req < i_cc < i_old < i_call
+ # Nothing turns vLLM off per rank between the gate and the agreement.
+ assert (
+ "fast_inference = False"
+ not in fm[fm.index("_drop_vllm_engine_kwargs(kwargs)", i_old) + 1 : i_call]
+ )
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def test_mixed_gpu_torchrun_ranks_take_the_same_path(tmp_path):
+ """Two gloo ranks, rank 0 on an sm80 card and rank 1 on a T4, loading Gemma 4: one decision."""
+ try:
+ from unsloth_zoo.vllm_utils import torchrun_ranks_agree # noqa: F401
+ except Exception as error:
+ pytest.skip(f"unsloth_zoo without torchrun_ranks_agree: {error}")
+ (tmp_path / "loader_helpers.py").write_text(_extract_source())
+ script = tmp_path / "worker.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import json, os, sys
+ import torch
+ sys.path.insert(0, sys.argv[1])
+ rank = int(os.environ["RANK"])
+ torch.cuda.get_device_capability = lambda device = None: (8, 0) if rank == 0 else (7, 5)
+ import loader_helpers as h
+ h.DEVICE_TYPE = "cuda"
+ kwargs = {"max_num_batched_tokens": 4096}
+ local = h._vllm_needs_newer_gpu(["gemma4"]) is None
+ final = h._agree_fast_inference_across_ranks(True, local, kwargs)
+ import torch.distributed as dist
+ json.dump({"local": local, "final": final, "kwargs": kwargs, "world": dist.get_world_size()},
+ open(os.path.join(sys.argv[1], "r%d.json" % rank), "w"))
+ dist.destroy_process_group()
+ """
+ )
+ )
+ env = dict(
+ os.environ,
+ CUDA_VISIBLE_DEVICES = "",
+ UNSLOTH_ALLOW_CPU = "1",
+ UNSLOTH_DDP_TIMEOUT = "30",
+ UNSLOTH_IS_PRESENT = "1",
+ )
+ cmd = [
+ sys.executable,
+ "-m",
+ "torch.distributed.run",
+ "--nnodes",
+ "1",
+ "--nproc-per-node",
+ "2",
+ "--master-addr",
+ "127.0.0.1",
+ "--master-port",
+ str(_free_port()),
+ str(script),
+ str(tmp_path),
+ ]
+ r = subprocess.run(cmd, env = env, capture_output = True, text = True, timeout = 180, cwd = tmp_path)
+ assert r.returncode == 0, r.stdout[-3000:] + r.stderr[-3000:]
+ out = [json.loads((tmp_path / f"r{rank}.json").read_text()) for rank in range(2)]
+ assert [o["local"] for o in out] == [True, False] # the per-card gates disagree
+ assert [o["final"] for o in out] == [False, False]
+ assert out[0]["kwargs"] == {} # rank 1's own gate drops them in from_pretrained
+ assert all(o["world"] == 2 for o in out)
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 17194d619a8..45924eee206 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -1195,6 +1195,27 @@ def _drop_vllm_engine_kwargs(kwargs):
print(f"Unsloth: Ignoring vLLM-only arguments {dropped} since vLLM is not used.")
+def _agree_fast_inference_across_ranks(requested, fast_inference, kwargs):
+ """Under torchrun keep vLLM only if every rank kept it, so one rank never waits alone in vLLM's collectives.
+
+ The capability gates read each rank's own card: on a mixed host (an A100 and a T4) one rank would build an external_launcher engine while the other loads HF and starts training. Outside torchrun, or with an older unsloth_zoo, the local answer stands."""
+ if not requested:
+ return fast_inference
+ try:
+ from unsloth_zoo.vllm_utils import torchrun_ranks_agree
+ except Exception:
+ return fast_inference
+ # Every rank that asked for vLLM takes part, including the ones that already dropped it.
+ if torchrun_ranks_agree(fast_inference) or not fast_inference:
+ return fast_inference
+ print(
+ "Unsloth: fast_inference (vLLM) is off on another torchrun rank (such as an older GPU) - "
+ "every rank will switch to Unsloth inference!"
+ )
+ _drop_vllm_engine_kwargs(kwargs)
+ return False
+
+
class FastLanguageModel(FastLlamaModel):
reap_prune = staticmethod(_reap_prune)
@@ -1375,6 +1396,7 @@ class FastLanguageModel(FastLlamaModel):
or dtype == torch.float32
)
+ fast_inference_requested = fast_inference
# Same fallback as FastModel, before requiring vLLM: it cannot run below compute capability 7.
if fast_inference and DEVICE_TYPE == "cuda" and torch.cuda.get_device_capability()[0] < 7:
print("Unsloth: vLLM does not work on older GPUs - will switch to Unsloth inference!")
@@ -1393,6 +1415,9 @@ class FastLanguageModel(FastLlamaModel):
fast_inference = False
_drop_vllm_engine_kwargs(kwargs)
break
+ fast_inference = _agree_fast_inference_across_ranks(
+ fast_inference_requested, fast_inference, kwargs
+ )
raise_if_fast_inference_under_fsdp2(fast_inference)
# bitsandbytes unusable (absent, or unstable on some AMD stacks). A capability check, so not gated on use_exact_model_name, which only suppresses repo-name remapping.
@@ -2372,6 +2397,7 @@ class FastModel(FastBaseModel):
# One whole model per rank; sharding one across the ranks' GPUs too would have every rank fighting for the same cards.
device_map = distributed_device_map
+ fast_inference_requested = fast_inference
# Same fallback as FastLanguageModel, before requiring vLLM: zoo's vLLM loader raises on compute capability < 7.
if fast_inference and DEVICE_TYPE == "cuda" and torch.cuda.get_device_capability()[0] < 7:
print("Unsloth: vLLM does not work on older GPUs - will switch to Unsloth inference!")
@@ -2674,6 +2700,10 @@ class FastModel(FastBaseModel):
)
fast_inference = False
_drop_vllm_engine_kwargs(kwargs)
+ # After the last per-card gate: under torchrun every rank takes the same vLLM-or-HF path.
+ fast_inference = _agree_fast_inference_across_ranks(
+ fast_inference_requested, fast_inference, kwargs
+ )
# Text-diffusion models (DiffusionGemma) take a transformers-only slow path: a custom block-diffusion generate over a novel backbone, so Unsloth's autoregressive kernel/compile patching is skipped and the unmodified HF model is loaded, keeping 4bit/8bit and PEFT LoRA.
# Remote-code checkpoints can report their backbone's type (Nemotron-Labs-Diffusion says "nemotron"), so also match the raw config.