Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Pin every rank of a distributed launch to its own device, not just quantized ones (#11129)
Daniel Hanfa4edb500504
diff --git a/tests/test_optimized_precision_conflicts.py b/tests/test_optimized_precision_conflicts.pyindex e55f248eda3..629fb93ad4e 100644--- a/tests/test_optimized_precision_conflicts.py+++ b/tests/test_optimized_precision_conflicts.py@@ -54,6 +54,7 @@ def loader():_requested_float32 = lambda dtype: False,hf_login = lambda token: token,requested_device_map = lambda device: device,+ is_automatic_device_map = lambda device: isinstance(device, str),prepare_device_map = lambda: ("sequential", False),ALLOW_BITSANDBYTES = True,ALLOW_PREQUANTIZED_MODELS = True,diff --git a/tests/test_unsloth_device_map_optin.py b/tests/test_unsloth_device_map_optin.pyindex a51d8f42616..2ed771d4039 100644--- a/tests/test_unsloth_device_map_optin.py+++ b/tests/test_unsloth_device_map_optin.py@@ -320,9 +320,9 @@ def test_a_distributed_launch_never_gets_an_intra_model_split():"""torchrun/DDP/FSDP already put one whole model per rank; splitting a model across thecards on top of that puts every rank on every card, which OOMs rather than fits.- prepare_device_map() in loader.py converts the string to a rank-local dict first, but- only when the load is quantized, so a 16-bit distributed run still arrives here holding- "unsloth". Hence the gate lives here too.+ prepare_device_map() in loader.py converts the string to a rank-local dict first, at+ every precision since #3459. The gate still lives here too: loader.py is not the only+ caller, and a rank-local dict is not a string, so the two never disagree."""ns = _load(distributed = True, planner = lambda *a, **k: pytest.fail("planned inside a distributed launch")@@ -984,3 +984,313 @@ def test_an_auto_class_still_plans():idx = vision.index("an explicit model class has no auto mapping")guard = vision[vision.rindex("if (", 0, idx) : idx]assert "_model_mapping" in guard, "the veto is not keyed on the class being concrete"+++LOADER = os.path.join(MODELS, "loader.py")+_QUANT_NAMES = ("is_quantized", "load_in_4bit", "load_in_8bit", "load_in_fp8")+++def _prepare_device_map_guards():+ """Every `prepare_device_map()` call in loader.py, with the `if` tests enclosing it."""+ source = open(LOADER, encoding = "utf-8").read()+ tree = ast.parse(source)+ found = []++ def walk(node, guards):+ for child in ast.iter_child_nodes(node):+ if (+ isinstance(child, ast.Call)+ and getattr(child.func, "id", None) == "prepare_device_map"+ ):+ found.append((child.lineno, list(guards)))+ if isinstance(child, ast.If):+ walk_body(child.body, guards + [ast.unparse(child.test)])+ walk_body(child.orelse, guards)+ else:+ walk(child, guards)++ def walk_body(body, guards):+ for stmt in body:+ walk(stmt, guards)++ walk_body(tree.body, [])+ return found+++def test_every_rank_of_a_16bit_distributed_launch_still_gets_its_own_device():+ """A distributed load must be pinned to the rank's card whatever its precision.++ `prepare_device_map()` rewrites the string device_map into `{"": "cuda:<local_rank>"}`.+ Gating that on the load being quantized left a 16-bit `accelerate launch` holding a+ string, which `resolve_unsloth_device_map` turns into "sequential" -- and sequential+ dispatch fills cuda:0 first, on every rank. Measured on 2 GPUs before the gate came+ off: rank 0 and rank 1 both reported `param_devices=['cuda:0']` with 4245 MiB on card+ 0 and 0 MiB on card 1, for both FastLanguageModel and FastVisionModel. That is+ unsloth#3459's "100% GPU utilisation, 0 VRAM, then it suddenly loads": the ranks are+ queueing for one card.++ Static, because the failure needs two real GPUs and a launcher. Re-adding a+ quantization term to the guard turns this red.+ """+ calls = _prepare_device_map_guards()+ assert calls, "loader.py no longer pins a distributed rank to its own device"+ for lineno, guards in calls:+ for guard in guards:+ offending = [name for name in _QUANT_NAMES if name in guard]+ assert not offending, (+ f"loader.py:{lineno}: prepare_device_map() is gated on {offending} "+ f"({guard!r}), so a 16-bit distributed load keeps a string device_map "+ f"and every rank dispatches onto cuda:0"+ )+++# --- The rank a distributed launch is pinned to, and the placements it may pin ---+++class _FakeDistributed:+ """torch.distributed as a rank sees it. `rank` is the GLOBAL rank, as get_rank() returns."""++ def __init__(+ self,+ rank = None,+ world_size = None,+ ):+ self._rank = rank+ self._world_size = world_size++ def is_available(self):+ return True++ def is_initialized(self):+ return self._rank is not None++ def get_rank(self):+ return self._rank++ def get_world_size(self):+ return self._world_size+++class _FakeAccelerator(_FakeCuda):+ def __init__(self, count):+ super().__init__(count)+ self.pinned = []++ def set_device(self, index):+ if index >= self._count:+ raise RuntimeError(f"CUDA error: invalid device ordinal ({index} of {self._count})")+ self.pinned.append(index)+++_DISTRIBUTED_NAMES = (+ "_get_env_int",+ "_infer_distributed_ranks",+ "_visible_device_count",+ "_infer_local_rank",+ "is_distributed",+ "prepare_device_map",+ "is_automatic_device_map",+ "requested_device_map",+)+_DISTRIBUTED_CONSTANTS = (+ "LOCAL_RANK_KEYS",+ "WORLD_SIZE_KEYS",+ "LOCAL_RANK_ONLY_KEYS",+ "UNSLOTH_DEVICE_MAP",+ "UNSLOTH_BALANCED_DEVICE_MAP",+ "DEFAULT_DEVICE_MAP",+ "TRANSFORMERS_PLACEMENT_STRATEGIES",+ "AUTOMATIC_DEVICE_MAPS",+)+++def _load_distributed(+ monkeypatch,+ *,+ device_type = "cuda",+ devices = 8,+ rank = None,+ world_size = None,+ env = None,+):+ """The placement helpers over a fabricated launcher environment and accelerator."""+ for key in ("LOCAL_RANK", "RANK", "WORLD_SIZE"):+ monkeypatch.delenv(key, raising = False)+ for key, value in (env or {}).items():+ monkeypatch.setenv(key, value)++ accelerator = _FakeAccelerator(devices)+ torch_stub = types.SimpleNamespace(+ distributed = _FakeDistributed(rank, world_size),+ cuda = accelerator,+ xpu = accelerator,+ npu = accelerator,+ )+ ns = {"os": os, "torch": torch_stub, "DEVICE_TYPE_TORCH": device_type}+ for node in ast.parse(_SRC).body:+ if isinstance(node, ast.FunctionDef) and node.name in _DISTRIBUTED_NAMES:+ exec(ast.get_source_segment(_SRC, node), ns)+ elif isinstance(node, ast.ClassDef) and node.name == "_DefaultDeviceMap":+ exec(ast.get_source_segment(_SRC, node), ns)+ elif (+ isinstance(node, ast.Assign)+ and getattr(node.targets[0], "id", None) in _DISTRIBUTED_CONSTANTS+ ):+ exec(ast.get_source_segment(_SRC, node), ns)+ return ns, accelerator+++def _pinned_device_map(ns):+ device_map, is_dist = ns["prepare_device_map"]()+ return device_map, is_dist+++def test_the_second_node_of_a_multi_node_job_pins_to_its_own_card():+ """Global rank 8 of a 2 x 8 job is local rank 0, not card 8.++ `torch.distributed.get_rank()` is documented as "a unique identifier assigned to each+ process within a distributed process group ... 0 to world_size" -- global, across nodes.+ Using it as a device index made every rank on the second node ask for cuda:8 on a host+ with eight cards; `set_device` raises "invalid device ordinal", the except swallows it,+ and the load then fails on the returned map. torchrun's LOCAL_RANK is the node-local one.+ """+ monkeypatch = pytest.MonkeyPatch()+ try:+ ns, accelerator = _load_distributed(+ monkeypatch,+ devices = 8,+ rank = 8,+ world_size = 16,+ env = {"LOCAL_RANK": "0", "RANK": "8", "WORLD_SIZE": "16"},+ )+ device_map, is_dist = _pinned_device_map(ns)+ assert is_dist+ assert device_map == {"": "cuda:0"}, device_map+ assert accelerator.pinned == [0], accelerator.pinned+ finally:+ monkeypatch.undo()+++def test_a_launcher_that_sets_no_local_rank_still_names_a_card_that_exists():+ """No LOCAL_RANK: the global rank is folded onto the visible cards rather than overflowing."""+ monkeypatch = pytest.MonkeyPatch()+ try:+ ns, accelerator = _load_distributed(+ monkeypatch,+ devices = 8,+ rank = 9,+ world_size = 16,+ )+ device_map, is_dist = _pinned_device_map(ns)+ assert is_dist+ assert device_map == {"": "cuda:1"}, device_map+ assert accelerator.pinned == [1], accelerator.pinned+ finally:+ monkeypatch.undo()+++def test_a_per_rank_cuda_visible_devices_pins_to_the_one_card_it_can_see():+ """LOCAL_RANK=3 with a single visible card is card 0 -- cuda:3 does not exist here."""+ monkeypatch = pytest.MonkeyPatch()+ try:+ ns, accelerator = _load_distributed(+ monkeypatch,+ devices = 1,+ rank = 3,+ world_size = 4,+ env = {"LOCAL_RANK": "3", "RANK": "3", "WORLD_SIZE": "4"},+ )+ device_map, _ = _pinned_device_map(ns)+ assert device_map == {"": "cuda:0"}, device_map+ assert accelerator.pinned == [0], accelerator.pinned+ finally:+ monkeypatch.undo()++[email protected]("local_rank", [0, 1])+def test_a_single_node_launch_keeps_the_rank_it_already_had(local_rank):+ """The case the PR exists for must not move: one node, one card per rank."""+ monkeypatch = pytest.MonkeyPatch()+ try:+ ns, accelerator = _load_distributed(+ monkeypatch,+ devices = 2,+ rank = local_rank,+ world_size = 2,+ env = {"LOCAL_RANK": str(local_rank), "RANK": str(local_rank), "WORLD_SIZE": "2"},+ )+ device_map, is_dist = _pinned_device_map(ns)+ assert is_dist+ assert device_map == {"": f"cuda:{local_rank}"}+ assert accelerator.pinned == [local_rank]+ finally:+ monkeypatch.undo()+++def test_a_single_process_run_is_never_pinned():+ """No launcher, no distribution: one GPU and CPU-only runs keep their own placement."""+ monkeypatch = pytest.MonkeyPatch()+ try:+ ns, accelerator = _load_distributed(monkeypatch, devices = 1)+ assert _pinned_device_map(ns) == (None, False)+ assert accelerator.pinned == []+ finally:+ monkeypatch.undo()+++ "device_map",+ ["auto", "balanced", "balanced_low_0", "sequential", "unsloth", "unsloth_balanced", "cuda"],+)+def test_an_unchosen_placement_is_the_one_a_rank_may_pin(device_map):+ """Every strategy name spreads one model over all the cards, which is the #3459 failure+ on every rank. A bare "cuda" names the type and leaves the index to us."""+ monkeypatch = pytest.MonkeyPatch()+ try:+ ns, _ = _load_distributed(monkeypatch)+ is_automatic = ns.get("is_automatic_device_map")+ assert is_automatic is not None, "loader_utils.py has no is_automatic_device_map"+ assert is_automatic(device_map) is True, device_map+ assert is_automatic(ns["DEFAULT_DEVICE_MAP"]) is True+ finally:+ monkeypatch.undo()+++ "device_map", ["cpu", "cuda:2", "mps", "xpu:1", "meta", {"": "cpu"}, {"": 0}, 0, None]+)+def test_a_device_the_caller_named_survives_a_distributed_launch(device_map):+ """`device_map = "cpu"` or `"cuda:2"` is a placement someone chose, and transformers+ reads it as one: `from_pretrained` turns every string outside+ ["auto", "balanced", "balanced_low_0", "sequential"] into `{"": torch.device(value)}`.+ Replacing it with the rank's card loads the model on hardware the caller deliberately+ avoided -- an unexpected GPU, or an OOM on a card they were keeping free.+ """+ monkeypatch = pytest.MonkeyPatch()+ try:+ ns, _ = _load_distributed(monkeypatch)+ is_automatic = ns.get("is_automatic_device_map")+ assert is_automatic is not None, "loader_utils.py has no is_automatic_device_map"+ assert is_automatic(device_map) is False, device_map+ finally:+ monkeypatch.undo()+++def test_the_loader_pins_on_the_placement_being_unchosen_not_on_it_being_a_string():+ """The guard at both `prepare_device_map()` call sites, read off loader.py.++ `isinstance(device_map, str)` is true of "cpu" and "cuda:2" as well, so it was the+ broadened condition that clobbered them.+ """+ calls = _prepare_device_map_guards()+ assert calls, "loader.py no longer pins a distributed rank to its own device"+ for lineno, guards in calls:+ assert any("is_automatic_device_map(device_map)" in guard for guard in guards), (+ f"loader.py:{lineno}: prepare_device_map() is guarded by {guards}, which does not "+ f"distinguish a placement nobody chose from a device the caller named"+ )+ for guard in guards:+ assert "isinstance(device_map, str)" not in guard, (+ f"loader.py:{lineno}: an explicit device_map='cpu' is a str too, so this guard "+ f"moves the load onto the rank's GPU"+ )diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 759316a8578..8d1a57ed17c 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -40,6 +40,7 @@ from .loader_utils import (_offline_quantize_to_fp8,_tag_model_with_fp8_torchao_config,get_model_name,+ is_automatic_device_map,prepare_device_map,requested_device_map,_offline_aware_load,@@ -441,10 +442,9 @@ class FastLanguageModel(FastLlamaModel):if isinstance(bnb_compute_dtype, torch.dtype):dtype = bnb_compute_dtype- # Distributed-safe placement for quantized models: under torchrun each rank must load on its own device, else Accelerate raises device relocation errors on quantized weights.- is_quantized = load_in_4bit or load_in_8bit or load_in_fp8+ # Distributed-safe placement: under torchrun / accelerate launch each rank must load on its own device. Quantized weights make it mandatory, since Accelerate raises device relocation errors on them, but a 16-bit load needs it just as much: an unchosen placement ends up dispatching to cuda:0, so all the ranks land on card 0 and contend for it while the rest of the cards stay empty (#3459). A device the caller named is left alone.device_map = requested_device_map(device_map)- if is_quantized and isinstance(device_map, str):+ if is_automatic_device_map(device_map):distributed_device_map, is_dist = prepare_device_map()if is_dist:# One whole model per rank; sharding one across the ranks' GPUs too would have every rank fighting for the same cards.@@ -1300,10 +1300,9 @@ class FastModel(FastBaseModel):if qat_scheme == "phone-deployment":qat_scheme = "int8-int4"- # Distributed-safe placement for quantized models: under torchrun each rank must load on its own device, else Accelerate raises device relocation errors.- is_quantized = load_in_4bit or load_in_8bit or load_in_fp8+ # Distributed-safe placement: under torchrun / accelerate launch each rank must load on its own device. Quantized weights make it mandatory, since Accelerate raises device relocation errors on them, but a 16-bit load needs it just as much: an unchosen placement ends up dispatching to cuda:0, so all the ranks land on card 0 and contend for it while the rest of the cards stay empty (#3459). A device the caller named is left alone.device_map = requested_device_map(device_map)- if is_quantized and isinstance(device_map, str):+ if is_automatic_device_map(device_map):distributed_device_map, is_dist = prepare_device_map()if is_dist:# One whole model per rank; sharding one across the ranks' GPUs too would have every rank fighting for the same cards.diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 45db9ef403c..87868b4046e 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -48,6 +48,8 @@ SUPPORTS_FOURBIT = transformers_version >= Version("4.37")LOCAL_RANK_KEYS = ("LOCAL_RANK", "RANK")WORLD_SIZE_KEYS = ("WORLD_SIZE",)+# Only the node-local one. torchrun documents RANK as "The global rank" and LOCAL_RANK as "The local rank", and torch.distributed.get_rank() is global too, so neither of those may index a device.+LOCAL_RANK_ONLY_KEYS = ("LOCAL_RANK",)BAD_MAPPINGS = {"unsloth/Qwen3-32B-unsloth-bnb-4bit".lower(): "unsloth/Qwen3-32B-bnb-4bit".lower(), # 32B dynamic quant is way too big@@ -92,13 +94,45 @@ def is_distributed():return (world_size or 1) > 1 or (rank is not None and rank > 0)+def _visible_device_count():+ """Devices this process can actually address, 0 when the backend has none."""+ try:+ module = getattr(torch, DEVICE_TYPE_TORCH, None)+ counter = getattr(module, "device_count", None)+ return int(counter()) if callable(counter) else 0+ except Exception:+ return 0+++def _infer_local_rank(rank):+ """The device index this rank owns *on this host*.++ `rank` is global: `torch.distributed.get_rank()` is "a unique identifier assigned to each+ process within a distributed process group ... 0 to world_size", and torchrun's RANK is+ documented as the global rank. Indexing a device with it names cuda:8 on the second node of+ a 2 x 8 job, which is an invalid ordinal, so read LOCAL_RANK first -- torchrun, accelerate+ launch and deepspeed all set it. The modulo is the last resort for a launcher that sets+ neither, and for a per-rank CUDA_VISIBLE_DEVICES where only one card is visible and the+ local rank still counts from the launcher's point of view.+ """+ local_rank = _get_env_int(LOCAL_RANK_ONLY_KEYS)+ if local_rank is None:+ local_rank = 0 if rank is None else rank+ if local_rank < 0:+ return 0+ count = _visible_device_count()+ if count > 0 and local_rank >= count:+ local_rank = local_rank % count+ return local_rank++def prepare_device_map():rank, world_size = _infer_distributed_ranks()distributed = (world_size or 1) > 1 or (rank is not None and rank > 0)if not distributed:return None, False- local_rank = 0 if rank is None else rank+ local_rank = _infer_local_rank(rank)device_map = {"": f"{DEVICE_TYPE_TORCH}:{local_rank}"}try:if DEVICE_TYPE_TORCH == "cuda":@@ -173,6 +207,29 @@ def requested_device_map(device_map):return device_map+# transformers routes exactly these four through infer_auto_device_map; every other string becomes `{"": torch.device(value)}` (modeling_utils.from_pretrained), i.e. a device the caller named.+TRANSFORMERS_PLACEMENT_STRATEGIES = frozenset({"auto", "balanced", "balanced_low_0", "sequential"})++AUTOMATIC_DEVICE_MAPS = TRANSFORMERS_PLACEMENT_STRATEGIES | {+ UNSLOTH_DEVICE_MAP,+ UNSLOTH_BALANCED_DEVICE_MAP,+}+++def is_automatic_device_map(device_map):+ """True when the placement is still ours to pick, so a distributed launch may pin it to the rank's own card.++ A strategy name spreads one model over whatever devices it finds, which is exactly what every rank must not do (#3459). A device the caller named -- "cpu", "cuda:2", "mps" -- is a choice, and rewriting it loads the model on hardware they deliberately avoided. A bare accelerator type is the in-between: they named the type, never the index, so filling in the rank's index honours it.+ """+ if isinstance(device_map, _DefaultDeviceMap):+ return True+ if not isinstance(device_map, str):+ return False+ if device_map in AUTOMATIC_DEVICE_MAPS:+ return True+ return DEVICE_TYPE_TORCH != "cpu" and device_map == DEVICE_TYPE_TORCH++def planner_quantization_kwargs(load_in_4bit = False,load_in_8bit = False,