Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Train with DeepSpeed ZeRO-2 and ZeRO-3 (#13587)
Daniel Hane700e03197b0
diff --git a/tests/test_accelerator_distributed_type_patch.py b/tests/test_accelerator_distributed_type_patch.pyindex e947cbedba3..a51ba6bece9 100644--- a/tests/test_accelerator_distributed_type_patch.py+++ b/tests/test_accelerator_distributed_type_patch.py@@ -15,15 +15,16 @@# along with this program. If not, see <https://www.gnu.org/licenses/>."""`_utils.py` must patch `Accelerator.distributed_type` with a property, not a barefunction: a function binds as a method and inverts accelerate's `!= NO` device_map-guards on a single device (#10016). `import unsloth` needs CUDA or XPU, so these run-the statement from source in a fresh interpreter, which also catches a revert.+guards on a single device (#10016). It must still read as DEEPSPEED under DeepSpeed, or+accelerate turns on a float16 autocast over a bfloat16 model (#225). `import unsloth` needs+CUDA or XPU, so these run the statements from source in a fresh interpreter, which also+catches a revert."""import astimport pathlibimport subprocessimport sys-import textwrapimport pytest@@ -47,6 +48,43 @@ def _patch_statement() -> ast.Assign:return matches[0]+def _getter_source() -> str:+ """The function the property wraps, so the fresh interpreter can define it first."""+ tree = ast.parse(_UTILS.read_text(encoding = "utf-8"))+ name = ast.unparse(_patch_statement().value.args[0])+ defs = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == name]+ assert len(defs) == 1, f"expected one {name}, found {len(defs)}"+ return ast.unparse(defs[0])+++def _read_distributed_type(state_type):+ """repr of the patched attribute and its `== NO` / `!= NO` on an Accelerator whose state says `state_type`."""+ script = "\n".join(+ [+ "import types",+ "import accelerate.accelerator",+ "from accelerate.utils.dataclasses import DistributedType",+ _getter_source(),+ ast.unparse(_patch_statement()),+ # Read it off an instance without standing up a real distributed backend.+ "accelerator = accelerate.accelerator.Accelerator.__new__(accelerate.accelerator.Accelerator)",+ *(+ []+ if state_type is None+ else [+ f"accelerator.state = types.SimpleNamespace(distributed_type = DistributedType.{state_type})"+ ]+ ),+ "value = accelerator.distributed_type",+ "print(repr(value))",+ "print(value == DistributedType.NO, value != DistributedType.NO)",+ ]+ )+ result = subprocess.run([sys.executable, "-c", script], capture_output = True, text = True)+ assert result.returncode == 0, result.stderr+ return result.stdout.strip().splitlines()[-2:]++def test_distributed_type_is_replaced_with_a_property():value = _patch_statement().valueassert isinstance(value, ast.Call) and ast.unparse(value.func) == "property", (@@ -57,26 +95,17 @@ def test_distributed_type_is_replaced_with_a_property():def test_patched_distributed_type_reads_as_no():"""Running the real assignment must leave the attribute reading as NO."""- statement = ast.unparse(_patch_statement())- script = textwrap.dedent(- f"""- import accelerate.accelerator- from accelerate.utils.dataclasses import DistributedType-- {statement}-- # Read it off an instance without standing up a real distributed backend.- accelerator = accelerate.accelerator.Accelerator.__new__(- accelerate.accelerator.Accelerator- )- value = accelerator.distributed_type- print(repr(value))- print(value == DistributedType.NO, value != DistributedType.NO)- """- )- result = subprocess.run([sys.executable, "-c", script], capture_output = True, text = True)- assert result.returncode == 0, result.stderr- value, comparisons = result.stdout.strip().splitlines()[-2:]- assert value == repr(accelerate.utils.dataclasses.DistributedType.NO), value- # The second is what accelerate's device_map guards actually test.- assert comparisons == "True False", comparisons+ DistributedType = accelerate.utils.dataclasses.DistributedType+ for state_type in (None, "NO", "MULTI_GPU", "FSDP"):+ value, comparisons = _read_distributed_type(state_type)+ assert value == repr(DistributedType.NO), (state_type, value)+ # The second is what accelerate's device_map guards actually test.+ assert comparisons == "True False", (state_type, comparisons)+++def test_patched_distributed_type_keeps_deepspeed():+ """Under DeepSpeed, NO enables native AMP and get_mixed_precision_context_manager's+ DeepSpeed fallthrough is a dtype-less, i.e. float16, autocast."""+ value, comparisons = _read_distributed_type("DEEPSPEED")+ assert value == repr(accelerate.utils.dataclasses.DistributedType.DEEPSPEED), value+ assert comparisons == "False True", comparisonsdiff --git a/tests/test_deepspeed_zero3.py b/tests/test_deepspeed_zero3.pynew file mode 100644index 00000000000..71e558d2bfc--- /dev/null+++ b/tests/test_deepspeed_zero3.py@@ -0,0 +1,53 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.+"""DeepSpeed ZeRO-3 (#225): the load must not pass a device_map transformers refuses."""++import pytest++transformers = pytest.importorskip("transformers")++from unsloth.models import loader_utils+++def zero3(monkeypatch):+ def set_enabled(enabled):+ monkeypatch.setattr(+ transformers.integrations, "is_deepspeed_zero3_enabled", lambda: enabled+ )++ return set_enabled+++def test_zero3_drops_the_placement_unsloth_picked(zero3):+ zero3(True)+ for device_map in (loader_utils.DEFAULT_DEVICE_MAP, "sequential", "auto", "cuda"):+ assert loader_utils.deepspeed_zero3_device_map(device_map) is None, device_map+++ "device_map", [{"model.layers.0": 0, "lm_head": 1}, {"": "cuda:1"}, "cuda:1", None]+)+def test_zero3_keeps_a_map_the_caller_chose(zero3, device_map):+ """transformers rejects these under ZeRO-3; silently dropping one would move the run to another card."""+ zero3(True)+ assert loader_utils.deepspeed_zero3_device_map(device_map) is device_map+++def test_without_zero3_the_device_map_is_untouched(zero3):+ zero3(False)+ for device_map in (loader_utils.DEFAULT_DEVICE_MAP, "sequential", {"": "cuda:1"}, None):+ assert loader_utils.deepspeed_zero3_device_map(device_map) is device_mapdiff --git a/tests/test_optimized_precision_conflicts.py b/tests/test_optimized_precision_conflicts.pyindex 0f747cc1038..68e5c17d924 100644--- a/tests/test_optimized_precision_conflicts.py+++ b/tests/test_optimized_precision_conflicts.py@@ -66,6 +66,7 @@ def loader():_requested_float32 = lambda dtype: False,hf_login = lambda token: token,requested_device_map = lambda device: device,+ deepspeed_zero3_device_map = lambda device: device,is_automatic_device_map = lambda device: isinstance(device, str),prepare_device_map = lambda: ("sequential", False),ALLOW_BITSANDBYTES = True,diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex be4c6a64df9..2029a5e0f78 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -4356,8 +4356,17 @@ if DEVICE_COUNT == 1 and int(os.environ.get("WORLD_SIZE", "1")) <= 1:import accelerate.stateaccelerate.state.PartialState._prepare_backend = _prepare_backend++ # DeepSpeed keeps its type even on one device: reporting NO turns on accelerate's native AMP, whose+ # dtype-less autocast under DeepSpeed is float16 and breaks a bfloat16 model (#225).+ def _single_device_distributed_type(self):+ real = getattr(self.__dict__.get("state"), "distributed_type", None)+ return (+ DistributedType.DEEPSPEED if real == DistributedType.DEEPSPEED else DistributedType.NO+ )+# Must be a property: a bare function binds as a method, inverting every `!= DistributedType.NO` guard in accelerate (#10016).- accelerate.accelerator.Accelerator.distributed_type = property(lambda self: DistributedType.NO)+ accelerate.accelerator.Accelerator.distributed_type = property(_single_device_distributed_type)_PER_LAYER_DEVICE_MISSING = object()diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 995476ce1fe..6dffa18f3f9 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -45,6 +45,7 @@ from .lora_init import adapter_used_fast_pissa, fast_lora_init, record_fast_pissfrom ..tokenizers_v1 import tokenizers_v1_on_returnfrom .loader_utils import (DEFAULT_DEVICE_MAP,+ deepspeed_zero3_device_map,OFFLOAD_EMBEDDING_AUTO,_exclude_rope_inv_freq_from_ddp,_get_fp8_mode_and_check_settings,@@ -1114,7 +1115,7 @@ class FastLanguageModel(FastLlamaModel):dtype = bnb_compute_dtype# Distributed-safe placement: under torchrun / accelerate launch each rank must load on its own device. Quantized weights make it mandatory, since Accelerate raises device relocation errors on them, but a 16-bit load needs it just as much: an unchosen placement ends up dispatching to cuda:0, so all the ranks land on card 0 and contend for it while the rest of the cards stay empty (#3459). A device the caller named is left alone.- device_map = requested_device_map(device_map)+ device_map = deepspeed_zero3_device_map(requested_device_map(device_map))if is_automatic_device_map(device_map):distributed_device_map, is_dist = prepare_device_map()if is_dist:@@ -2096,7 +2097,7 @@ class FastModel(FastBaseModel):qat_scheme = "int8-int4"# Distributed-safe placement: under torchrun / accelerate launch each rank must load on its own device. Quantized weights make it mandatory, since Accelerate raises device relocation errors on them, but a 16-bit load needs it just as much: an unchosen placement ends up dispatching to cuda:0, so all the ranks land on card 0 and contend for it while the rest of the cards stay empty (#3459). A device the caller named is left alone.- device_map = requested_device_map(device_map)+ device_map = deepspeed_zero3_device_map(requested_device_map(device_map))if is_automatic_device_map(device_map):distributed_device_map, is_dist = prepare_device_map()if is_dist:diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex e89486c203a..8f40869a4e5 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -197,6 +197,19 @@ def prepare_device_map():return device_map, True+def deepspeed_zero3_device_map(device_map):+ """None under DeepSpeed ZeRO-3 init: transformers refuses any device_map there, since ZeRO-3 places the+ weights itself (#225). Only a placement left to us is dropped; a map the caller chose still raises."""+ try:+ from transformers.integrations import is_deepspeed_zero3_enabled+ zero3 = is_deepspeed_zero3_enabled()+ except Exception:+ zero3 = False+ if not zero3:+ return device_map+ return None if is_automatic_device_map(device_map) else device_map++UNSLOTH_DEVICE_MAP = "unsloth"# Same planner, different answer when it declines: every veto path ends in "sequential", which fills the first device to its whole free budget, wrong for a caller that picked several cards.