Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block Swap: Stream frozen transformer blocks from host RAM so a dense model can train past the card's VRAM (#11832)

Leo Borcherding79116720a3f2
diff --git a/tests/test_block_swap_install.py b/tests/test_block_swap_install.py
new file mode 100644
index 00000000000..b8bdbfd7693
--- /dev/null
+++ b/tests/test_block_swap_install.py
@@ -0,0 +1,690 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Block swap helpers from _utils.py, extracted with ast to avoid importing torch's CUDA stack."""
+
+import ast, os, types
+import pytest
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py")
+NAMES = (
+ "_check_block_swap",
+ "_new_block_swap",
+ "refuse_block_swap_load",
+ "_training_reserve_bytes",
+ "_auto_block_swap_indices",
+ "install_block_swap",
+ "trim_config_for_block_swap",
+)
+
+
+def _load(
+ *,
+ moe = False,
+ integrated = False,
+ zoo = True,
+ cuda = True,
+ auto_pick = ([], 0),
+):
+ src = open(UTILS, encoding = "utf-8").read()
+ mod = ast.parse(src)
+ nodes = {n.name: n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name in NAMES}
+ assert set(nodes) == set(NAMES), f"missing from _utils.py: {set(NAMES) - set(nodes)}"
+
+ calls = []
+
+ class FakeSwap:
+ def __init__(
+ self,
+ layers,
+ n,
+ depth,
+ placement = "tail",
+ ):
+ calls.append((layers, n, depth, placement))
+
+ ns = {
+ "is_moe_model": lambda m: moe,
+ "is_integrated_unified_memory_gpu": lambda: integrated,
+ "BlockSwap": FakeSwap if zoo else None,
+ "build_host_layers": object() if zoo else None,
+ "find_decoder_layers": lambda m: m.layers,
+ "auto_swap_indices": lambda layers, reserve, depth: auto_pick,
+ "_offload_embedding_for_room": lambda model: False,
+ "estimate_training_reserve_bytes": lambda config, seq_len, extra_bytes = 0: 2**30,
+ "torch": types.SimpleNamespace(cuda = types.SimpleNamespace(is_available = lambda: cuda)),
+ }
+ for name in NAMES:
+ exec(ast.get_source_segment(src, nodes[name]), ns)
+ return ns, calls
+
+
+class _Layers(list):
+ pass
+
+
+class _Model:
+ config = None
+
+ def __init__(self, vllm = None):
+ self.layers = _Layers(["L0", "L1", "L2"])
+ if vllm is not None:
+ self.vllm_engine = vllm
+
+ def parameters(self):
+ return []
+
+
+def test_zero_is_a_no_op():
+ ns, calls = _load()
+ m = _Model()
+ assert ns["install_block_swap"](m, 0) is None
+ assert ns["install_block_swap"](m, None) is None
+ assert calls == [] and not hasattr(m, "_unsloth_block_swap")
+
+
+def test_happy_path_installs_and_records():
+ ns, calls = _load()
+ m = _Model()
+ sw = ns["install_block_swap"](m, 2, prefetch_depth = 3)
+ assert calls == [(m.layers, 2, 3, "spread")]
+ assert m._unsloth_block_swap is sw
+ assert m.layers._unsloth_block_swap is sw
+
+
+def test_refuses_vllm():
+ ns, calls = _load()
+ with pytest.raises(ValueError, match = "fast_inference"):
+ ns["install_block_swap"](_Model(vllm = object()), 2)
+ assert calls == []
+
+
+def test_moe_installs_with_a_notice(capsys):
+ ns, calls = _load(moe = True)
+ ns["install_block_swap"](_Model(), 2)
+ assert len(calls) == 1 and "MoE" in capsys.readouterr().out
+
+
+def test_refuses_unified_memory():
+ ns, calls = _load(integrated = True)
+ with pytest.raises(ValueError, match = "unified-memory"):
+ ns["install_block_swap"](_Model(), 2)
+ assert calls == []
+
+
+def test_refuses_without_gradient_checkpointing():
+ ns, calls = _load()
+ for off in (False, None):
+ with pytest.raises(ValueError, match = "use_gradient_checkpointing"):
+ ns["install_block_swap"](_Model(), 2, use_gradient_checkpointing = off)
+ assert calls == []
+
+
+def test_old_zoo_is_an_import_error():
+ ns, calls = _load(zoo = False)
+ with pytest.raises(ImportError, match = "unsloth_zoo"):
+ ns["install_block_swap"](_Model(), 2)
+ assert calls == []
+
+
+def test_refusal_order_checks_cheap_things_first():
+ ns, _ = _load(integrated = True)
+ with pytest.raises(ValueError, match = "fast_inference"):
+ ns["install_block_swap"](_Model(vllm = object()), 2)
+
+
+def test_swapper_from_load_is_reused():
+ ns, calls = _load()
+ m = _Model()
+ m._unsloth_block_swap = existing = object()
+ assert ns["install_block_swap"](m, 0) is existing
+ assert ns["install_block_swap"](m, 8) is existing
+ assert calls == []
+ with pytest.raises(ValueError, match = "use_gradient_checkpointing"):
+ ns["install_block_swap"](m, 0, use_gradient_checkpointing = False)
+
+
+class _Config:
+ def __init__(
+ self,
+ n = 8,
+ **extra,
+ ):
+ self.num_hidden_layers = n
+ self.layer_types = ["full_attention"] * n
+ self.other_list = [1, 2, 3]
+ self.__dict__.update(extra)
+
+
+def test_trim_off_at_zero():
+ ns, _ = _load()
+ cfg = _Config()
+ assert ns["trim_config_for_block_swap"](cfg, 0) is None
+ assert cfg.num_hidden_layers == 8 and len(cfg.layer_types) == 8
+
+
+def test_trim_shortens_per_layer_lists_and_returns_originals():
+ ns, _ = _load()
+ cfg = _Config()
+ saved = ns["trim_config_for_block_swap"](cfg, 5)
+ assert cfg.num_hidden_layers == 3 and cfg.layer_types == ["full_attention"] * 3
+ assert cfg.other_list == [1, 2, 3], "lists not sized to the layer count are untouched"
+ assert saved == {"num_hidden_layers": 8, "layer_types": ["full_attention"] * 8}
+
+
+def test_trim_keeps_at_least_one_layer_on_the_card():
+ ns, _ = _load()
+ cfg = _Config()
+ ns["trim_config_for_block_swap"](cfg, 100)
+ assert cfg.num_hidden_layers == 1
+
+
+def test_trim_refuses_what_install_refuses():
+ ns, _ = _load(integrated = True)
+ with pytest.raises(ValueError, match = "unified-memory"):
+ ns["trim_config_for_block_swap"](_Config(), 4)
+ ns, _ = _load(zoo = False)
+ with pytest.raises(ImportError, match = "unsloth_zoo"):
+ ns["trim_config_for_block_swap"](_Config(), 4)
+
+
+def _refusals(fn, needle):
+ """Lines of `if <needle ...>: block_swap_layers = refuse_block_swap_load(...)` in `fn`."""
+ return [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.If)
+ and needle in ast.unparse(n.test) + ast.unparse(n.body)
+ and "refuse_block_swap_load" in ast.unparse(n.body)
+ ]
+
+
+def _peft_signature_and_calls(path):
+ mod = ast.parse(open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read())
+ for cls in (n for n in mod.body if isinstance(n, ast.ClassDef)):
+ for fn in cls.body:
+ if isinstance(fn, ast.FunctionDef) and fn.name == "get_peft_model":
+ return fn
+ raise AssertionError(f"get_peft_model not found in {path}")
+
+
[email protected]("path", ["llama.py", "vision.py"])
+def test_block_swap_layers_is_last_so_positional_callers_keep_their_slots(path):
+ fn = _peft_signature_and_calls(path)
+ assert [a.arg for a in fn.args.args[-2:]] == ["block_swap_layers", "checkpoint_skip_layers"]
+ assert fn.args.kwarg is not None
+
+
+def test_new_model_route_forwards_block_swap_layers():
+ fn = _peft_signature_and_calls("llama.py")
+ forwarded = [
+ call
+ for call in ast.walk(fn)
+ if isinstance(call, ast.Call)
+ and isinstance(call.func, ast.Attribute)
+ and call.func.attr == "get_peft_model"
+ and getattr(call.func.value, "id", None) == "FastBaseModel"
+ ]
+ assert forwarded, "llama get_peft_model no longer delegates to FastBaseModel"
+ for call in forwarded:
+ assert {"block_swap_layers", "checkpoint_skip_layers"} <= {k.arg for k in call.keywords}
+
+
[email protected]("path", ["llama.py", "gemma.py", "gemma2.py"])
+def test_fast_decode_loops_fetch_swapped_layers(path):
+ # Decode loops read layer weights without calling the layer, so the swap hooks never fire.
+ src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()
+ assert "block_swap.enter(idx)" in src and "block_swap.leave(idx)" in src
+
+
+def _load_key_filter(monkeypatch, has_method = True):
+ import re, sys, types
+
+ mod = ast.parse(open(UTILS, encoding = "utf-8").read())
+ keep = [
+ n
+ for n in mod.body
+ if (isinstance(n, ast.FunctionDef) and n.name == "skip_swapped_checkpoint_keys")
+ or (
+ isinstance(n, ast.Assign)
+ and any(getattr(t, "id", None) == "_SWAPPED_LAYER_KEY" for t in n.targets)
+ )
+ ]
+ assert len(keep) == 2
+ seen = []
+
+ class PreTrainedModel:
+ pass
+
+ if has_method:
+
+ def _get_key_renaming_mapping(
+ self,
+ checkpoint_keys,
+ key_mapping = None,
+ ):
+ seen.append(list(checkpoint_keys))
+ return {k: k for k in checkpoint_keys}
+
+ PreTrainedModel._get_key_renaming_mapping = _get_key_renaming_mapping
+ fake = types.ModuleType("transformers.modeling_utils")
+ fake.PreTrainedModel = PreTrainedModel
+ monkeypatch.setitem(sys.modules, "transformers.modeling_utils", fake)
+ ns = {"re": re}
+ exec(compile(ast.Module(body = keep, type_ignores = []), UTILS, "exec"), ns)
+ return ns["skip_swapped_checkpoint_keys"], PreTrainedModel, seen
+
+
+def test_swapped_tail_keys_are_hidden_from_a_4x_load_then_restored(monkeypatch):
+ skip, cls, seen = _load_key_filter(monkeypatch)
+ original = cls.__dict__["_get_key_renaming_mapping"]
+ undo = skip({"num_hidden_layers": 4}, 2)
+ keys = [
+ "model.embed_tokens.weight",
+ "model.layers.1.mlp.down_proj.weight",
+ "model.layers.2.mlp.down_proj.weight",
+ "model.layers.3.self_attn.q_proj.weight.absmax",
+ "lm_head.weight",
+ ]
+ cls()._get_key_renaming_mapping(keys, key_mapping = None)
+ assert seen[-1] == [
+ "model.embed_tokens.weight",
+ "model.layers.1.mlp.down_proj.weight",
+ "lm_head.weight",
+ ]
+ undo()
+ assert cls.__dict__["_get_key_renaming_mapping"] is original
+
+
+def test_key_filter_is_inert_when_off_or_on_5x(monkeypatch):
+ skip, cls, _ = _load_key_filter(monkeypatch)
+ original = cls.__dict__["_get_key_renaming_mapping"]
+ skip(None, 2)()
+ assert cls.__dict__["_get_key_renaming_mapping"] is original
+ skip, cls, _ = _load_key_filter(monkeypatch, has_method = False)
+ skip({"num_hidden_layers": 4}, 2)()
+ assert "_get_key_renaming_mapping" not in cls.__dict__
+
+
+def test_checkpoint_tensors_reads_the_requested_variant(tmp_path):
+ torch = pytest.importorskip("torch")
+ safetensors_torch = pytest.importorskip("safetensors.torch")
+ mod = ast.parse(open(UTILS, encoding = "utf-8").read())
+ fn = next(
+ n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name == "_checkpoint_tensors"
+ )
+ ns = {"os": os}
+ exec(compile(ast.Module(body = [fn], type_ignores = []), UTILS, "exec"), ns)
+ safetensors_torch.save_file({"w": torch.zeros(2)}, str(tmp_path / "model.safetensors"))
+ safetensors_torch.save_file({"w": torch.ones(2)}, str(tmp_path / "model.fp16.safetensors"))
+ tensors, _ = ns["_checkpoint_tensors"](str(tmp_path), variant = "fp16")
+ assert torch.equal(tensors["w"](), torch.ones(2))
+ tensors, _ = ns["_checkpoint_tensors"](str(tmp_path))
+ assert torch.equal(tensors["w"](), torch.zeros(2))
+
+
+def test_non_safetensors_formats_are_refused_before_the_prefix_loads():
+ mod = ast.parse(
+ open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()
+ )
+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")
+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")
+ refusal = _refusals(fn, "needs a safetensors checkpoint")
+ trim = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "trim_config_for_block_swap"
+ ]
+ assert refusal and trim and refusal[0] < trim[0]
+
+
+def test_refuses_without_a_cuda_or_rocm_gpu():
+ ns, calls = _load(cuda = False)
+ with pytest.raises(ValueError, match = "CUDA or ROCm"):
+ ns["install_block_swap"](_Model(), 4)
+ assert not calls
+
+
+def test_prewrapped_peft_and_quantized_checkpoints_are_covered():
+ src = open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()
+ mod = ast.parse(src)
+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")
+ peft = next(
+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "get_peft_model"
+ )
+ installs = [
+ n
+ for n in ast.walk(peft)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "install_block_swap"
+ ]
+ assert len(installs) >= 2, "the pre-wrapped PEFT return path must install the swap too"
+ load = next(
+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained"
+ )
+ assert _refusals(load, "_ckpt_quant_method")
+
+
+def test_every_custom_decode_loop_is_served():
+ # A FastXModel with its own model-level decode loop must call block_swap.enter/leave.
+ models = os.path.join(HERE, "unsloth", "models")
+ for name in sorted(os.listdir(models)):
+ if not name.endswith(".py"):
+ continue
+ src = open(os.path.join(models, name), encoding = "utf-8").read()
+ loops = [
+ n
+ for n in ast.walk(ast.parse(src))
+ if isinstance(n, ast.FunctionDef)
+ and n.name.endswith("fast_forward_inference")
+ and "Attention" not in n.name
+ ]
+ if loops:
+ assert (
+ "block_swap" in src
+ ), f"{name} has a decode loop that never fetches swapped blocks"
+
+
+def test_a_caller_quantization_config_is_refused_before_loading():
+ mod = ast.parse(
+ open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()
+ )
+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")
+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")
+ raises = _refusals(fn, "quantization_config")
+ trim = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "trim_config_for_block_swap"
+ ]
+ assert raises and trim and raises[0] < trim[0]
+
+
+def test_every_unsupported_load_mode_is_refused_before_trimming():
+ mod = ast.parse(
+ open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()
+ )
+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")
+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")
+ trim = min(
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "trim_config_for_block_swap"
+ )
+ for needle in (
+ "load_in_8bit",
+ "quantization_config",
+ "state_dict",
+ "_ckpt_quant_method",
+ "safetensors",
+ ):
+ hits = _refusals(fn, needle)
+ assert hits and min(hits) < trim, needle
+
+
+def test_older_zoo_without_placement_still_installs():
+ ns, calls = _load()
+
+ class OldSwap:
+ def __init__(self, layers, n, depth):
+ calls.append((layers, n, depth))
+
+ ns["BlockSwap"] = OldSwap
+ m = _Model()
+ ns["install_block_swap"](m, 2)
+ assert calls == [(m.layers, 2, 2)]
+
+
+def test_auto_swaps_nothing_when_the_card_has_room(capsys):
+ ns, calls = _load(auto_pick = ([], 0))
+ assert ns["install_block_swap"](_Model(), "auto") is None
+ assert calls == [] and "swaps nothing" in capsys.readouterr().out
+
+
+def test_auto_swaps_the_layers_the_shortfall_needs():
+ ns, calls = _load(auto_pick = ([0, 2], 0))
+ m = _Model()
+ ns["install_block_swap"](m, "auto")
+ assert calls == [(m.layers, [0, 2], 2, "spread")]
+
+
+def test_auto_without_a_gpu_is_a_quiet_no_op():
+ ns, calls = _load(cuda = False)
+ assert ns["install_block_swap"](_Model(), "auto") is None
+ assert calls == []
+
+
+def test_load_refusals_fall_back_for_auto_and_raise_for_a_count(capsys):
+ ns, _ = _load()
+ assert ns["refuse_block_swap_load"]("auto", "does not take a state_dict.") == 0
+ assert "loads every layer onto the GPU" in capsys.readouterr().out
+ with pytest.raises(ValueError, match = "state_dict"):
+ ns["refuse_block_swap_load"](4, "does not take a state_dict.")
+
+
+def test_auto_moves_the_embedding_before_any_layer():
+ ns, calls = _load()
+ picks = iter([([0, 2], 0), ([], 0)])
+ ns["auto_swap_indices"] = lambda layers, reserve, depth: next(picks)
+ moved = []
+ ns["_offload_embedding_for_room"] = lambda model: moved.append(model) or True
+ m = _Model()
+ assert ns["install_block_swap"](m, "auto") is None
+ assert moved == [m] and calls == []
+
+
+def _load_host_helpers(device_count = 1, cuda = True):
+ src = open(UTILS, encoding = "utf-8").read()
+ mod = ast.parse(src)
+ names = ("block_swap_load_device", "begin_block_swap_load")
+ nodes = {n.name: n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name in names}
+ import contextlib, torch
+
+ seen = []
+ ns = {
+ "DEVICE_TYPE_TORCH": "cuda",
+ "contextlib": contextlib,
+ "load_layers_to_host": lambda n, **kw: seen.append((n, kw))
+ or contextlib.nullcontext("loading"),
+ "torch": types.SimpleNamespace(
+ device = torch.device,
+ cuda = types.SimpleNamespace(
+ is_available = lambda: cuda,
+ device_count = lambda: device_count,
+ current_device = lambda: 0,
+ ),
+ ),
+ }
+ for name in names:
+ exec(ast.get_source_segment(src, nodes[name]), ns)
+ return ns, seen
+
+
+ "device_map, count, expected",
+ [
+ ({"": 0}, 2, 0),
+ ({"": "cuda:1"}, 2, 1),
+ ("cuda:1", 2, 1),
+ ("sequential", 1, 0),
+ ("sequential", 2, None), # a strategy would split across the two cards
+ ({"a": 0, "b": 1}, 2, None),
+ ({"": "cpu"}, 1, None),
+ (None, 1, 0),
+ ],
+)
+def test_host_load_needs_one_card(device_map, count, expected):
+ ns, _ = _load_host_helpers(device_count = count)
+ assert ns["block_swap_load_device"](device_map) == expected
+
+
+def test_host_load_without_cuda_has_no_card():
+ ns, _ = _load_host_helpers(cuda = False)
+ assert ns["block_swap_load_device"]({"": 0}) is None
+
+
+def test_host_load_runs_for_layers_or_streamed_embeddings_only():
+ ns, seen = _load_host_helpers()
+ with ns["begin_block_swap_load"](0, {"": 0}) as state:
+ assert state is None
+ with ns["begin_block_swap_load"](0, {"": 0}, embeddings = True):
+ pass
+ with ns["begin_block_swap_load"](3, {"": 0}):
+ pass
+ assert seen == [(0, {"placement": "spread", "embeddings": True}), (3, {"placement": "spread"})]
+
+
+def _load_skip():
+ src = open(UTILS, encoding = "utf-8").read()
+ mod = ast.parse(src)
+ names = ("_skip_aware_flag", "skip_checkpointing")
+ ns = {"find_decoder_layers": lambda m: m.layers}
+ for n in mod.body:
+ if isinstance(n, ast.FunctionDef) and n.name in names:
+ exec(ast.get_source_segment(src, n), ns)
+ return ns
+
+
+def _skip_model(n = 4, swapped = None):
+ class Layer:
+ gradient_checkpointing = False
+
+ model = types.SimpleNamespace(layers = [Layer() for _ in range(n)])
+ for layer in model.layers:
+ layer.gradient_checkpointing = True
+ if swapped is not None:
+ model._unsloth_block_swap = types.SimpleNamespace(indices = swapped)
+ return model
+
+
+def test_skipped_layers_stay_off_when_checkpointing_is_turned_back_on():
+ ns = _load_skip()
+ model = _skip_model()
+ assert ns["skip_checkpointing"](model, 2) == [1, 3]
+ # for_training / gradient_checkpointing_enable write the flag on every layer again.
+ for layer in model.layers:
+ layer.gradient_checkpointing = True
+ assert [l.gradient_checkpointing for l in model.layers] == [True, False, True, False]
+ for layer in model.layers:
+ layer.gradient_checkpointing = False
+ assert not any(l.gradient_checkpointing for l in model.layers)
+
+
+def test_skip_leaves_block_swapped_layers_checkpointed():
+ ns = _load_skip()
+ model = _skip_model(4, swapped = [0, 2])
+ assert ns["skip_checkpointing"](model, "max") == [1, 3]
+ assert [l.gradient_checkpointing for l in model.layers] == [True, False, True, False]
+
+
+def test_skip_zero_changes_nothing():
+ ns = _load_skip()
+ model = _skip_model()
+ assert ns["skip_checkpointing"](model, 0) == []
+ assert all(l.gradient_checkpointing for l in model.layers)
+
+
+def _method(path, cls_name, name):
+ mod = ast.parse(open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read())
+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == cls_name)
+ return next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == name)
+
+
+def test_generic_load_refuses_unified_memory_before_loading_to_host():
+ fn = _method("vision.py", "FastBaseModel", "from_pretrained")
+ begin = min(
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "begin_block_swap_load"
+ )
+ hits = _refusals(fn, "is_integrated_unified_memory_gpu")
+ assert hits and min(hits) < begin
+
+
+def test_headless_load_swaps_onto_the_retained_layers_card():
+ torch = pytest.importorskip("torch")
+ mod = ast.parse(open(UTILS, encoding = "utf-8").read())
+ fn = next(
+ n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name == "finish_block_swap_load"
+ )
+ built = []
+ card = torch.device("cuda", 1)
+ ns = {
+ "torch": types.SimpleNamespace(
+ device = torch.device, cuda = types.SimpleNamespace(current_device = lambda: 0)
+ ),
+ "_new_block_swap": lambda layers, idx, depth, device, placement: built.append(device)
+ or object(),
+ }
+ exec(compile(ast.Module(body = [fn], type_ignores = []), UTILS, "exec"), ns)
+
+ class _Layer:
+ def __init__(self, device):
+ self.device = device
+
+ def parameters(self):
+ return [types.SimpleNamespace(device = self.device)]
+
+ class _Headless:
+ def get_output_embeddings(self):
+ return None
+
+ layers = _Layers([_Layer(card), _Layer(torch.device("cpu")), _Layer(torch.device("cpu"))])
+ ns["finish_block_swap_load"](_Headless(), types.SimpleNamespace(layers = layers, indices = [1, 2]))
+ assert built == [card]
+
+
[email protected]("path", ["llama.py", "vision.py"])
+def test_auto_plan_never_counts_on_an_embedding_offload_the_platform_refuses(path):
+ src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()
+ call = next(
+ n
+ for n in ast.walk(ast.parse(src))
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "resolve_auto_block_swap"
+ )
+ kw = next(k for k in call.keywords if k.arg == "offload_embedding")
+ assert "_offload_embedding_unsupported_platform" in ast.get_source_segment(src, kw.value)
+
+
+def test_load_time_swappers_allocate_the_planned_prefetch_depth():
+ torch = pytest.importorskip("torch")
+ mod = ast.parse(open(UTILS, encoding = "utf-8").read())
+ fns = [
+ n
+ for n in mod.body
+ if isinstance(n, ast.FunctionDef)
+ and n.name in ("finish_block_swap_load", "planned_prefetch_depth")
+ ]
+ built = []
+ ns = {
+ "torch": types.SimpleNamespace(
+ device = torch.device, cuda = types.SimpleNamespace(current_device = lambda: 0)
+ ),
+ "_new_block_swap": lambda layers, idx, depth, device, placement: built.append(depth)
+ or object(),
+ }
+ exec(compile(ast.Module(body = fns, type_ignores = []), UTILS, "exec"), ns)
+
+ class _Headless:
+ def get_output_embeddings(self):
+ return None
+
+ depth = ns["planned_prefetch_depth"]({"prefetch_depth": 1})
+ ns["finish_block_swap_load"](
+ _Headless(), types.SimpleNamespace(layers = _Layers([]), indices = [0]), depth
+ )
+ assert built == [1] and ns["planned_prefetch_depth"](None) == 2
+ # Both load paths hand the planner's depth to the swapper they build.
+ for path, call in (
+ ("llama.py", "attach_block_swap_layers"),
+ ("vision.py", "finish_block_swap_load"),
+ ):
+ src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()
+ node = next(
+ n
+ for n in ast.walk(ast.parse(src))
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == call
+ )
+ assert "planned_prefetch_depth(device_map_planner_kwargs)" in ast.get_source_segment(
+ src, node
+ )
diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.py
index 4612ef3d0e0..f99eb574973 100644
--- a/tests/test_modelopt_fp8_loading.py
+++ b/tests/test_modelopt_fp8_loading.py
@@ -549,9 +549,12 @@ def test_config_branch_moves_rope_extension_onto_the_config():
import inspect
from unsloth.models import llama
+ import re
+
source = inspect.getsource(llama.FastLlamaModel.from_pretrained)
- branch = source.split("if user_config is not None or _modelopt_rewritten", 1)[1]
- branch = branch.split("AutoModelForCausalLM.from_pretrained(", 1)[0]
+ # The condition may wrap over several lines once it grows.
+ start = re.search(r"if \(?\s*user_config is not None\s+or _modelopt_rewritten", source)
+ branch = source[start.end() :].split("AutoModelForCausalLM.from_pretrained(", 1)[0]
assert 'kwargs.pop("rope_scaling", None)' in branch
diff --git a/tests/test_offload_tied_autodisable.py b/tests/test_offload_tied_autodisable.py
index 3b3720b4a47..d0e394c7d05 100644
--- a/tests/test_offload_tied_autodisable.py
+++ b/tests/test_offload_tied_autodisable.py
@@ -10,10 +10,11 @@ Every platform branch is driven explicitly, so the assertions hold on Linux,
macOS, Windows and WSL alike: the host's own os.name never decides.
"""
-import ast, os
+import ast, os, types
from contextlib import contextmanager
import pytest
+from real_accelerator import has_real_cuda
# Skip rather than error where torch is absent. Only `nn.Embedding` / `nn.Linear` / `torch.device` are wanted here, no
# GPU, but a bare module-level import turns a machine without torch into a collection error, which aborts the whole
@@ -41,6 +42,8 @@ def _load(*names):
"os": os,
"OFFLOAD_EMBEDDING_AUTO": "auto",
"is_distributed": lambda: _DISTRIBUTED[0],
+ # An unsloth_zoo without the reserve estimate: "auto" keeps the size rule tested below.
+ "_zoo_reserve_estimate": None,
}
wanted = set(names)
for node in mod.body:
@@ -185,7 +188,7 @@ def test_resolved_before_multidevice_hooks():
# Hook attach returns early while offload_embedding is still True.
call = _SRC.index("offload_embedding = _resolve_offload_embedding(")
# Anchor on the indented CALL, not the module-level `def`.
- hooks = _SRC.index("\n _attach_bnb_multidevice_hooks(")
+ hooks = _SRC.index("\n _attach_bnb_multidevice_hooks(")
assert call < hooks, "offload_embedding must be resolved before hook attach"
@@ -388,3 +391,116 @@ def test_a_distributed_launch_also_declines_an_explicit_request(capsys):
def test_a_single_process_run_is_unaffected():
with _as_platform("posix"), _card(16 * 2**30):
assert resolve(_sized_model(int(2.5 * 2**30)), "auto") is True
+
+
+def test_auto_follows_free_memory_once_the_reserve_estimate_exists():
+ """With the estimate, `"auto"` never offloads on size alone; it waits for a memory shortfall
+ (`needed`, from offload_embedding_if_tight or the block swap planner)."""
+ _NS["_zoo_reserve_estimate"] = object()
+ try:
+ with _as_platform("posix"), _card(16 * 2**30):
+ big = _sized_model(int(2.5 * 2**30))
+ assert resolve(big, "auto") is False
+ assert resolve(big, "auto", needed = True) is True
+ assert resolve(big, True) is True
+ finally:
+ _NS["_zoo_reserve_estimate"] = None
+
+
+class _PerLayerModel(nn.Module):
+ """Tied input embedding plus a large per-layer table (Gemma 3n / 4 layout)."""
+
+ def __init__(self, device):
+ super().__init__()
+ self.embed_tokens = nn.Embedding(64, 8, device = device)
+ self.per_layer = nn.Embedding(64, 32, device = device)
+ self.lm_head = nn.Linear(8, 64, bias = False, device = device)
+ self.lm_head.weight = self.embed_tokens.weight
+
+ def get_input_embeddings(self):
+ return self.embed_tokens
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+
+def _spare_ns():
+ ns = _load(
+ "_embeddings_are_tied",
+ "_offload_embedding_unsupported_platform",
+ "_embedding_dispatch_device",
+ "_input_side_embeddings",
+ "_accelerator_device",
+ "offload_spare_embeddings",
+ )
+ ns["_EXTRA_EMBEDDING_MIN_BYTES"] = 64 * 32 * 4 # the per-layer table above, no bigger
+ moved = []
+ ns["offload_input_embedding"] = lambda model, embeddings: moved.extend(embeddings) or 1
+ return ns, moved
+
+
[email protected](not has_real_cuda(), reason = "needs a card to offload from")
+def test_tied_model_still_offloads_its_per_layer_table():
+ ns, moved = _spare_ns()
+ model = _PerLayerModel("cuda")
+ model.requires_grad_(False)
+ with _as_platform("posix"):
+ assert ns["offload_spare_embeddings"](model) == 1
+ assert moved == [model.per_layer]
+
+
[email protected](not has_real_cuda(), reason = "needs a card to offload from")
+def test_trainable_or_cpu_model_tables_stay():
+ ns, moved = _spare_ns()
+ model = _PerLayerModel("cuda")
+ with _as_platform("posix"):
+ # get_peft_model: a table still trainable is not offloaded.
+ assert ns["offload_spare_embeddings"](model, require_frozen = True) == 0
+ model = _PerLayerModel("cpu").requires_grad_(False)
+ with _as_platform("posix"):
+ # A model on the CPU has nothing to offload.
+ assert ns["offload_spare_embeddings"](model) == 0
+ assert moved == []
+
+
+class _HeadlessPerLayerModel(_PerLayerModel):
+ """AutoModel backbone: no output head."""
+
+ def get_output_embeddings(self):
+ return None
+
+
[email protected](not has_real_cuda(), reason = "needs a card to offload from")
+def test_headless_model_hooks_a_table_the_block_swap_load_left_on_cpu():
+ ns, moved = _spare_ns()
+ model = _HeadlessPerLayerModel("cuda").requires_grad_(False)
+ # The block swap load streamed the table to host without hooks; the backbone stays on the card.
+ model.per_layer.to("cpu")
+ with _as_platform("posix"):
+ assert ns["offload_spare_embeddings"](model) == 1
+ # Nothing ties the input embedding without a head, so it may go too.
+ assert any(m is model.per_layer for m in moved)
+ moved.clear()
+ model = _HeadlessPerLayerModel("cpu").requires_grad_(False)
+ with _as_platform("posix"):
+ assert ns["offload_spare_embeddings"](model) == 0
+ assert moved == []
+
+
[email protected](not has_real_cuda(), reason = "needs a card to offload from")
+def test_headless_streamed_table_returns_rows_to_the_decoder_card():
+ ns = _load(
+ "_embeddings_are_tied",
+ "_input_side_embeddings",
+ "_accelerator_device",
+ "offload_input_embedding",
+ )
+ ns["_EXTRA_EMBEDDING_MIN_BYTES"] = 64 * 32 * 4
+ hooked = []
+ ns["_install_offload_embedding_hooks"] = lambda emb, out, device: hooked.append((emb, device))
+ ns["_pin_device_to_decoder"] = ns["clean_gpu_cache"] = lambda *a: None
+ ns["gc"] = types.SimpleNamespace(collect = lambda: None)
+ model = _HeadlessPerLayerModel("cuda").requires_grad_(False)
+ model.per_layer.to("cpu")
+ ns["offload_input_embedding"](model, [model.per_layer])
+ assert [(e is model.per_layer, d.type) for e, d in hooked] == [(True, "cuda")]
diff --git a/tests/test_unsloth_device_map_optin.py b/tests/test_unsloth_device_map_optin.py
index 92480d1ba5b..0b510bcd88e 100644
--- a/tests/test_unsloth_device_map_optin.py
+++ b/tests/test_unsloth_device_map_optin.py
@@ -869,13 +869,15 @@ def test_a_caller_supplied_config_declines_planning():
raise AssertionError("vision.py plans without vetoing a caller-supplied config")
-def test_the_optimized_path_says_so_when_it_drops_an_offload_request():
- """`FastLanguageModel` accepts `offload_embedding`, but the optimized architectures
- take a path that has never had the parameter, so the request went nowhere in silence.
- The `"auto"` default stays quiet, since off is a decision it is entitled to make."""
- source = open(os.path.join(MODELS, "loader.py"), encoding = "utf-8").read()
- assert "does not support it" in source
- assert "offload_embedding != OFFLOAD_EMBEDDING_AUTO" in source
+def test_the_optimized_path_honours_offload_embedding():
+ """`FastLanguageModel` accepts `offload_embedding`, and the optimized architectures used to
+ drop it in silence. Now the loader forwards it and FastLlamaModel applies the same offload as
+ FastModel, through the shared helper."""
+ loader = open(os.path.join(MODELS, "loader.py"), encoding = "utf-8").read()
+ assert "does not support it" not in loader
+ llama = open(os.path.join(MODELS, "llama.py"), encoding = "utf-8").read()
+ assert 'kwargs.pop("offload_embedding", False)' in llama
+ assert "offload_input_embedding(model)" in llama
def test_the_auto_mode_is_recognised_by_value_everywhere():
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index c4a28ee385f..467ee63773b 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -87,6 +87,18 @@ __all__ = [
"hf_login",
"maybe_prefetch_hf_snapshot",
"is_moe_model",
+ "install_block_swap",
+ "offload_embedding_if_tight",
+ "usable_cuda_bytes",
+ "skip_checkpointing",
+ "refuse_block_swap_load",
+ "block_swap_load_device",
+ "begin_block_swap_load",
+ "finish_block_swap_load",
+ "planned_prefetch_depth",
+ "trim_config_for_block_swap",
+ "attach_block_swap_layers",
+ "skip_swapped_checkpoint_keys",
"get_moe_target_parameters",
"get_moe_target_modules",
"get_moe_expert_submodule_leaves",
@@ -159,6 +171,24 @@ from unsloth_zoo.patching_utils import (
patch_model_and_tokenizer,
patch_compiled_autograd,
)
+from ._uma_safetensors import is_integrated_unified_memory_gpu
+
+try:
+ from unsloth_zoo.block_swap import BlockSwap, find_decoder_layers
+except ImportError: # unsloth_zoo predates block_swap
+ BlockSwap = find_decoder_layers = None
+try:
+ from unsloth_zoo.block_swap import build_host_layers
+except ImportError: # unsloth_zoo predates loading straight to host
+ build_host_layers = None
+try:
+ from unsloth_zoo.block_swap import load_layers_to_host
+except ImportError: # unsloth_zoo predates loading any architecture straight to host
+ load_layers_to_host = None
+try:
+ from unsloth_zoo.block_swap import auto_swap_indices, estimate_training_reserve_bytes
+except ImportError: # unsloth_zoo predates block_swap_layers = "auto"
+ auto_swap_indices = estimate_training_reserve_bytes = None
from unsloth_zoo.gradient_checkpointing import (
Unsloth_Offloaded_Gradient_Checkpointer,
unsloth_offloaded_gradient_checkpoint,
@@ -6090,6 +6120,482 @@ def hf_login(token: Optional[str] = None) -> Optional[str]:
return token
+def _check_block_swap(model_or_config):
+ if BlockSwap is None:
+ raise ImportError(
+ "Unsloth: block_swap_layers needs a newer unsloth_zoo. "
+ "Run `pip install --upgrade unsloth_zoo`."
+ )
+ if is_moe_model(model_or_config):
+ print(
+ "Unsloth: block_swap_layers on an MoE model copies every expert across PCIe "
+ "while only the routed ones compute, so steps can be copy-bound."
+ )
+ if not torch.cuda.is_available():
+ # Prefetch runs on CUDA/HIP streams; XPU, NPU and CPU have none.
+ raise ValueError("Unsloth: block_swap_layers needs a CUDA or ROCm GPU.")
+ if is_integrated_unified_memory_gpu():
+ raise ValueError(
+ "Unsloth: block_swap_layers has nothing to swap to on a unified-memory "
+ "GPU; host and device already share the same RAM."
+ )
+
+
+def _new_block_swap(layers, n, *args, placement, **kwargs):
+ try:
+ return BlockSwap(layers, n, *args, placement = placement, **kwargs)
+ except TypeError as e:
+ if "placement" not in str(e):
+ raise
+ # unsloth_zoo before placement support always swaps the last n.
+ return BlockSwap(layers, n, *args, **kwargs)
+
+
+def refuse_block_swap_load(block_swap_layers, reason):
+ """A load-to-host restriction: `"auto"` falls back to loading onto the GPU (0), a count raises."""
+ if block_swap_layers == "auto":
+ print(
+ f"Unsloth: block_swap_layers = 'auto' loads every layer onto the GPU: loading into host RAM {reason}"
+ )
+ return 0
+ raise ValueError(f"Unsloth: from_pretrained(block_swap_layers = ...) {reason}")
+
+
+def block_swap_load_device(device_map):
+ """The one CUDA card a load puts every layer on, else None: loading to host fetches onto one card."""
+ if DEVICE_TYPE_TORCH != "cuda" or not torch.cuda.is_available():
+ return None
+ target = device_map
+ if isinstance(device_map, dict):
+ if len(set(device_map.values())) != 1:
+ return None
+ target = next(iter(device_map.values()))
+ if target is None or target in ("auto", "balanced", "balanced_low_0", "sequential"):
+ return torch.cuda.current_device() if torch.cuda.device_count() == 1 else None
+ try:
+ target = torch.device("cuda", target) if isinstance(target, int) else torch.device(target)
+ except (RuntimeError, TypeError, ValueError):
+ return None
+ if target.type != "cuda":
+ return None
+ return target.index if target.index is not None else torch.cuda.current_device()
+
+
+def begin_block_swap_load(
+ block_swap_layers,
+ device_map,
+ embeddings = False,
+):
+ """from_pretrained(block_swap_layers = N) on any architecture: the context the weight load runs in,
+ moving N decoder layers (and with `embeddings`, the large extra token tables) to host RAM as each
+ finishes loading. nullcontext when N is 0."""
+ if not block_swap_layers and not embeddings:
+ return contextlib.nullcontext()
+ if embeddings:
+ return load_layers_to_host(block_swap_layers, placement = "spread", embeddings = True)
+ return load_layers_to_host(block_swap_layers, placement = "spread")
+
+
+def planned_prefetch_depth(device_map_planner_kwargs):
+ # The depth "auto" sized the slot pool with; the swapper must allocate the same pool.
+ return int((device_map_planner_kwargs or {}).get("prefetch_depth", 2))
+
+
+def finish_block_swap_load(
+ model,
+ state,
+ prefetch_depth = 2,
+):
+ """Install the swap over the layers `begin_block_swap_load` moved, once the loader is done
+ patching (their host copies are what the slot pool streams)."""
+ if state is None or getattr(state, "layers", None) is None or not state.indices:
+ return None
+ head = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None
+ weight = getattr(head, "weight", None)
+ device = weight.device if weight is not None and weight.device.type == "cuda" else None
+ if device is None:
+ # Headless backbones (AutoModel): a retained layer names the card, which need not be the current one.
+ swapped = set(state.indices)
+ device = next(
+ (
+ p.device
+ for i, layer in enumerate(state.layers)
+ if i not in swapped
+ for p in layer.parameters()
+ if p.device.type == "cuda"
+ ),
+ None,
+ )
+ if device is None:
+ device = torch.device("cuda", torch.cuda.current_device())
+ swapper = _new_block_swap(
+ state.layers, state.indices, prefetch_depth, device = device, placement = "spread"
+ )
+ state.layers._unsloth_block_swap = swapper
+ model._unsloth_block_swap = swapper
+ return swapper
+
+
+# loader_utils.OFFLOAD_EMBEDDING_AUTO; loader_utils imports this module, so not imported from there.
+_OFFLOAD_EMBEDDING_AUTO = "auto"
+
+
+def _offload_embedding_for_room(model, require_frozen = True):
+ """Move the input-side embeddings lm_head does not share to host RAM, unless the caller passed
+ offload_embedding = False. After get_peft_model they must also be frozen."""
+ if getattr(model, "_unsloth_offload_embedding_mode", _OFFLOAD_EMBEDDING_AUTO) is False:
+ return False
+ from .vision import offload_spare_embeddings
+
+ return offload_spare_embeddings(model, require_frozen = require_frozen) > 0
+
+
+def _training_reserve_bytes(
+ model,
+ seq_len = None,
+ trainable = True,
+):
+ # Grads, AdamW's two fp32 moments and the foreach temp; before get_peft_model every param still says requires_grad.
+ extra = 0
+ if trainable:
+ extra = sum(
+ p.numel() * (p.element_size() + 12) for p in model.parameters() if p.requires_grad
+ )
+ seq_len = seq_len or getattr(model, "max_seq_length", None) or 2048
+ return estimate_training_reserve_bytes(model.config, seq_len, extra_bytes = extra), seq_len
+
+
+def _skip_aware_flag(cls):
+ # for_training, gradient_checkpointing_enable and the trainer rewrite the flag; a marked layer keeps reading False.
+ if cls.__dict__.get("_unsloth_skip_aware", False):
+ return
+
+ def _get(self):
+ if self.__dict__.get("_unsloth_skip_checkpoint", False):
+ return False
+ # Unmarked layers of the class still hold the flag in their own dict.
+ return self.__dict__.get(
+ "_unsloth_gc_flag", self.__dict__.get("gradient_checkpointing", False)
+ )
+
+ def _set(self, value):
+ self.__dict__["_unsloth_gc_flag"] = value
+
+ cls.gradient_checkpointing = property(_get, _set)
+ cls._unsloth_skip_aware = True
+
+
+def skip_checkpointing(model, layers = 0):
+ """get_peft_model(checkpoint_skip_layers = K | "max"): K decoder layers (evenly spaced; "max" =
+ all) keep their activations instead of recomputing them in backward. Faster, at the cost of
+ each layer's activations on the card. Block-swapped layers keep checkpointing."""
+ if not layers:
+ return []
+ layer_list = find_decoder_layers(model)
+ swapper = getattr(model, "_unsloth_block_swap", None)
+ swapped = set(getattr(swapper, "indices", None) or ())
+ eligible = [i for i in range(len(layer_list)) if i not in swapped]
+ if layers == "max":
+ chosen = eligible
+ else:
+ count = max(0, min(int(layers), len(eligible)))
+ chosen = (
+ [eligible[(k + 1) * len(eligible) // count - 1] for k in range(count)] if count else []
+ )
+ for i in chosen:
+ layer = layer_list[i]
+ if not hasattr(layer, "gradient_checkpointing"):
+ continue
+ current = layer.__dict__.pop("gradient_checkpointing", None)
+ _skip_aware_flag(type(layer))
+ if current is not None:
+ layer.__dict__["_unsloth_gc_flag"] = current
+ layer.__dict__["_unsloth_skip_checkpoint"] = True
+ return chosen
+
+
+def usable_cuda_bytes(device):
+ """Bytes this process can still allocate on `device`: free card memory plus blocks torch's
+ caching allocator holds unused, capped by torch.cuda.set_per_process_memory_fraction."""
+ free, total = torch.cuda.mem_get_info(device)
+ allocated = torch.cuda.memory_allocated(device)
+ usable = free + torch.cuda.memory_reserved(device) - allocated
+ get_fraction = getattr(torch.cuda, "get_per_process_memory_fraction", None)
+ try:
+ fraction = get_fraction(device) if get_fraction is not None else 1.0
+ except Exception:
+ fraction = 1.0
+ if fraction < 1.0:
+ usable = min(usable, int(total * fraction) - allocated)
+ return max(0, int(usable))
+
+
+def offload_embedding_if_tight(
+ model,
+ seq_len = None,
+ at_load = False,
+):
+ """offload_embedding = "auto": move the input embedding to host RAM only when its GPU lacks a
+ training step's reserve (activations, logits, trainable state), on any architecture. Runs after
+ the load and again in get_peft_model, once the trainable parameters are known."""
+ if estimate_training_reserve_bytes is None or not torch.cuda.is_available():
+ return False
+ if getattr(model, "_unsloth_offload_embedding_mode", None) != _OFFLOAD_EMBEDDING_AUTO:
+ return False
+ try:
+ weight = getattr(model.get_input_embeddings(), "weight", None)
+ except Exception:
+ return False
+ if weight is None or weight.device.type != "cuda":
+ return False
+ reserve, _ = _training_reserve_bytes(model, seq_len, trainable = not at_load)
+ if usable_cuda_bytes(weight.device) >= reserve:
+ return False
+ return _offload_embedding_for_room(model, require_frozen = not at_load)
+
+
+def _auto_block_swap_indices(model, prefetch_depth):
+ """Layers to swap so each GPU keeps a training step's reserve free; [] when it already does."""
+ if auto_swap_indices is None:
+ raise ImportError(
+ "Unsloth: block_swap_layers = 'auto' needs a newer unsloth_zoo. "
+ "Run `pip install --upgrade unsloth_zoo`."
+ )
+ layers = find_decoder_layers(model)
+ reserve, seq_len = _training_reserve_bytes(model)
+ indices, left = auto_swap_indices(layers, reserve, prefetch_depth)
+ # Only the looked-up rows cross PCIe, so the embedding goes before any layer.
+ if indices and _offload_embedding_for_room(model):
+ indices, left = auto_swap_indices(layers, reserve, prefetch_depth)
+ if not indices:
+ print(
+ f"Unsloth: block_swap_layers = 'auto' swaps nothing: every GPU keeps the "
+ f"{reserve / 2**30:.2f} GiB a {seq_len}-token step needs."
+ )
+ else:
+ print(
+ f"Unsloth: block_swap_layers = 'auto' keeps {len(indices)} of {len(layers)} decoder layers "
+ f"in host RAM so {reserve / 2**30:.2f} GiB stays free for a {seq_len}-token step."
+ )
+ if left > 0:
+ print(
+ f"Unsloth: block_swap_layers = 'auto' is still {left / 2**30:.2f} GiB short; "
+ "lower max_seq_length, or load with from_pretrained(block_swap_layers = 'auto')."
+ )
+ return indices
+
+
+def install_block_swap(
+ model,
+ block_swap_layers = 0,
+ prefetch_depth = 2,
+ use_gradient_checkpointing = "unsloth",
+):
+ """Stream frozen decoder blocks from pinned host RAM: `block_swap_layers` of them, or "auto" for
+ as few as keep a training step's reserve free on every GPU; off at 0."""
+ existing = getattr(model, "_unsloth_block_swap", None)
+ if existing is None and (
+ not block_swap_layers or (block_swap_layers != "auto" and block_swap_layers <= 0)
+ ):
+ return None
+ if not use_gradient_checkpointing:
+ raise ValueError(
+ "Unsloth: block_swap_layers needs use_gradient_checkpointing. Without "
+ "it every swapped block stays on the card until backward, so there is "
+ "nothing to save and the slot pool runs dry mid-forward."
+ )
+ if existing is not None:
+ return existing
+ if getattr(model, "vllm_engine", None) is not None:
+ raise ValueError(
+ "Unsloth: block_swap_layers cannot be combined with fast_inference = True, "
+ "since evicted weights would sync to vLLM as empty tensors."
+ )
+ if block_swap_layers == "auto":
+ # Nothing to swap to without a discrete CUDA / ROCm card: auto means no swap there.
+ if not torch.cuda.is_available() or is_integrated_unified_memory_gpu():
+ return None
+ if BlockSwap is None:
+ _check_block_swap(model)
+ block_swap_layers = _auto_block_swap_indices(model, prefetch_depth)
+ if not block_swap_layers:
+ return None
+ _check_block_swap(model)
+ layers = find_decoder_layers(model)
+ # Spaced evenly, each copy hides behind several layers of compute instead of one.
+ swapper = _new_block_swap(layers, block_swap_layers, prefetch_depth, placement = "spread")
+ # On the layer list too: the fast decode loop only sees the inner model.
+ layers._unsloth_block_swap = swapper
+ model._unsloth_block_swap = swapper
+ return swapper
+
+
+def _checkpoint_tensors(
+ model_name,
+ token = None,
+ revision = None,
+ cache_dir = None,
+ local_files_only = False,
+ subfolder = None,
+ variant = None,
+):
+ from safetensors import safe_open
+ import json
+
+ def _named(filename):
+ if variant is None:
+ return filename
+ stem, ext = filename.rsplit(".", 1)
+ return f"{stem}.{variant}.{ext}"
+
+ def _get(filename):
+ if os.path.isdir(model_name):
+ path = os.path.join(model_name, subfolder or "", filename)
+ return path if os.path.exists(path) else None
+ from huggingface_hub import hf_hub_download
+ try:
+ return hf_hub_download(
+ model_name,
+ filename,
+ subfolder = subfolder,
+ revision = revision,
+ token = token,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ )
+ except Exception:
+ return None
+
+ index = _get(_named("model.safetensors.index.json"))
+ if index is not None:
+ with open(index, "r", encoding = "utf-8") as f:
+ shards = sorted(set(json.load(f)["weight_map"].values()))
+ else:
+ shards = [_named("model.safetensors")]
+ handles, tensors = [], {}
+ for shard in shards:
+ path = _get(shard)
+ if path is None:
+ raise RuntimeError(
+ f"Unsloth: block_swap_layers could not find {shard} for {model_name}."
+ )
+ h = safe_open(path, framework = "pt", device = "cpu")
+ handles.append(h)
+ for key in h.keys():
+ tensors[key] = lambda h = h, key = key: h.get_tensor(key)
+ return tensors, handles
+
+
+def trim_config_for_block_swap(config, block_swap_layers):
+ """Trim the swapped tail off the config; returns the originals to restore, or None."""
+ if not block_swap_layers or block_swap_layers <= 0:
+ return None
+ if build_host_layers is None:
+ raise ImportError(
+ "Unsloth: from_pretrained(block_swap_layers = ...) needs a newer unsloth_zoo. "
+ "Run `pip install --upgrade unsloth_zoo`."
+ )
+ _check_block_swap(config)
+ total = config.num_hidden_layers
+ n = min(int(block_swap_layers), total - 1)
+ saved = {"num_hidden_layers": total}
+ for key, value in list(vars(config).items()):
+ if isinstance(value, (list, tuple)) and len(value) == total:
+ saved[key] = value
+ setattr(config, key, type(value)(value[: total - n]))
+ config.num_hidden_layers = total - n
+ return saved
+
+
+_SWAPPED_LAYER_KEY = re.compile(r"(?:^|\.)layers\.(\d+)\.")
+
+
+def skip_swapped_checkpoint_keys(saved, kept):
+ """Hide the swapped tail's checkpoint keys from the standard load; returns an undo callable.
+
+ transformers 4.x still loads every key its renaming mapping keeps: the tail's bnb stats reach the
+ quantizer (ModuleList has no attribute `N`) and plain weights log "were not used", which
+ RaiseUninitialized turns into an error. 5.x has no such method and already drops them."""
+ from transformers.modeling_utils import PreTrainedModel
+
+ original = PreTrainedModel.__dict__.get("_get_key_renaming_mapping")
+ if saved is None or original is None:
+ return lambda: None
+
+ def _get_key_renaming_mapping(self, checkpoint_keys, *args, **kwargs):
+ keep = []
+ for key in checkpoint_keys:
+ m = _SWAPPED_LAYER_KEY.search(key)
+ if m is None or int(m.group(1)) < kept:
+ keep.append(key)
+ return original(self, keep, *args, **kwargs)
+
+ PreTrainedModel._get_key_renaming_mapping = _get_key_renaming_mapping
+
+ def undo():
+ PreTrainedModel._get_key_renaming_mapping = original
+
+ return undo
+
+
+def attach_block_swap_layers(
+ model,
+ saved,
+ model_name,
+ dtype,
+ load_in_4bit,
+ skip_modules = (),
+ prefetch_depth = 2,
+ **hub_kwargs,
+):
+ if saved is None:
+ return None
+ layers = find_decoder_layers(model)
+ first = len(layers)
+ config = model.config
+ for key, value in saved.items():
+ setattr(config, key, value)
+ count = config.num_hidden_layers - first
+ layer_cls = type(layers[0])
+ # The tail fetches onto the head's card, where a multi-GPU plan reserved its slot pool.
+ head = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None
+ weight = getattr(head, "weight", None)
+ device = weight.device if weight is not None else next(layers[first - 1].parameters()).device
+ if device.type != "cuda":
+ device = torch.device("cuda", torch.cuda.current_device())
+ tensors, handles = _checkpoint_tensors(model_name, **hub_kwargs)
+ try:
+ new = build_host_layers(
+ lambda idx: layer_cls(config, idx),
+ first,
+ count,
+ tensors,
+ device = device,
+ compute_dtype = dtype,
+ quantize_4bit = load_in_4bit,
+ skip_modules = skip_modules,
+ )
+ finally:
+ del handles
+ # Carry over plain attributes post_patch set on loaded layers (Gemma's norm variance_epsilon).
+ reference = dict(layers[0].named_modules())
+ for layer in new:
+ for name, module in layer.named_modules():
+ ref = reference.get(name)
+ if ref is None:
+ continue
+ for key, value in vars(ref).items():
+ if key.startswith("_") or key in vars(module):
+ continue
+ if not isinstance(value, (torch.Tensor, torch.nn.Module)):
+ setattr(module, key, value)
+ layers.append(layer)
+ swapper = _new_block_swap(layers, count, prefetch_depth, device = device, placement = "tail")
+ layers._unsloth_block_swap = swapper
+ model._unsloth_block_swap = swapper
+ return swapper
+
+
def is_moe_model(model) -> bool:
"""Detect whether a model (or config) is a Mixture of Experts model."""
config = getattr(model, "config", model)
diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py
index fce9af39ef9..c81e30550d9 100644
--- a/unsloth/models/cohere.py
+++ b/unsloth/models/cohere.py
@@ -445,7 +445,10 @@ def CohereModel_fast_forward_inference(
attention_mask = None
next_decoder_cache = []
+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
for idx, decoder_layer in enumerate(self.model.layers):
+ if block_swap is not None:
+ block_swap.enter(idx)
layer_device, device_index = per_layer_device(decoder_layer)
hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)
residual = hidden_states
@@ -465,6 +468,8 @@ def CohereModel_fast_forward_inference(
residual += hidden_states_attention
residual += hidden_states_mlp
hidden_states = residual
+ if block_swap is not None:
+ block_swap.leave(idx)
next_decoder_cache.append(present_key_value)
hidden_states = fast_layernorm_inference(
diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py
index 543ecf762f2..2175a3fed64 100644
--- a/unsloth/models/falcon_h1.py
+++ b/unsloth/models/falcon_h1.py
@@ -512,8 +512,11 @@ def _FalconH1_fast_forward_inference(
attention_mask = None
next_decoder_cache = []
+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
for idx, decoder_layer in enumerate(self.model.layers):
+ if block_swap is not None:
+ block_swap.enter(idx)
residual.copy_(X)
X = fast_rms_layernorm_inference(
decoder_layer.input_layernorm,
@@ -559,6 +562,8 @@ def _FalconH1_fast_forward_inference(
down_multiplier = down_multiplier,
)
X += residual
+ if block_swap is not None:
+ block_swap.leave(idx)
next_decoder_cache.append(present_key_value)
X = fast_rms_layernorm_inference(
diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py
index 4b227b5e5c0..2df1d8e6c3e 100644
--- a/unsloth/models/gemma.py
+++ b/unsloth/models/gemma.py
@@ -190,7 +190,10 @@ def GemmaModel_fast_forward_inference(
rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)
next_decoder_cache = []
+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
for idx, decoder_layer in enumerate(self.model.layers):
+ if block_swap is not None:
+ block_swap.enter(idx)
layer_device, device_index = per_layer_device(decoder_layer)
hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)
if attention_mask is not None:
@@ -220,6 +223,8 @@ def GemmaModel_fast_forward_inference(
hidden_states = fast_geglu_inference(decoder_layer.mlp, hidden_states)
hidden_states += residual
+ if block_swap is not None:
+ block_swap.leave(idx)
next_decoder_cache.append(present_key_value)
hidden_states = fast_rms_layernorm_inference_gemma(
self.model.norm, hidden_states, out_weights[device_index]
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index a9bd96821b2..562328f6231 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -568,7 +568,10 @@ def Gemma2Model_fast_forward_inference(
SWA = None
GA = None
next_decoder_cache = []
+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
for idx, decoder_layer in enumerate(self.model.layers):
+ if block_swap is not None:
+ block_swap.enter(idx)
# For pipeline parallelism every tensor must be on the same device; this movement happens once per GPU in PP.
layer_device, device_index = per_layer_device(decoder_layer)
hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)
@@ -617,6 +620,8 @@ def Gemma2Model_fast_forward_inference(
)
hidden_states += residual
+ if block_swap is not None:
+ block_swap.leave(idx)
next_decoder_cache.append(present_key_value)
hidden_states = fast_rms_layernorm_inference_gemma(
self.model.norm, hidden_states, out_weights[device_index]
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index ff190178f9b..293378a5e73 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -451,7 +451,10 @@ def GraniteModel_fast_forward_inference(
)
next_decoder_cache = []
+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
for idx, decoder_layer in enumerate(self.model.layers):
+ if block_swap is not None:
+ block_swap.enter(idx)
layer_device, _ = per_layer_device(decoder_layer)
hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)
@@ -475,6 +478,8 @@ def GraniteModel_fast_forward_inference(
)
hidden_states = fast_swiglu_inference(decoder_layer.mlp, hidden_states)
hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)
+ if block_swap is not None:
+ block_swap.leave(idx)
next_decoder_cache.append(present_key_value)
hidden_states = fast_rms_layernorm_inference(self.model.norm, hidden_states)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index ee135948bdb..8cd6fa87fef 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -26,6 +26,7 @@ from ._utils import per_layer_device
from ._utils import embedding_applies_scale
from ._utils import (
_get_inference_mode_context_manager,
+ _offload_embedding_for_room,
_prepare_model_for_qat,
is_bfloat16_supported,
get_quant_type,
@@ -36,6 +37,7 @@ from ._utils import (
)
from .loader_utils import (
DEFAULT_DEVICE_MAP,
+ OFFLOAD_EMBEDDING_AUTO,
_exclude_rope_inv_freq_from_ddp,
_get_fp8_mode_and_check_settings,
_restore_dropped_fp8_scales,
@@ -51,6 +53,7 @@ from .loader_utils import (
planner_quantization_kwargs,
requested_device_map,
resolve_unsloth_device_map,
+ resolve_auto_block_swap,
warn_if_bitsandbytes_quantized_nothing,
)
from ..utils.packing import (
@@ -116,6 +119,12 @@ from ..kernels import *
from ..kernels.utils import has_mxfp4_base
from ..tokenizer_utils import *
from .vision import FastBaseModel, _is_text_seq2seq_config
+from .vision import (
+ _offload_embedding_unsupported_platform,
+ _resolve_offload_embedding,
+ offload_input_embedding,
+ restore_input_embedding,
+)
from transformers.models.llama.modeling_llama import (
LlamaAttention,
@@ -1373,8 +1382,11 @@ def _LlamaModel_fast_forward_inference(
rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)
next_decoder_cache = []
+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
for idx, decoder_layer in enumerate(self.model.layers):
+ if block_swap is not None:
+ block_swap.enter(idx)
layer_device, device_index = per_layer_device(decoder_layer)
X, residual, position_ids = move_to_device(layer_device, X, residual, position_ids)
# self_attn is called directly, so no accelerate hook moves the mask to a split layer's device.
@@ -1414,6 +1426,8 @@ def _LlamaModel_fast_forward_inference(
temp_up = temp_ups[device_index],
)
X += residual
+ if block_swap is not None:
+ block_swap.leave(idx)
next_decoder_cache.append(present_key_value)
X = fast_rms_layernorm_inference(
@@ -2720,6 +2734,34 @@ class FastLlamaModel:
# Respect a user-provided config so it is the single config object used everywhere below; else
# HF gets it again through **kwargs alongside our config= and fails with a duplicate kwarg.
user_config = kwargs.pop("config", None)
+ block_swap_layers = kwargs.pop("block_swap_layers", 0)
+ offload_embedding = kwargs.pop("offload_embedding", False)
+ if offload_embedding and fast_inference:
+ if offload_embedding != OFFLOAD_EMBEDDING_AUTO:
+ print(
+ "Unsloth: Not offloading embeddings; incompatible with fast_inference (vLLM)."
+ )
+ offload_embedding = False
+ _offload_embedding_mode = offload_embedding
+ # None: no memory plan; True / False: the block swap planner decided.
+ _embedding_needed = None
+ if block_swap_layers and kwargs.get("state_dict") is not None:
+ block_swap_layers = refuse_block_swap_load(
+ block_swap_layers,
+ "does not take a state_dict; save it as a safetensors checkpoint and load that.",
+ )
+ if block_swap_layers and kwargs.get("quantization_config") is not None:
+ block_swap_layers = refuse_block_swap_load(
+ block_swap_layers,
+ "does not take a quantization_config; pass load_in_4bit = True instead.",
+ )
+ if block_swap_layers and (
+ kwargs.get("gguf_file") or kwargs.get("use_safetensors") is False
+ ):
+ block_swap_layers = refuse_block_swap_load(
+ block_swap_layers,
+ "needs a safetensors checkpoint; it does not support gguf_file or use_safetensors = False.",
+ )
if user_config is not None:
model_config = user_config
# model_name may have been remapped to a prequantized repo whose checkpoint needs its
@@ -2901,6 +2943,16 @@ class FastLlamaModel:
# checkpoint left unarmed (vLLM reads it itself) must not reach vLLM as a bitsandbytes load.
if not (_explicit_bnb_4bit and _checked_4bit):
load_in_4bit, load_in_8bit = _checked_4bit, _checked_8bit
+ if block_swap_layers and load_in_8bit:
+ block_swap_layers = refuse_block_swap_load(
+ block_swap_layers, "supports 16-bit and 4-bit loads, not load_in_8bit."
+ )
+ if block_swap_layers and _ckpt_quant_method not in (None, "bitsandbytes"):
+ # The host tail is rebuilt as dense or bnb 4-bit layers; packed formats would not survive it.
+ block_swap_layers = refuse_block_swap_load(
+ block_swap_layers,
+ f"does not support {_ckpt_quant_method} checkpoints; use a bitsandbytes or 16-bit checkpoint.",
+ )
from .modelopt_fp8 import (
keep_fp8_scale_names_on_save,
move_config_overrides_onto_config,
@@ -2968,6 +3020,42 @@ class FastLlamaModel:
"load adds, so the plan would size dense weights at 4bit"
)
+ if block_swap_layers and (fast_inference or num_labels is not None):
+ block_swap_layers = refuse_block_swap_load(
+ block_swap_layers, "does not support fast_inference or classification heads."
+ )
+ if block_swap_layers == "auto":
+ block_swap_layers, device_map, _embedding_needed = resolve_auto_block_swap(
+ requested_device_map(device_map),
+ model_name,
+ max_seq_length = max_seq_length,
+ offload_embedding = bool(offload_embedding)
+ and num_labels is None
+ and _offload_embedding_unsupported_platform() is None,
+ planner_kwargs = planner_kwargs_with_max_memory(device_map_planner_kwargs, kwargs),
+ skip_reason = _planner_skip_reason,
+ **planner_config_overrides(kwargs),
+ token = token,
+ trust_remote_code = trust_remote_code,
+ **planner_hub_kwargs(kwargs),
+ revision = revision,
+ **add_dtype_kwargs(dtype),
+ **planner_quantization_kwargs(
+ **compressed_tensors_planner_quantization(
+ model_config,
+ load_in_4bit,
+ load_in_8bit,
+ kwargs.get("quantization_config", None),
+ ),
+ rewritten_quantization_config = modelopt_planner_quantization_config(model_config)
+ if _modelopt_rewritten
+ else fp8_to_nf4_planner_quantization_config(
+ model_config,
+ SKIP_QUANTIZATION_MODULES + (["out_proj"] if IS_FALCON_H1 else []),
+ ),
+ extra_skip_modules = ["out_proj"] if IS_FALCON_H1 else None,
+ ),
+ )
# Here, not in loader.py: the mapper there can still substitute the repo (a -bnb-4bit name
# resolving to its 16-bit twin), so a plan sized for the caller's name is the wrong plan.
device_map = resolve_unsloth_device_map(
@@ -3053,6 +3141,16 @@ class FastLlamaModel:
kwargs = add_dtype_kwargs(dtype, kwargs)
+ if block_swap_layers:
+ import copy as _copy
+
+ # Trim a copy: HF deep-copies config= anyway, so a trimmed caller config would stay short.
+ model_config = _copy.deepcopy(model_config)
+ _block_swap_saved = trim_config_for_block_swap(model_config, block_swap_layers)
+ _undo_block_swap_keys = skip_swapped_checkpoint_keys(
+ _block_swap_saved, model_config.num_hidden_layers
+ )
+
raise_handler = RaiseUninitialized()
try:
if num_labels is not None:
@@ -3133,7 +3231,13 @@ class FastLlamaModel:
_ct_requant = (
getattr(model_config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None) is not None
)
- if user_config is not None or _modelopt_rewritten or _ct_requant or _fp8_to_nf4:
+ if (
+ user_config is not None
+ or _modelopt_rewritten
+ or _ct_requant
+ or _fp8_to_nf4
+ or _block_swap_saved is not None
+ ):
# Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override
# on the config and pass the single config object through.
if max_position_embeddings is not None:
@@ -3252,6 +3356,7 @@ class FastLlamaModel:
model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)
finally:
raise_handler.remove()
+ _undo_block_swap_keys()
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer
if _modelopt_rewritten:
keep_fp8_scale_names_on_save(model)
@@ -3276,6 +3381,38 @@ class FastLlamaModel:
model, tokenizer = patch_tokenizer(model, tokenizer)
model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)
+ attach_block_swap_layers(
+ model,
+ _block_swap_saved,
+ model_name,
+ dtype,
+ load_in_4bit,
+ # The load's own skip list: Falcon-H1 adds out_proj, which its mamba kernels need unquantized.
+ skip_modules = llm_int8_skip_modules if load_in_4bit else SKIP_QUANTIZATION_MODULES,
+ prefetch_depth = planned_prefetch_depth(device_map_planner_kwargs),
+ token = token,
+ revision = revision,
+ cache_dir = kwargs.get("cache_dir"),
+ local_files_only = kwargs.get("local_files_only", False),
+ subfolder = kwargs.get("subfolder"),
+ variant = kwargs.get("variant"),
+ )
+ # "auto" moves it only when memory is tight (block swap plan or free VRAM vs the step reserve); True always does.
+ model._unsloth_offload_embedding_mode = (
+ _offload_embedding_mode if num_labels is None else False
+ )
+ if num_labels is not None:
+ pass
+ elif offload_embedding == OFFLOAD_EMBEDDING_AUTO:
+ if _embedding_needed:
+ _offload_embedding_for_room(model, require_frozen = False)
+ elif _embedding_needed is None:
+ offload_embedding_if_tight(model, max_seq_length, at_load = True)
+ elif offload_embedding:
+ if _resolve_offload_embedding(model, offload_embedding):
+ offload_input_embedding(model)
+ else:
+ _offload_embedding_for_room(model, require_frozen = False)
for idx, layer in enumerate(model.model.layers):
layer.self_attn.apply_qkv = original_apply_qkv
@@ -3514,6 +3651,8 @@ class FastLlamaModel:
qat_scheme = None,
target_parameters = None, # For MoE expert layers (nn.Parameter)
ensure_weight_tying = None, # None = auto (tie when we redirect a tied pair)
+ block_swap_layers = 0,
+ checkpoint_skip_layers = 0,
**kwargs,
):
# The flag reflects the LAST load, not this model.
@@ -3551,6 +3690,8 @@ class FastLlamaModel:
temporary_location = temporary_location,
target_parameters = target_parameters,
ensure_weight_tying = ensure_weight_tying,
+ block_swap_layers = block_swap_layers,
+ checkpoint_skip_layers = checkpoint_skip_layers,
**kwargs,
)
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
@@ -3677,6 +3818,10 @@ class FastLlamaModel:
# apply_unsloth_gradient_checkpointing above already re-patched global state to match (#4735).
model._unsloth_gradient_checkpointing = use_gradient_checkpointing
model = _exclude_rope_inv_freq_from_ddp(model)
+ install_block_swap(
+ model, block_swap_layers, use_gradient_checkpointing = use_gradient_checkpointing
+ )
+ skip_checkpointing(model, checkpoint_skip_layers)
return model
else:
raise TypeError(
@@ -3934,6 +4079,8 @@ class FastLlamaModel:
_saved_temp_tokenizer = model._saved_temp_tokenizer
lora_config = LoraConfig(**arguments)
+ if train_embed_tokens and restore_input_embedding(model):
+ print("Unsloth: Moved the offloaded input embedding back to the GPU to train it.")
input_embeddings_device = model.get_input_embeddings().weight.device
if is_classification:
output_embeddings_device = model.score.weight.device
@@ -3978,6 +4125,11 @@ class FastLlamaModel:
model._saved_temp_tokenizer = _saved_temp_tokenizer
model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)
+ offload_embedding_if_tight(model)
+ install_block_swap(
+ model, block_swap_layers, use_gradient_checkpointing = use_gradient_checkpointing
+ )
+ skip_checkpointing(model, checkpoint_skip_layers)
if ensure_weight_tying:
try:
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 456457165d4..5bc947eb8c5 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -1459,14 +1459,6 @@ class FastLanguageModel(FastLlamaModel):
except Exception as e:
print(f"Unsloth: Could not patch bitsandbytes for torch.compile - {e}")
- # The optimized path never carried offload_embedding, so a request for one is dropped rather than honoured; say so instead of leaving the caller to infer it from memory use. "auto" stays quiet: it promises a decision, and off is one.
- if offload_embedding != OFFLOAD_EMBEDDING_AUTO and offload_embedding:
- print(
- "Unsloth: Not offloading embeddings; the optimized path for this "
- "architecture does not support it. Pass `device_map` or use FastModel "
- "if you need the offload."
- )
-
model, tokenizer = dispatch_model.from_pretrained(
model_name = model_name,
max_seq_length = max_seq_length,
@@ -1491,6 +1483,12 @@ class FastLanguageModel(FastLlamaModel):
max_lora_rank = max_lora_rank,
disable_log_stats = disable_log_stats,
load_in_fp8 = load_in_fp8,
+ # resize_token_embeddings below replaces the embedding and its hooks: only an explicit request offloads.
+ offload_embedding = (
+ False
+ if resize_model_vocab is not None and offload_embedding == OFFLOAD_EMBEDDING_AUTO
+ else offload_embedding
+ ),
*args,
**kwargs,
)
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 296920ea38f..31d88838afc 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -643,6 +643,126 @@ def resolve_unsloth_device_map(
return plan.device_map
+_BLOCK_SWAP_PLANNER_KEYS = (
+ "batch_size",
+ "lora_rank",
+ "reserve_bytes",
+ "prefetch_depth",
+ "rows_per_chunk",
+ "retained_rows",
+ "headroom_bytes",
+ "safety_bytes",
+ "free_space_policy",
+ "no_split_module_classes",
+)
+
+
+def resolve_auto_block_swap(
+ device_map,
+ model_name,
+ *,
+ max_seq_length,
+ offload_embedding = False,
+ planner_kwargs = None,
+ skip_reason = None,
+ placement = "tail",
+ **config_kwargs,
+):
+ """`from_pretrained(block_swap_layers = "auto")`: `(layers, device_map, embedding)`, the trailing
+ decoder layers to build in host RAM so the rest plus a training step's reserve fits, the map to load
+ the rest with, and whether to move the input embedding to host RAM first (one GPU, when
+ `offload_embedding` allows; None when nothing was planned). 0, the map unchanged and no move when
+ everything fits, so nothing slows down. `device_map = "unsloth"` / `"unsloth_balanced"` sizes every
+ card through the multi-GPU planner; anything else sizes the one card the load uses."""
+
+ def _none(reason):
+ print(f"Unsloth: block_swap_layers = 'auto' loads every layer onto the GPU: {reason}.")
+ return 0, device_map, None
+
+ if skip_reason is not None:
+ return _none(skip_reason)
+ if DEVICE_TYPE_TORCH != "cuda" or not torch.cuda.is_available():
+ return _none("block swap needs a CUDA or ROCm GPU")
+ if is_distributed():
+ return _none("each rank of a distributed launch owns its own device")
+ try:
+ from unsloth_zoo.device_map_planner import plan_block_swap
+ except ImportError:
+ return _none("this unsloth_zoo cannot plan block swap")
+
+ planner_kwargs = dict(planner_kwargs or {})
+ requested_memory = planner_kwargs.pop("max_memory", None)
+ devices = []
+ if isinstance(device_map, str) and device_map in _PLANNED_DEVICE_MAPS:
+ devices = (
+ [d for d in requested_memory if isinstance(d, int) and not isinstance(d, bool)]
+ if requested_memory
+ else list(range(torch.cuda.device_count()))
+ )
+ multi = len(devices) > 1
+ if multi:
+ target = None
+ elif devices:
+ target = devices[0]
+ else:
+ if isinstance(device_map, dict) and set(device_map) == {""}:
+ target = torch.device(device_map[""])
+ if target.type != "cuda":
+ return _none(f"the load places the model on {target}")
+ target = target.index if target.index is not None else torch.cuda.current_device()
+ elif device_map is None or (
+ isinstance(device_map, str)
+ and (device_map in TRANSFORMERS_PLACEMENT_STRATEGIES or device_map.startswith("cuda"))
+ ):
+ target = (
+ torch.device(device_map).index
+ if isinstance(device_map, str) and device_map.startswith("cuda:")
+ else torch.cuda.current_device()
+ )
+ else:
+ return _none(
+ "it sizes one GPU or the `device_map = 'unsloth'` planner, not an explicit map"
+ )
+ devices = [target]
+ from ._utils import usable_cuda_bytes
+
+ max_memory = {}
+ for d in devices:
+ free = usable_cuda_bytes(d)
+ cap = _as_bytes((requested_memory or {}).get(d)) if requested_memory else None
+ max_memory[d] = free if cap is None else min(free, cap)
+
+ options = {k: planner_kwargs[k] for k in _BLOCK_SWAP_PLANNER_KEYS if k in planner_kwargs}
+ try:
+ planner_parameters = inspect.signature(plan_block_swap).parameters
+ except (TypeError, ValueError):
+ planner_parameters = {}
+ if offload_embedding and not multi and "offload_embedding" in planner_parameters:
+ options["offload_embedding"] = True
+ if placement != "tail" and not multi:
+ if "placement" not in planner_parameters:
+ return _none("this unsloth_zoo plans only trailing layers")
+ options["placement"] = placement
+ plan = plan_block_swap(
+ model_name,
+ max_memory = max_memory,
+ seq_len = max_seq_length,
+ **options,
+ **config_kwargs,
+ )
+ print(
+ "Unsloth: "
+ + plan.describe().splitlines()[0].replace("block swap:", "block_swap_layers = 'auto':")
+ )
+ embedding = bool(getattr(plan, "offload_embedding", False))
+ if not plan.layers:
+ return 0, device_map, embedding
+ if multi and plan.device_plan is not None:
+ print(plan.device_plan.describe())
+ return plan.layers, plan.device_plan.device_map, embedding
+ return plan.layers, {"": target}, embedding
+
+
def __get_model_name(
model_name,
load_in_4bit = True,
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 21646b3be8a..82f683e7a52 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -98,7 +98,9 @@ from ..kernels import (
from ._utils import (
__version__,
importlib_version,
+ _offload_embedding_for_room,
_prepare_model_for_qat,
+ load_layers_to_host,
resolve_model_class,
resolve_remote_code_model_class,
attention_class_for_load,
@@ -123,6 +125,8 @@ from ._utils import (
set_module_gradient_checkpointing,
)
from ._utils import *
+from ._uma_safetensors import is_integrated_unified_memory_gpu
+from ._utils import estimate_training_reserve_bytes as _zoo_reserve_estimate
from ._remote_code_buffers import restore_remote_code_non_persistent_buffers
from ._custom_dtype import resolve_dtype, trusted_custom_dtype
from .remote_code_shims import apply_remote_code_shims
@@ -146,6 +150,7 @@ from .loader_utils import (
exclude_no_placement_params,
planner_quantization_kwargs,
requested_device_map,
+ resolve_auto_block_swap,
resolve_unsloth_device_map,
warn_if_bitsandbytes_quantized_nothing,
)
@@ -705,6 +710,11 @@ def _install_offload_embedding_hooks(embed_tokens, output_embeddings, return_dev
return output.to(target)
return output
+ # Host lookups and cross-device copies cannot be traced: a compiled caller breaks around them.
+ disable = getattr(getattr(torch, "compiler", None), "disable", None)
+ if disable is not None:
+ _unsloth_offload_pre_hook = disable(_unsloth_offload_pre_hook)
+ _unsloth_offload_post_hook = disable(_unsloth_offload_post_hook)
embed_tokens.register_forward_pre_hook(_unsloth_offload_pre_hook, prepend = True)
embed_tokens.register_forward_hook(_unsloth_offload_post_hook, prepend = True)
embed_tokens._unsloth_offload_hooks_installed = True
@@ -788,9 +798,14 @@ def _embedding_is_worth_offloading(input_embeddings):
return size >= _OFFLOAD_EMBEDDING_MIN_BYTES and size / total >= _OFFLOAD_EMBEDDING_MIN_FRACTION
-def _resolve_offload_embedding(model, offload_embedding):
+def _resolve_offload_embedding(
+ model,
+ offload_embedding,
+ needed = False,
+):
"""Report `offload_embedding` as True only when the offload will really run. It is a VRAM optimisation, not a correctness switch, so turn it off where it cannot help instead of failing the load. It also gates `_attach_bnb_multidevice_hooks`, which must still run whenever no offload happens, so every "no offload" case has to answer False. `"auto"` (the default) decides from the size of the embedding, and the declines below stay silent for it: they explain why something a caller asked for is not happening, and nobody asked for a default."""
automatic = offload_embedding == OFFLOAD_EMBEDDING_AUTO
+ # `needed`: a memory plan wants the room, so "auto" skips the size rule.
def _decline(reason):
if not automatic:
@@ -819,7 +834,132 @@ def _resolve_offload_embedding(model, offload_embedding):
if is_distributed():
# The offload leaves embed_tokens on the CPU while the rest of the rank stays on CUDA; under full finetuning it is trainable, and DDP with device_ids refuses a module whose trainable parameters span both.
return _decline("a distributed launch cannot wrap a model split across CPU and GPU.")
- return _embedding_is_worth_offloading(in_embed) if automatic else True
+ if automatic and (
+ getattr(in_embed, "weight", None) is None or in_embed.weight.device.type in ("cpu", "meta")
+ ):
+ return False
+ if not automatic:
+ return True
+ # "auto" follows free VRAM; an unsloth_zoo without the reserve estimate keeps the size rule.
+ return needed or (_zoo_reserve_estimate is None and _embedding_is_worth_offloading(in_embed))
+
+
+# Other large token tables (Gemma 3n / 4 per-layer embeddings) move too; small position tables stay.
+_EXTRA_EMBEDDING_MIN_BYTES = 256 * 2**20
+
+
+def _input_side_embeddings(model):
+ """The input embedding plus every other large nn.Embedding not shared with the output head."""
+ main = model.get_input_embeddings()
+ out_embed = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None
+ head = getattr(out_embed, "weight", None)
+ found = [main]
+ for module in model.modules():
+ weight = getattr(module, "weight", None)
+ if (
+ module is main
+ or not isinstance(module, torch.nn.Embedding)
+ or weight is None
+ or weight is head
+ or weight.numel() * weight.element_size() < _EXTRA_EMBEDDING_MIN_BYTES
+ ):
+ continue
+ found.append(module)
+ return found, out_embed
+
+
+def _accelerator_device(model):
+ # Headless backbones (AutoModel) have no head to say where the decoder runs.
+ return next(
+ (p.device for p in model.parameters() if p.device.type not in ("cpu", "meta")), None
+ )
+
+
+def offload_input_embedding(model, embeddings = None):
+ """Move the input embeddings to host RAM: lookups run there and only the looked-up rows go back
+ to the decoder's card. Returns the bytes moved."""
+ found, out_embed = _input_side_embeddings(model)
+ head = getattr(out_embed, "weight", None)
+ nbytes = 0
+ for embedding in found if embeddings is None else embeddings:
+ weight = embedding.weight
+ if weight.device.type == "cpu":
+ # Streamed to host by the block swap load: hook it (home = head's card) and move its buffers (embed_scale).
+ if getattr(embedding, "_unsloth_offload_hooks_installed", False):
+ continue
+ _embed_device = head.device if head is not None else _accelerator_device(model)
+ if _embed_device is None:
+ _embed_device = torch.device("cpu")
+ embedding.to("cpu")
+ else:
+ _embed_device = weight.device # decoder device, before offload
+ embedding.to("cpu")
+ nbytes += weight.numel() * weight.element_size()
+ _install_offload_embedding_hooks(embedding, out_embed, _embed_device)
+ print(f"Unsloth: Offloading embeddings to RAM to save {round(nbytes / 1024**3, 2)} GB.")
+ # The transformers model, not a PEFT wrapper: its `device` property is the one callers read.
+ _pin_device_to_decoder(model.get_base_model() if hasattr(model, "get_base_model") else model)
+ # GPU memory must be freed explicitly or it will not be freed.
+ clean_gpu_cache()
+ gc.collect()
+ return nbytes
+
+
+def offload_spare_embeddings(model, require_frozen = True):
+ """offload_embedding = "auto" under memory pressure: move every input-side embedding that
+ lm_head does not share (and, after get_peft_model, that is frozen) to host RAM. A tied input
+ embedding stays, but Gemma 3n / 4 per-layer tables still go. Returns the bytes moved."""
+ if _offload_embedding_unsupported_platform() is not None or is_distributed():
+ return 0
+ try:
+ found, out_embed = _input_side_embeddings(model)
+ except Exception:
+ return 0
+ # Already on the host only via the block swap load (unhooked, decoder on an accelerator); CPU models have nothing to offload.
+ head = getattr(out_embed, "weight", None)
+ if head is not None:
+ accelerated = head.device.type not in ("cpu", "meta")
+ else:
+ accelerated = _accelerator_device(model) is not None
+ candidates = []
+ for embedding in found:
+ weight = getattr(embedding, "weight", None)
+ if (
+ weight is None
+ or weight.device.type == "meta"
+ or (
+ weight.device.type == "cpu"
+ and (
+ not accelerated or getattr(embedding, "_unsloth_offload_hooks_installed", False)
+ )
+ )
+ or _embeddings_are_tied(embedding, out_embed)
+ or _embedding_dispatch_device(embedding) is not None
+ or (require_frozen and any(p.requires_grad for p in embedding.parameters()))
+ ):
+ continue
+ candidates.append(embedding)
+ return offload_input_embedding(model, candidates) if candidates else 0
+
+
+def restore_input_embedding(model):
+ """Undo `offload_input_embedding` for the input embedding (training it needs it on the card).
+ Frozen extra tables stay on the host. The hooks stay and become no-ops."""
+ base = model.get_base_model() if hasattr(model, "get_base_model") else model
+ if not getattr(base, "_unsloth_embedding_offloaded", False):
+ return False
+ embedding = model.get_input_embeddings()
+ out_embed = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None
+ weight = getattr(out_embed, "weight", None)
+ if weight is None or weight.device.type == "cpu":
+ return False
+ if embedding.weight.device.type != "cpu":
+ return False
+ embedding.to(weight.device)
+ base._unsloth_embedding_offloaded = any(
+ e.weight.device.type == "cpu" for e in _input_side_embeddings(model)[0]
+ )
+ return True
VLLM_SUPPORTED_VLM = [
@@ -2487,6 +2627,26 @@ class FastBaseModel:
user_config = kwargs.pop("config", None)
if auto_config is None and user_config is not None:
auto_config = user_config
+ _block_swap_layers = kwargs.pop("block_swap_layers", 0)
+ if _block_swap_layers and load_layers_to_host is None:
+ _block_swap_layers = refuse_block_swap_load(
+ _block_swap_layers,
+ "needs a newer unsloth_zoo (`pip install --upgrade unsloth_zoo`); "
+ "get_peft_model(block_swap_layers = ...) still swaps once the model is on the card.",
+ )
+ if _block_swap_layers and (fast_inference or full_finetuning):
+ _block_swap_layers = refuse_block_swap_load(
+ _block_swap_layers,
+ "streams frozen LoRA base weights, so it cannot be combined with "
+ "fast_inference or full_finetuning.",
+ )
+ if _block_swap_layers and (
+ not torch.cuda.is_available() or is_integrated_unified_memory_gpu()
+ ):
+ _block_swap_layers = refuse_block_swap_load(
+ _block_swap_layers,
+ "needs a discrete CUDA or ROCm GPU; unified memory has no separate RAM to load into.",
+ )
# Offline snapshot for the loads below; not popped, so the weight load still reads local_files_only from **kwargs. See _get_effective_local_files_only.
local_files_only = _get_effective_local_files_only(kwargs)
@@ -2932,6 +3092,54 @@ class FastBaseModel:
raise RuntimeError(
"Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!"
)
+ if _block_swap_layers and block_swap_load_device(device_map) is None:
+ _block_swap_layers = refuse_block_swap_load(
+ _block_swap_layers,
+ "needs every layer on one GPU; pass block_swap_layers to get_peft_model instead.",
+ )
+ _embedding_needed = None
+ if _block_swap_layers == "auto":
+ _block_swap_layers, device_map, _embedding_needed = resolve_auto_block_swap(
+ requested_device_map(device_map),
+ model_name,
+ max_seq_length = max_seq_length,
+ # Windows / WSL decline the offload later: a plan counting on it would leave a streamed table unhooked.
+ offload_embedding = bool(offload_embedding)
+ and _offload_embedding_unsupported_platform() is None,
+ planner_kwargs = planner_kwargs_with_max_memory(device_map_planner_kwargs, kwargs),
+ placement = "spread",
+ skip_reason = _planner_skip_reason
+ or (
+ "the planner cannot rebuild this checkpoint's prepared config"
+ if _planner_config is not None
+ else None
+ ),
+ **planner_config_overrides(kwargs),
+ token = token,
+ trust_remote_code = trust_remote_code,
+ **planner_hub_kwargs(kwargs),
+ revision = _revision,
+ **add_dtype_kwargs(torch_dtype),
+ **planner_quantization_kwargs(
+ **compressed_tensors_planner_quantization(
+ auto_config, load_in_4bit, load_in_8bit, user_quantization_config
+ ),
+ rewritten_quantization_config = modelopt_planner_quantization_config(
+ auto_config, dequantize = load_in_16bit
+ )
+ if _modelopt_rewritten
+ else fp8_to_nf4_planner_quantization_config(
+ auto_config,
+ SKIP_QUANTIZATION_MODULES + _architecture_skip_modules(model_types),
+ ),
+ extra_skip_modules = _architecture_skip_modules(model_types) or None,
+ ),
+ )
+ _block_swap_device = block_swap_load_device(device_map)
+ if (_block_swap_layers or _embedding_needed) and _block_swap_device is not None:
+ # A placement strategy would spill the host-bound weights to the CPU, which 4-bit loads refuse.
+ device_map = {"": _block_swap_device}
+ _block_swap_state = None
# Prefetch the repo (killable child) so the in-process load is a cache hit. vLLM owns the download only when actually available; if it was requested but is missing, the load falls through in-process, so warm the weights here.
_vllm_owns_weights = fast_inference and is_vLLM_available()
@@ -3120,6 +3328,8 @@ class FastBaseModel:
raise_handler = RaiseUninitialized()
try:
+ # get_peft_model(block_swap_layers = "auto") may move it later, unless the caller said no.
+ _offload_embedding_mode = False if fast_inference else offload_embedding
if offload_embedding and fast_inference:
if offload_embedding != OFFLOAD_EMBEDDING_AUTO:
print(
@@ -3149,14 +3359,17 @@ class FastBaseModel:
)
):
try:
- model = auto_model.from_pretrained(
- model_name,
- config = model_config,
- device_map = device_map,
- token = token,
- trust_remote_code = trust_remote_code,
- **kwargs,
- )
+ with begin_block_swap_load(
+ _block_swap_layers, device_map, embeddings = bool(_embedding_needed)
+ ) as (_block_swap_state):
+ model = auto_model.from_pretrained(
+ model_name,
+ config = model_config,
+ device_map = device_map,
+ token = token,
+ trust_remote_code = trust_remote_code,
+ **kwargs,
+ )
finally:
# The load deep-copied the config; give the caller's object its fp8 block back.
disarm_fp8_to_nf4(model_config)
@@ -3182,13 +3395,15 @@ class FastBaseModel:
offload_embedding,
)
- _attach_bnb_multidevice_hooks(
- model,
- load_in_4bit = load_in_4bit,
- load_in_8bit = load_in_8bit,
- offload_embedding = offload_embedding,
- fast_inference = fast_inference,
- )
+ # One card by construction: the layers in host RAM are not an offloaded dispatch.
+ if _block_swap_state is None:
+ _attach_bnb_multidevice_hooks(
+ model,
+ load_in_4bit = load_in_4bit,
+ load_in_8bit = load_in_8bit,
+ offload_embedding = offload_embedding,
+ fast_inference = fast_inference,
+ )
_no_placement_hooked = _hook_no_placement_ancestors(model)
if _no_placement_hooked:
logger.info(
@@ -3230,24 +3445,17 @@ class FastBaseModel:
if hasattr(model, "generate"):
model.fast_generate = make_fast_generate_wrapper(model.generate)
model.fast_generate_batches = error_out_no_vllm
+ # "auto" moves it only when memory is tight, here and again in get_peft_model.
+ model._unsloth_offload_embedding_mode = _offload_embedding_mode
if offload_embedding:
- embed_tokens = model.get_input_embeddings()
- out_embed = (
- model.get_output_embeddings()
- if hasattr(model, "get_output_embeddings")
- else None
- )
- nbytes = embed_tokens.weight.numel() * embed_tokens.weight.itemsize
- ngb = round(nbytes / 1024 / 1024 / 1024, 2)
- print(f"Unsloth: Offloading embeddings to RAM to save {ngb} GB.")
- _embed_device = embed_tokens.weight.device # decoder device, before offload
- embed_tokens.to("cpu")
-
- _install_offload_embedding_hooks(embed_tokens, out_embed, _embed_device)
- _pin_device_to_decoder(model)
- # GPU memory must be freed explicitly or it will not be freed.
- clean_gpu_cache()
- gc.collect()
+ offload_input_embedding(model)
+ elif _offload_embedding_mode == OFFLOAD_EMBEDDING_AUTO and _embedding_needed:
+ _offload_embedding_for_room(model, require_frozen = False)
+ elif _offload_embedding_mode == OFFLOAD_EMBEDDING_AUTO:
+ offload_embedding_if_tight(model, max_seq_length, at_load = True)
+ elif _offload_embedding_mode:
+ # A tied input embedding stays; untied per-layer tables still move.
+ offload_spare_embeddings(model, require_frozen = False)
else:
from unsloth_zoo.vllm_utils import (
load_vllm,
@@ -3695,9 +3903,20 @@ class FastBaseModel:
_mark_loaded_revision(tokenizer, _tokenizer_revision)
model = _mark_forced_float32(model, do_forced_float32)
model = _mark_full_finetuning(model, full_finetuning)
+ # Last, so the host copies carry the fp32 recasts the passes above make.
+ finish_block_swap_load(
+ model, _block_swap_state, planned_prefetch_depth(device_map_planner_kwargs)
+ )
# LAST, like the llama loader: patch_model_and_tokenizer below REPLACES the embedding and lm_head with fresh modules carrying the weights but not the _hf_hook, so a model split across cards would still meet the original cross-device index_select. Idempotent.
- if not fast_inference and not offload_embedding:
+ # Embeddings offloaded later sit on the host on purpose: a dispatch hook would pull them back.
+ _base = model.get_base_model() if hasattr(model, "get_base_model") else model
+ if (
+ not fast_inference
+ and not offload_embedding
+ and not getattr(_base, "_unsloth_embedding_offloaded", False)
+ and _block_swap_state is None
+ ):
try:
_repaired = _repair_dispatch_hooks(model)
if _repaired:
@@ -3740,6 +3959,8 @@ class FastBaseModel:
target_parameters = None, # For MoE expert layers (nn.Parameter)
ensure_weight_tying = None, # None = auto (tie when we redirect a tied pair)
finetune_audio_layers = False, # placed last to preserve existing positional argument order
+ block_swap_layers = 0,
+ checkpoint_skip_layers = 0,
**kwargs,
):
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
@@ -3842,6 +4063,9 @@ class FastBaseModel:
)
modules_to_save = _scope_modules_to_save_to_core(model, modules_to_save)
_raise_if_fast_inference_modules_to_save(model, modules_to_save)
+ if any(str(m).split(".")[-1] == "embed_tokens" for m in (modules_to_save or ())):
+ if restore_input_embedding(model):
+ print("Unsloth: Moved the offloaded input embedding back to the GPU to train it.")
# Only a regex generated here may be widened to expert submodules below.
_target_modules_auto_regex = False
@@ -4107,6 +4331,11 @@ class FastBaseModel:
model.max_seq_length = max_seq_length
for module in model.modules():
module.max_seq_length = max_seq_length
+ offload_embedding_if_tight(model)
+ install_block_swap(
+ model, block_swap_layers, use_gradient_checkpointing = use_gradient_checkpointing
+ )
+ skip_checkpointing(model, checkpoint_skip_layers)
for _ in range(3):
gc.collect()
clean_gpu_cache()