Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block Swap: Stream frozen transformer blocks from host RAM so a dense model can train past the card's VRAM (#11832)
Leo Borcherding79116720a3f2
diff --git a/tests/test_block_swap_install.py b/tests/test_block_swap_install.pynew file mode 100644index 00000000000..b8bdbfd7693--- /dev/null+++ b/tests/test_block_swap_install.py@@ -0,0 +1,690 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Block swap helpers from _utils.py, extracted with ast to avoid importing torch's CUDA stack."""++import ast, os, types+import pytest++HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py")+NAMES = (+ "_check_block_swap",+ "_new_block_swap",+ "refuse_block_swap_load",+ "_training_reserve_bytes",+ "_auto_block_swap_indices",+ "install_block_swap",+ "trim_config_for_block_swap",+)+++def _load(+ *,+ moe = False,+ integrated = False,+ zoo = True,+ cuda = True,+ auto_pick = ([], 0),+):+ src = open(UTILS, encoding = "utf-8").read()+ mod = ast.parse(src)+ nodes = {n.name: n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name in NAMES}+ assert set(nodes) == set(NAMES), f"missing from _utils.py: {set(NAMES) - set(nodes)}"++ calls = []++ class FakeSwap:+ def __init__(+ self,+ layers,+ n,+ depth,+ placement = "tail",+ ):+ calls.append((layers, n, depth, placement))++ ns = {+ "is_moe_model": lambda m: moe,+ "is_integrated_unified_memory_gpu": lambda: integrated,+ "BlockSwap": FakeSwap if zoo else None,+ "build_host_layers": object() if zoo else None,+ "find_decoder_layers": lambda m: m.layers,+ "auto_swap_indices": lambda layers, reserve, depth: auto_pick,+ "_offload_embedding_for_room": lambda model: False,+ "estimate_training_reserve_bytes": lambda config, seq_len, extra_bytes = 0: 2**30,+ "torch": types.SimpleNamespace(cuda = types.SimpleNamespace(is_available = lambda: cuda)),+ }+ for name in NAMES:+ exec(ast.get_source_segment(src, nodes[name]), ns)+ return ns, calls+++class _Layers(list):+ pass+++class _Model:+ config = None++ def __init__(self, vllm = None):+ self.layers = _Layers(["L0", "L1", "L2"])+ if vllm is not None:+ self.vllm_engine = vllm++ def parameters(self):+ return []+++def test_zero_is_a_no_op():+ ns, calls = _load()+ m = _Model()+ assert ns["install_block_swap"](m, 0) is None+ assert ns["install_block_swap"](m, None) is None+ assert calls == [] and not hasattr(m, "_unsloth_block_swap")+++def test_happy_path_installs_and_records():+ ns, calls = _load()+ m = _Model()+ sw = ns["install_block_swap"](m, 2, prefetch_depth = 3)+ assert calls == [(m.layers, 2, 3, "spread")]+ assert m._unsloth_block_swap is sw+ assert m.layers._unsloth_block_swap is sw+++def test_refuses_vllm():+ ns, calls = _load()+ with pytest.raises(ValueError, match = "fast_inference"):+ ns["install_block_swap"](_Model(vllm = object()), 2)+ assert calls == []+++def test_moe_installs_with_a_notice(capsys):+ ns, calls = _load(moe = True)+ ns["install_block_swap"](_Model(), 2)+ assert len(calls) == 1 and "MoE" in capsys.readouterr().out+++def test_refuses_unified_memory():+ ns, calls = _load(integrated = True)+ with pytest.raises(ValueError, match = "unified-memory"):+ ns["install_block_swap"](_Model(), 2)+ assert calls == []+++def test_refuses_without_gradient_checkpointing():+ ns, calls = _load()+ for off in (False, None):+ with pytest.raises(ValueError, match = "use_gradient_checkpointing"):+ ns["install_block_swap"](_Model(), 2, use_gradient_checkpointing = off)+ assert calls == []+++def test_old_zoo_is_an_import_error():+ ns, calls = _load(zoo = False)+ with pytest.raises(ImportError, match = "unsloth_zoo"):+ ns["install_block_swap"](_Model(), 2)+ assert calls == []+++def test_refusal_order_checks_cheap_things_first():+ ns, _ = _load(integrated = True)+ with pytest.raises(ValueError, match = "fast_inference"):+ ns["install_block_swap"](_Model(vllm = object()), 2)+++def test_swapper_from_load_is_reused():+ ns, calls = _load()+ m = _Model()+ m._unsloth_block_swap = existing = object()+ assert ns["install_block_swap"](m, 0) is existing+ assert ns["install_block_swap"](m, 8) is existing+ assert calls == []+ with pytest.raises(ValueError, match = "use_gradient_checkpointing"):+ ns["install_block_swap"](m, 0, use_gradient_checkpointing = False)+++class _Config:+ def __init__(+ self,+ n = 8,+ **extra,+ ):+ self.num_hidden_layers = n+ self.layer_types = ["full_attention"] * n+ self.other_list = [1, 2, 3]+ self.__dict__.update(extra)+++def test_trim_off_at_zero():+ ns, _ = _load()+ cfg = _Config()+ assert ns["trim_config_for_block_swap"](cfg, 0) is None+ assert cfg.num_hidden_layers == 8 and len(cfg.layer_types) == 8+++def test_trim_shortens_per_layer_lists_and_returns_originals():+ ns, _ = _load()+ cfg = _Config()+ saved = ns["trim_config_for_block_swap"](cfg, 5)+ assert cfg.num_hidden_layers == 3 and cfg.layer_types == ["full_attention"] * 3+ assert cfg.other_list == [1, 2, 3], "lists not sized to the layer count are untouched"+ assert saved == {"num_hidden_layers": 8, "layer_types": ["full_attention"] * 8}+++def test_trim_keeps_at_least_one_layer_on_the_card():+ ns, _ = _load()+ cfg = _Config()+ ns["trim_config_for_block_swap"](cfg, 100)+ assert cfg.num_hidden_layers == 1+++def test_trim_refuses_what_install_refuses():+ ns, _ = _load(integrated = True)+ with pytest.raises(ValueError, match = "unified-memory"):+ ns["trim_config_for_block_swap"](_Config(), 4)+ ns, _ = _load(zoo = False)+ with pytest.raises(ImportError, match = "unsloth_zoo"):+ ns["trim_config_for_block_swap"](_Config(), 4)+++def _refusals(fn, needle):+ """Lines of `if <needle ...>: block_swap_layers = refuse_block_swap_load(...)` in `fn`."""+ return [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.If)+ and needle in ast.unparse(n.test) + ast.unparse(n.body)+ and "refuse_block_swap_load" in ast.unparse(n.body)+ ]+++def _peft_signature_and_calls(path):+ mod = ast.parse(open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read())+ for cls in (n for n in mod.body if isinstance(n, ast.ClassDef)):+ for fn in cls.body:+ if isinstance(fn, ast.FunctionDef) and fn.name == "get_peft_model":+ return fn+ raise AssertionError(f"get_peft_model not found in {path}")++[email protected]("path", ["llama.py", "vision.py"])+def test_block_swap_layers_is_last_so_positional_callers_keep_their_slots(path):+ fn = _peft_signature_and_calls(path)+ assert [a.arg for a in fn.args.args[-2:]] == ["block_swap_layers", "checkpoint_skip_layers"]+ assert fn.args.kwarg is not None+++def test_new_model_route_forwards_block_swap_layers():+ fn = _peft_signature_and_calls("llama.py")+ forwarded = [+ call+ for call in ast.walk(fn)+ if isinstance(call, ast.Call)+ and isinstance(call.func, ast.Attribute)+ and call.func.attr == "get_peft_model"+ and getattr(call.func.value, "id", None) == "FastBaseModel"+ ]+ assert forwarded, "llama get_peft_model no longer delegates to FastBaseModel"+ for call in forwarded:+ assert {"block_swap_layers", "checkpoint_skip_layers"} <= {k.arg for k in call.keywords}++[email protected]("path", ["llama.py", "gemma.py", "gemma2.py"])+def test_fast_decode_loops_fetch_swapped_layers(path):+ # Decode loops read layer weights without calling the layer, so the swap hooks never fire.+ src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()+ assert "block_swap.enter(idx)" in src and "block_swap.leave(idx)" in src+++def _load_key_filter(monkeypatch, has_method = True):+ import re, sys, types++ mod = ast.parse(open(UTILS, encoding = "utf-8").read())+ keep = [+ n+ for n in mod.body+ if (isinstance(n, ast.FunctionDef) and n.name == "skip_swapped_checkpoint_keys")+ or (+ isinstance(n, ast.Assign)+ and any(getattr(t, "id", None) == "_SWAPPED_LAYER_KEY" for t in n.targets)+ )+ ]+ assert len(keep) == 2+ seen = []++ class PreTrainedModel:+ pass++ if has_method:++ def _get_key_renaming_mapping(+ self,+ checkpoint_keys,+ key_mapping = None,+ ):+ seen.append(list(checkpoint_keys))+ return {k: k for k in checkpoint_keys}++ PreTrainedModel._get_key_renaming_mapping = _get_key_renaming_mapping+ fake = types.ModuleType("transformers.modeling_utils")+ fake.PreTrainedModel = PreTrainedModel+ monkeypatch.setitem(sys.modules, "transformers.modeling_utils", fake)+ ns = {"re": re}+ exec(compile(ast.Module(body = keep, type_ignores = []), UTILS, "exec"), ns)+ return ns["skip_swapped_checkpoint_keys"], PreTrainedModel, seen+++def test_swapped_tail_keys_are_hidden_from_a_4x_load_then_restored(monkeypatch):+ skip, cls, seen = _load_key_filter(monkeypatch)+ original = cls.__dict__["_get_key_renaming_mapping"]+ undo = skip({"num_hidden_layers": 4}, 2)+ keys = [+ "model.embed_tokens.weight",+ "model.layers.1.mlp.down_proj.weight",+ "model.layers.2.mlp.down_proj.weight",+ "model.layers.3.self_attn.q_proj.weight.absmax",+ "lm_head.weight",+ ]+ cls()._get_key_renaming_mapping(keys, key_mapping = None)+ assert seen[-1] == [+ "model.embed_tokens.weight",+ "model.layers.1.mlp.down_proj.weight",+ "lm_head.weight",+ ]+ undo()+ assert cls.__dict__["_get_key_renaming_mapping"] is original+++def test_key_filter_is_inert_when_off_or_on_5x(monkeypatch):+ skip, cls, _ = _load_key_filter(monkeypatch)+ original = cls.__dict__["_get_key_renaming_mapping"]+ skip(None, 2)()+ assert cls.__dict__["_get_key_renaming_mapping"] is original+ skip, cls, _ = _load_key_filter(monkeypatch, has_method = False)+ skip({"num_hidden_layers": 4}, 2)()+ assert "_get_key_renaming_mapping" not in cls.__dict__+++def test_checkpoint_tensors_reads_the_requested_variant(tmp_path):+ torch = pytest.importorskip("torch")+ safetensors_torch = pytest.importorskip("safetensors.torch")+ mod = ast.parse(open(UTILS, encoding = "utf-8").read())+ fn = next(+ n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name == "_checkpoint_tensors"+ )+ ns = {"os": os}+ exec(compile(ast.Module(body = [fn], type_ignores = []), UTILS, "exec"), ns)+ safetensors_torch.save_file({"w": torch.zeros(2)}, str(tmp_path / "model.safetensors"))+ safetensors_torch.save_file({"w": torch.ones(2)}, str(tmp_path / "model.fp16.safetensors"))+ tensors, _ = ns["_checkpoint_tensors"](str(tmp_path), variant = "fp16")+ assert torch.equal(tensors["w"](), torch.ones(2))+ tensors, _ = ns["_checkpoint_tensors"](str(tmp_path))+ assert torch.equal(tensors["w"](), torch.zeros(2))+++def test_non_safetensors_formats_are_refused_before_the_prefix_loads():+ mod = ast.parse(+ open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()+ )+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")+ refusal = _refusals(fn, "needs a safetensors checkpoint")+ trim = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "trim_config_for_block_swap"+ ]+ assert refusal and trim and refusal[0] < trim[0]+++def test_refuses_without_a_cuda_or_rocm_gpu():+ ns, calls = _load(cuda = False)+ with pytest.raises(ValueError, match = "CUDA or ROCm"):+ ns["install_block_swap"](_Model(), 4)+ assert not calls+++def test_prewrapped_peft_and_quantized_checkpoints_are_covered():+ src = open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()+ mod = ast.parse(src)+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")+ peft = next(+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "get_peft_model"+ )+ installs = [+ n+ for n in ast.walk(peft)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "install_block_swap"+ ]+ assert len(installs) >= 2, "the pre-wrapped PEFT return path must install the swap too"+ load = next(+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained"+ )+ assert _refusals(load, "_ckpt_quant_method")+++def test_every_custom_decode_loop_is_served():+ # A FastXModel with its own model-level decode loop must call block_swap.enter/leave.+ models = os.path.join(HERE, "unsloth", "models")+ for name in sorted(os.listdir(models)):+ if not name.endswith(".py"):+ continue+ src = open(os.path.join(models, name), encoding = "utf-8").read()+ loops = [+ n+ for n in ast.walk(ast.parse(src))+ if isinstance(n, ast.FunctionDef)+ and n.name.endswith("fast_forward_inference")+ and "Attention" not in n.name+ ]+ if loops:+ assert (+ "block_swap" in src+ ), f"{name} has a decode loop that never fetches swapped blocks"+++def test_a_caller_quantization_config_is_refused_before_loading():+ mod = ast.parse(+ open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()+ )+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")+ raises = _refusals(fn, "quantization_config")+ trim = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "trim_config_for_block_swap"+ ]+ assert raises and trim and raises[0] < trim[0]+++def test_every_unsupported_load_mode_is_refused_before_trimming():+ mod = ast.parse(+ open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()+ )+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")+ trim = min(+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "trim_config_for_block_swap"+ )+ for needle in (+ "load_in_8bit",+ "quantization_config",+ "state_dict",+ "_ckpt_quant_method",+ "safetensors",+ ):+ hits = _refusals(fn, needle)+ assert hits and min(hits) < trim, needle+++def test_older_zoo_without_placement_still_installs():+ ns, calls = _load()++ class OldSwap:+ def __init__(self, layers, n, depth):+ calls.append((layers, n, depth))++ ns["BlockSwap"] = OldSwap+ m = _Model()+ ns["install_block_swap"](m, 2)+ assert calls == [(m.layers, 2, 2)]+++def test_auto_swaps_nothing_when_the_card_has_room(capsys):+ ns, calls = _load(auto_pick = ([], 0))+ assert ns["install_block_swap"](_Model(), "auto") is None+ assert calls == [] and "swaps nothing" in capsys.readouterr().out+++def test_auto_swaps_the_layers_the_shortfall_needs():+ ns, calls = _load(auto_pick = ([0, 2], 0))+ m = _Model()+ ns["install_block_swap"](m, "auto")+ assert calls == [(m.layers, [0, 2], 2, "spread")]+++def test_auto_without_a_gpu_is_a_quiet_no_op():+ ns, calls = _load(cuda = False)+ assert ns["install_block_swap"](_Model(), "auto") is None+ assert calls == []+++def test_load_refusals_fall_back_for_auto_and_raise_for_a_count(capsys):+ ns, _ = _load()+ assert ns["refuse_block_swap_load"]("auto", "does not take a state_dict.") == 0+ assert "loads every layer onto the GPU" in capsys.readouterr().out+ with pytest.raises(ValueError, match = "state_dict"):+ ns["refuse_block_swap_load"](4, "does not take a state_dict.")+++def test_auto_moves_the_embedding_before_any_layer():+ ns, calls = _load()+ picks = iter([([0, 2], 0), ([], 0)])+ ns["auto_swap_indices"] = lambda layers, reserve, depth: next(picks)+ moved = []+ ns["_offload_embedding_for_room"] = lambda model: moved.append(model) or True+ m = _Model()+ assert ns["install_block_swap"](m, "auto") is None+ assert moved == [m] and calls == []+++def _load_host_helpers(device_count = 1, cuda = True):+ src = open(UTILS, encoding = "utf-8").read()+ mod = ast.parse(src)+ names = ("block_swap_load_device", "begin_block_swap_load")+ nodes = {n.name: n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name in names}+ import contextlib, torch++ seen = []+ ns = {+ "DEVICE_TYPE_TORCH": "cuda",+ "contextlib": contextlib,+ "load_layers_to_host": lambda n, **kw: seen.append((n, kw))+ or contextlib.nullcontext("loading"),+ "torch": types.SimpleNamespace(+ device = torch.device,+ cuda = types.SimpleNamespace(+ is_available = lambda: cuda,+ device_count = lambda: device_count,+ current_device = lambda: 0,+ ),+ ),+ }+ for name in names:+ exec(ast.get_source_segment(src, nodes[name]), ns)+ return ns, seen+++ "device_map, count, expected",+ [+ ({"": 0}, 2, 0),+ ({"": "cuda:1"}, 2, 1),+ ("cuda:1", 2, 1),+ ("sequential", 1, 0),+ ("sequential", 2, None), # a strategy would split across the two cards+ ({"a": 0, "b": 1}, 2, None),+ ({"": "cpu"}, 1, None),+ (None, 1, 0),+ ],+)+def test_host_load_needs_one_card(device_map, count, expected):+ ns, _ = _load_host_helpers(device_count = count)+ assert ns["block_swap_load_device"](device_map) == expected+++def test_host_load_without_cuda_has_no_card():+ ns, _ = _load_host_helpers(cuda = False)+ assert ns["block_swap_load_device"]({"": 0}) is None+++def test_host_load_runs_for_layers_or_streamed_embeddings_only():+ ns, seen = _load_host_helpers()+ with ns["begin_block_swap_load"](0, {"": 0}) as state:+ assert state is None+ with ns["begin_block_swap_load"](0, {"": 0}, embeddings = True):+ pass+ with ns["begin_block_swap_load"](3, {"": 0}):+ pass+ assert seen == [(0, {"placement": "spread", "embeddings": True}), (3, {"placement": "spread"})]+++def _load_skip():+ src = open(UTILS, encoding = "utf-8").read()+ mod = ast.parse(src)+ names = ("_skip_aware_flag", "skip_checkpointing")+ ns = {"find_decoder_layers": lambda m: m.layers}+ for n in mod.body:+ if isinstance(n, ast.FunctionDef) and n.name in names:+ exec(ast.get_source_segment(src, n), ns)+ return ns+++def _skip_model(n = 4, swapped = None):+ class Layer:+ gradient_checkpointing = False++ model = types.SimpleNamespace(layers = [Layer() for _ in range(n)])+ for layer in model.layers:+ layer.gradient_checkpointing = True+ if swapped is not None:+ model._unsloth_block_swap = types.SimpleNamespace(indices = swapped)+ return model+++def test_skipped_layers_stay_off_when_checkpointing_is_turned_back_on():+ ns = _load_skip()+ model = _skip_model()+ assert ns["skip_checkpointing"](model, 2) == [1, 3]+ # for_training / gradient_checkpointing_enable write the flag on every layer again.+ for layer in model.layers:+ layer.gradient_checkpointing = True+ assert [l.gradient_checkpointing for l in model.layers] == [True, False, True, False]+ for layer in model.layers:+ layer.gradient_checkpointing = False+ assert not any(l.gradient_checkpointing for l in model.layers)+++def test_skip_leaves_block_swapped_layers_checkpointed():+ ns = _load_skip()+ model = _skip_model(4, swapped = [0, 2])+ assert ns["skip_checkpointing"](model, "max") == [1, 3]+ assert [l.gradient_checkpointing for l in model.layers] == [True, False, True, False]+++def test_skip_zero_changes_nothing():+ ns = _load_skip()+ model = _skip_model()+ assert ns["skip_checkpointing"](model, 0) == []+ assert all(l.gradient_checkpointing for l in model.layers)+++def _method(path, cls_name, name):+ mod = ast.parse(open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read())+ cls = next(n for n in mod.body if isinstance(n, ast.ClassDef) and n.name == cls_name)+ return next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == name)+++def test_generic_load_refuses_unified_memory_before_loading_to_host():+ fn = _method("vision.py", "FastBaseModel", "from_pretrained")+ begin = min(+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "begin_block_swap_load"+ )+ hits = _refusals(fn, "is_integrated_unified_memory_gpu")+ assert hits and min(hits) < begin+++def test_headless_load_swaps_onto_the_retained_layers_card():+ torch = pytest.importorskip("torch")+ mod = ast.parse(open(UTILS, encoding = "utf-8").read())+ fn = next(+ n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name == "finish_block_swap_load"+ )+ built = []+ card = torch.device("cuda", 1)+ ns = {+ "torch": types.SimpleNamespace(+ device = torch.device, cuda = types.SimpleNamespace(current_device = lambda: 0)+ ),+ "_new_block_swap": lambda layers, idx, depth, device, placement: built.append(device)+ or object(),+ }+ exec(compile(ast.Module(body = [fn], type_ignores = []), UTILS, "exec"), ns)++ class _Layer:+ def __init__(self, device):+ self.device = device++ def parameters(self):+ return [types.SimpleNamespace(device = self.device)]++ class _Headless:+ def get_output_embeddings(self):+ return None++ layers = _Layers([_Layer(card), _Layer(torch.device("cpu")), _Layer(torch.device("cpu"))])+ ns["finish_block_swap_load"](_Headless(), types.SimpleNamespace(layers = layers, indices = [1, 2]))+ assert built == [card]++[email protected]("path", ["llama.py", "vision.py"])+def test_auto_plan_never_counts_on_an_embedding_offload_the_platform_refuses(path):+ src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()+ call = next(+ n+ for n in ast.walk(ast.parse(src))+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "resolve_auto_block_swap"+ )+ kw = next(k for k in call.keywords if k.arg == "offload_embedding")+ assert "_offload_embedding_unsupported_platform" in ast.get_source_segment(src, kw.value)+++def test_load_time_swappers_allocate_the_planned_prefetch_depth():+ torch = pytest.importorskip("torch")+ mod = ast.parse(open(UTILS, encoding = "utf-8").read())+ fns = [+ n+ for n in mod.body+ if isinstance(n, ast.FunctionDef)+ and n.name in ("finish_block_swap_load", "planned_prefetch_depth")+ ]+ built = []+ ns = {+ "torch": types.SimpleNamespace(+ device = torch.device, cuda = types.SimpleNamespace(current_device = lambda: 0)+ ),+ "_new_block_swap": lambda layers, idx, depth, device, placement: built.append(depth)+ or object(),+ }+ exec(compile(ast.Module(body = fns, type_ignores = []), UTILS, "exec"), ns)++ class _Headless:+ def get_output_embeddings(self):+ return None++ depth = ns["planned_prefetch_depth"]({"prefetch_depth": 1})+ ns["finish_block_swap_load"](+ _Headless(), types.SimpleNamespace(layers = _Layers([]), indices = [0]), depth+ )+ assert built == [1] and ns["planned_prefetch_depth"](None) == 2+ # Both load paths hand the planner's depth to the swapper they build.+ for path, call in (+ ("llama.py", "attach_block_swap_layers"),+ ("vision.py", "finish_block_swap_load"),+ ):+ src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()+ node = next(+ n+ for n in ast.walk(ast.parse(src))+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == call+ )+ assert "planned_prefetch_depth(device_map_planner_kwargs)" in ast.get_source_segment(+ src, node+ )diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.pyindex 4612ef3d0e0..f99eb574973 100644--- a/tests/test_modelopt_fp8_loading.py+++ b/tests/test_modelopt_fp8_loading.py@@ -549,9 +549,12 @@ def test_config_branch_moves_rope_extension_onto_the_config():import inspectfrom unsloth.models import llama+ import re+source = inspect.getsource(llama.FastLlamaModel.from_pretrained)- branch = source.split("if user_config is not None or _modelopt_rewritten", 1)[1]- branch = branch.split("AutoModelForCausalLM.from_pretrained(", 1)[0]+ # The condition may wrap over several lines once it grows.+ start = re.search(r"if \(?\s*user_config is not None\s+or _modelopt_rewritten", source)+ branch = source[start.end() :].split("AutoModelForCausalLM.from_pretrained(", 1)[0]assert 'kwargs.pop("rope_scaling", None)' in branchdiff --git a/tests/test_offload_tied_autodisable.py b/tests/test_offload_tied_autodisable.pyindex 3b3720b4a47..d0e394c7d05 100644--- a/tests/test_offload_tied_autodisable.py+++ b/tests/test_offload_tied_autodisable.py@@ -10,10 +10,11 @@ Every platform branch is driven explicitly, so the assertions hold on Linux,macOS, Windows and WSL alike: the host's own os.name never decides."""-import ast, os+import ast, os, typesfrom contextlib import contextmanagerimport pytest+from real_accelerator import has_real_cuda# Skip rather than error where torch is absent. Only `nn.Embedding` / `nn.Linear` / `torch.device` are wanted here, no# GPU, but a bare module-level import turns a machine without torch into a collection error, which aborts the whole@@ -41,6 +42,8 @@ def _load(*names):"os": os,"OFFLOAD_EMBEDDING_AUTO": "auto","is_distributed": lambda: _DISTRIBUTED[0],+ # An unsloth_zoo without the reserve estimate: "auto" keeps the size rule tested below.+ "_zoo_reserve_estimate": None,}wanted = set(names)for node in mod.body:@@ -185,7 +188,7 @@ def test_resolved_before_multidevice_hooks():# Hook attach returns early while offload_embedding is still True.call = _SRC.index("offload_embedding = _resolve_offload_embedding(")# Anchor on the indented CALL, not the module-level `def`.- hooks = _SRC.index("\n _attach_bnb_multidevice_hooks(")+ hooks = _SRC.index("\n _attach_bnb_multidevice_hooks(")assert call < hooks, "offload_embedding must be resolved before hook attach"@@ -388,3 +391,116 @@ def test_a_distributed_launch_also_declines_an_explicit_request(capsys):def test_a_single_process_run_is_unaffected():with _as_platform("posix"), _card(16 * 2**30):assert resolve(_sized_model(int(2.5 * 2**30)), "auto") is True+++def test_auto_follows_free_memory_once_the_reserve_estimate_exists():+ """With the estimate, `"auto"` never offloads on size alone; it waits for a memory shortfall+ (`needed`, from offload_embedding_if_tight or the block swap planner)."""+ _NS["_zoo_reserve_estimate"] = object()+ try:+ with _as_platform("posix"), _card(16 * 2**30):+ big = _sized_model(int(2.5 * 2**30))+ assert resolve(big, "auto") is False+ assert resolve(big, "auto", needed = True) is True+ assert resolve(big, True) is True+ finally:+ _NS["_zoo_reserve_estimate"] = None+++class _PerLayerModel(nn.Module):+ """Tied input embedding plus a large per-layer table (Gemma 3n / 4 layout)."""++ def __init__(self, device):+ super().__init__()+ self.embed_tokens = nn.Embedding(64, 8, device = device)+ self.per_layer = nn.Embedding(64, 32, device = device)+ self.lm_head = nn.Linear(8, 64, bias = False, device = device)+ self.lm_head.weight = self.embed_tokens.weight++ def get_input_embeddings(self):+ return self.embed_tokens++ def get_output_embeddings(self):+ return self.lm_head+++def _spare_ns():+ ns = _load(+ "_embeddings_are_tied",+ "_offload_embedding_unsupported_platform",+ "_embedding_dispatch_device",+ "_input_side_embeddings",+ "_accelerator_device",+ "offload_spare_embeddings",+ )+ ns["_EXTRA_EMBEDDING_MIN_BYTES"] = 64 * 32 * 4 # the per-layer table above, no bigger+ moved = []+ ns["offload_input_embedding"] = lambda model, embeddings: moved.extend(embeddings) or 1+ return ns, moved++[email protected](not has_real_cuda(), reason = "needs a card to offload from")+def test_tied_model_still_offloads_its_per_layer_table():+ ns, moved = _spare_ns()+ model = _PerLayerModel("cuda")+ model.requires_grad_(False)+ with _as_platform("posix"):+ assert ns["offload_spare_embeddings"](model) == 1+ assert moved == [model.per_layer]++[email protected](not has_real_cuda(), reason = "needs a card to offload from")+def test_trainable_or_cpu_model_tables_stay():+ ns, moved = _spare_ns()+ model = _PerLayerModel("cuda")+ with _as_platform("posix"):+ # get_peft_model: a table still trainable is not offloaded.+ assert ns["offload_spare_embeddings"](model, require_frozen = True) == 0+ model = _PerLayerModel("cpu").requires_grad_(False)+ with _as_platform("posix"):+ # A model on the CPU has nothing to offload.+ assert ns["offload_spare_embeddings"](model) == 0+ assert moved == []+++class _HeadlessPerLayerModel(_PerLayerModel):+ """AutoModel backbone: no output head."""++ def get_output_embeddings(self):+ return None++[email protected](not has_real_cuda(), reason = "needs a card to offload from")+def test_headless_model_hooks_a_table_the_block_swap_load_left_on_cpu():+ ns, moved = _spare_ns()+ model = _HeadlessPerLayerModel("cuda").requires_grad_(False)+ # The block swap load streamed the table to host without hooks; the backbone stays on the card.+ model.per_layer.to("cpu")+ with _as_platform("posix"):+ assert ns["offload_spare_embeddings"](model) == 1+ # Nothing ties the input embedding without a head, so it may go too.+ assert any(m is model.per_layer for m in moved)+ moved.clear()+ model = _HeadlessPerLayerModel("cpu").requires_grad_(False)+ with _as_platform("posix"):+ assert ns["offload_spare_embeddings"](model) == 0+ assert moved == []++[email protected](not has_real_cuda(), reason = "needs a card to offload from")+def test_headless_streamed_table_returns_rows_to_the_decoder_card():+ ns = _load(+ "_embeddings_are_tied",+ "_input_side_embeddings",+ "_accelerator_device",+ "offload_input_embedding",+ )+ ns["_EXTRA_EMBEDDING_MIN_BYTES"] = 64 * 32 * 4+ hooked = []+ ns["_install_offload_embedding_hooks"] = lambda emb, out, device: hooked.append((emb, device))+ ns["_pin_device_to_decoder"] = ns["clean_gpu_cache"] = lambda *a: None+ ns["gc"] = types.SimpleNamespace(collect = lambda: None)+ model = _HeadlessPerLayerModel("cuda").requires_grad_(False)+ model.per_layer.to("cpu")+ ns["offload_input_embedding"](model, [model.per_layer])+ assert [(e is model.per_layer, d.type) for e, d in hooked] == [(True, "cuda")]diff --git a/tests/test_unsloth_device_map_optin.py b/tests/test_unsloth_device_map_optin.pyindex 92480d1ba5b..0b510bcd88e 100644--- a/tests/test_unsloth_device_map_optin.py+++ b/tests/test_unsloth_device_map_optin.py@@ -869,13 +869,15 @@ def test_a_caller_supplied_config_declines_planning():raise AssertionError("vision.py plans without vetoing a caller-supplied config")-def test_the_optimized_path_says_so_when_it_drops_an_offload_request():- """`FastLanguageModel` accepts `offload_embedding`, but the optimized architectures- take a path that has never had the parameter, so the request went nowhere in silence.- The `"auto"` default stays quiet, since off is a decision it is entitled to make."""- source = open(os.path.join(MODELS, "loader.py"), encoding = "utf-8").read()- assert "does not support it" in source- assert "offload_embedding != OFFLOAD_EMBEDDING_AUTO" in source+def test_the_optimized_path_honours_offload_embedding():+ """`FastLanguageModel` accepts `offload_embedding`, and the optimized architectures used to+ drop it in silence. Now the loader forwards it and FastLlamaModel applies the same offload as+ FastModel, through the shared helper."""+ loader = open(os.path.join(MODELS, "loader.py"), encoding = "utf-8").read()+ assert "does not support it" not in loader+ llama = open(os.path.join(MODELS, "llama.py"), encoding = "utf-8").read()+ assert 'kwargs.pop("offload_embedding", False)' in llama+ assert "offload_input_embedding(model)" in llamadef test_the_auto_mode_is_recognised_by_value_everywhere():diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex c4a28ee385f..467ee63773b 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -87,6 +87,18 @@ __all__ = ["hf_login","maybe_prefetch_hf_snapshot","is_moe_model",+ "install_block_swap",+ "offload_embedding_if_tight",+ "usable_cuda_bytes",+ "skip_checkpointing",+ "refuse_block_swap_load",+ "block_swap_load_device",+ "begin_block_swap_load",+ "finish_block_swap_load",+ "planned_prefetch_depth",+ "trim_config_for_block_swap",+ "attach_block_swap_layers",+ "skip_swapped_checkpoint_keys","get_moe_target_parameters","get_moe_target_modules","get_moe_expert_submodule_leaves",@@ -159,6 +171,24 @@ from unsloth_zoo.patching_utils import (patch_model_and_tokenizer,patch_compiled_autograd,)+from ._uma_safetensors import is_integrated_unified_memory_gpu++try:+ from unsloth_zoo.block_swap import BlockSwap, find_decoder_layers+except ImportError: # unsloth_zoo predates block_swap+ BlockSwap = find_decoder_layers = None+try:+ from unsloth_zoo.block_swap import build_host_layers+except ImportError: # unsloth_zoo predates loading straight to host+ build_host_layers = None+try:+ from unsloth_zoo.block_swap import load_layers_to_host+except ImportError: # unsloth_zoo predates loading any architecture straight to host+ load_layers_to_host = None+try:+ from unsloth_zoo.block_swap import auto_swap_indices, estimate_training_reserve_bytes+except ImportError: # unsloth_zoo predates block_swap_layers = "auto"+ auto_swap_indices = estimate_training_reserve_bytes = Nonefrom unsloth_zoo.gradient_checkpointing import (Unsloth_Offloaded_Gradient_Checkpointer,unsloth_offloaded_gradient_checkpoint,@@ -6090,6 +6120,482 @@ def hf_login(token: Optional[str] = None) -> Optional[str]:return token+def _check_block_swap(model_or_config):+ if BlockSwap is None:+ raise ImportError(+ "Unsloth: block_swap_layers needs a newer unsloth_zoo. "+ "Run `pip install --upgrade unsloth_zoo`."+ )+ if is_moe_model(model_or_config):+ print(+ "Unsloth: block_swap_layers on an MoE model copies every expert across PCIe "+ "while only the routed ones compute, so steps can be copy-bound."+ )+ if not torch.cuda.is_available():+ # Prefetch runs on CUDA/HIP streams; XPU, NPU and CPU have none.+ raise ValueError("Unsloth: block_swap_layers needs a CUDA or ROCm GPU.")+ if is_integrated_unified_memory_gpu():+ raise ValueError(+ "Unsloth: block_swap_layers has nothing to swap to on a unified-memory "+ "GPU; host and device already share the same RAM."+ )+++def _new_block_swap(layers, n, *args, placement, **kwargs):+ try:+ return BlockSwap(layers, n, *args, placement = placement, **kwargs)+ except TypeError as e:+ if "placement" not in str(e):+ raise+ # unsloth_zoo before placement support always swaps the last n.+ return BlockSwap(layers, n, *args, **kwargs)+++def refuse_block_swap_load(block_swap_layers, reason):+ """A load-to-host restriction: `"auto"` falls back to loading onto the GPU (0), a count raises."""+ if block_swap_layers == "auto":+ print(+ f"Unsloth: block_swap_layers = 'auto' loads every layer onto the GPU: loading into host RAM {reason}"+ )+ return 0+ raise ValueError(f"Unsloth: from_pretrained(block_swap_layers = ...) {reason}")+++def block_swap_load_device(device_map):+ """The one CUDA card a load puts every layer on, else None: loading to host fetches onto one card."""+ if DEVICE_TYPE_TORCH != "cuda" or not torch.cuda.is_available():+ return None+ target = device_map+ if isinstance(device_map, dict):+ if len(set(device_map.values())) != 1:+ return None+ target = next(iter(device_map.values()))+ if target is None or target in ("auto", "balanced", "balanced_low_0", "sequential"):+ return torch.cuda.current_device() if torch.cuda.device_count() == 1 else None+ try:+ target = torch.device("cuda", target) if isinstance(target, int) else torch.device(target)+ except (RuntimeError, TypeError, ValueError):+ return None+ if target.type != "cuda":+ return None+ return target.index if target.index is not None else torch.cuda.current_device()+++def begin_block_swap_load(+ block_swap_layers,+ device_map,+ embeddings = False,+):+ """from_pretrained(block_swap_layers = N) on any architecture: the context the weight load runs in,+ moving N decoder layers (and with `embeddings`, the large extra token tables) to host RAM as each+ finishes loading. nullcontext when N is 0."""+ if not block_swap_layers and not embeddings:+ return contextlib.nullcontext()+ if embeddings:+ return load_layers_to_host(block_swap_layers, placement = "spread", embeddings = True)+ return load_layers_to_host(block_swap_layers, placement = "spread")+++def planned_prefetch_depth(device_map_planner_kwargs):+ # The depth "auto" sized the slot pool with; the swapper must allocate the same pool.+ return int((device_map_planner_kwargs or {}).get("prefetch_depth", 2))+++def finish_block_swap_load(+ model,+ state,+ prefetch_depth = 2,+):+ """Install the swap over the layers `begin_block_swap_load` moved, once the loader is done+ patching (their host copies are what the slot pool streams)."""+ if state is None or getattr(state, "layers", None) is None or not state.indices:+ return None+ head = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None+ weight = getattr(head, "weight", None)+ device = weight.device if weight is not None and weight.device.type == "cuda" else None+ if device is None:+ # Headless backbones (AutoModel): a retained layer names the card, which need not be the current one.+ swapped = set(state.indices)+ device = next(+ (+ p.device+ for i, layer in enumerate(state.layers)+ if i not in swapped+ for p in layer.parameters()+ if p.device.type == "cuda"+ ),+ None,+ )+ if device is None:+ device = torch.device("cuda", torch.cuda.current_device())+ swapper = _new_block_swap(+ state.layers, state.indices, prefetch_depth, device = device, placement = "spread"+ )+ state.layers._unsloth_block_swap = swapper+ model._unsloth_block_swap = swapper+ return swapper+++# loader_utils.OFFLOAD_EMBEDDING_AUTO; loader_utils imports this module, so not imported from there.+_OFFLOAD_EMBEDDING_AUTO = "auto"+++def _offload_embedding_for_room(model, require_frozen = True):+ """Move the input-side embeddings lm_head does not share to host RAM, unless the caller passed+ offload_embedding = False. After get_peft_model they must also be frozen."""+ if getattr(model, "_unsloth_offload_embedding_mode", _OFFLOAD_EMBEDDING_AUTO) is False:+ return False+ from .vision import offload_spare_embeddings++ return offload_spare_embeddings(model, require_frozen = require_frozen) > 0+++def _training_reserve_bytes(+ model,+ seq_len = None,+ trainable = True,+):+ # Grads, AdamW's two fp32 moments and the foreach temp; before get_peft_model every param still says requires_grad.+ extra = 0+ if trainable:+ extra = sum(+ p.numel() * (p.element_size() + 12) for p in model.parameters() if p.requires_grad+ )+ seq_len = seq_len or getattr(model, "max_seq_length", None) or 2048+ return estimate_training_reserve_bytes(model.config, seq_len, extra_bytes = extra), seq_len+++def _skip_aware_flag(cls):+ # for_training, gradient_checkpointing_enable and the trainer rewrite the flag; a marked layer keeps reading False.+ if cls.__dict__.get("_unsloth_skip_aware", False):+ return++ def _get(self):+ if self.__dict__.get("_unsloth_skip_checkpoint", False):+ return False+ # Unmarked layers of the class still hold the flag in their own dict.+ return self.__dict__.get(+ "_unsloth_gc_flag", self.__dict__.get("gradient_checkpointing", False)+ )++ def _set(self, value):+ self.__dict__["_unsloth_gc_flag"] = value++ cls.gradient_checkpointing = property(_get, _set)+ cls._unsloth_skip_aware = True+++def skip_checkpointing(model, layers = 0):+ """get_peft_model(checkpoint_skip_layers = K | "max"): K decoder layers (evenly spaced; "max" =+ all) keep their activations instead of recomputing them in backward. Faster, at the cost of+ each layer's activations on the card. Block-swapped layers keep checkpointing."""+ if not layers:+ return []+ layer_list = find_decoder_layers(model)+ swapper = getattr(model, "_unsloth_block_swap", None)+ swapped = set(getattr(swapper, "indices", None) or ())+ eligible = [i for i in range(len(layer_list)) if i not in swapped]+ if layers == "max":+ chosen = eligible+ else:+ count = max(0, min(int(layers), len(eligible)))+ chosen = (+ [eligible[(k + 1) * len(eligible) // count - 1] for k in range(count)] if count else []+ )+ for i in chosen:+ layer = layer_list[i]+ if not hasattr(layer, "gradient_checkpointing"):+ continue+ current = layer.__dict__.pop("gradient_checkpointing", None)+ _skip_aware_flag(type(layer))+ if current is not None:+ layer.__dict__["_unsloth_gc_flag"] = current+ layer.__dict__["_unsloth_skip_checkpoint"] = True+ return chosen+++def usable_cuda_bytes(device):+ """Bytes this process can still allocate on `device`: free card memory plus blocks torch's+ caching allocator holds unused, capped by torch.cuda.set_per_process_memory_fraction."""+ free, total = torch.cuda.mem_get_info(device)+ allocated = torch.cuda.memory_allocated(device)+ usable = free + torch.cuda.memory_reserved(device) - allocated+ get_fraction = getattr(torch.cuda, "get_per_process_memory_fraction", None)+ try:+ fraction = get_fraction(device) if get_fraction is not None else 1.0+ except Exception:+ fraction = 1.0+ if fraction < 1.0:+ usable = min(usable, int(total * fraction) - allocated)+ return max(0, int(usable))+++def offload_embedding_if_tight(+ model,+ seq_len = None,+ at_load = False,+):+ """offload_embedding = "auto": move the input embedding to host RAM only when its GPU lacks a+ training step's reserve (activations, logits, trainable state), on any architecture. Runs after+ the load and again in get_peft_model, once the trainable parameters are known."""+ if estimate_training_reserve_bytes is None or not torch.cuda.is_available():+ return False+ if getattr(model, "_unsloth_offload_embedding_mode", None) != _OFFLOAD_EMBEDDING_AUTO:+ return False+ try:+ weight = getattr(model.get_input_embeddings(), "weight", None)+ except Exception:+ return False+ if weight is None or weight.device.type != "cuda":+ return False+ reserve, _ = _training_reserve_bytes(model, seq_len, trainable = not at_load)+ if usable_cuda_bytes(weight.device) >= reserve:+ return False+ return _offload_embedding_for_room(model, require_frozen = not at_load)+++def _auto_block_swap_indices(model, prefetch_depth):+ """Layers to swap so each GPU keeps a training step's reserve free; [] when it already does."""+ if auto_swap_indices is None:+ raise ImportError(+ "Unsloth: block_swap_layers = 'auto' needs a newer unsloth_zoo. "+ "Run `pip install --upgrade unsloth_zoo`."+ )+ layers = find_decoder_layers(model)+ reserve, seq_len = _training_reserve_bytes(model)+ indices, left = auto_swap_indices(layers, reserve, prefetch_depth)+ # Only the looked-up rows cross PCIe, so the embedding goes before any layer.+ if indices and _offload_embedding_for_room(model):+ indices, left = auto_swap_indices(layers, reserve, prefetch_depth)+ if not indices:+ print(+ f"Unsloth: block_swap_layers = 'auto' swaps nothing: every GPU keeps the "+ f"{reserve / 2**30:.2f} GiB a {seq_len}-token step needs."+ )+ else:+ print(+ f"Unsloth: block_swap_layers = 'auto' keeps {len(indices)} of {len(layers)} decoder layers "+ f"in host RAM so {reserve / 2**30:.2f} GiB stays free for a {seq_len}-token step."+ )+ if left > 0:+ print(+ f"Unsloth: block_swap_layers = 'auto' is still {left / 2**30:.2f} GiB short; "+ "lower max_seq_length, or load with from_pretrained(block_swap_layers = 'auto')."+ )+ return indices+++def install_block_swap(+ model,+ block_swap_layers = 0,+ prefetch_depth = 2,+ use_gradient_checkpointing = "unsloth",+):+ """Stream frozen decoder blocks from pinned host RAM: `block_swap_layers` of them, or "auto" for+ as few as keep a training step's reserve free on every GPU; off at 0."""+ existing = getattr(model, "_unsloth_block_swap", None)+ if existing is None and (+ not block_swap_layers or (block_swap_layers != "auto" and block_swap_layers <= 0)+ ):+ return None+ if not use_gradient_checkpointing:+ raise ValueError(+ "Unsloth: block_swap_layers needs use_gradient_checkpointing. Without "+ "it every swapped block stays on the card until backward, so there is "+ "nothing to save and the slot pool runs dry mid-forward."+ )+ if existing is not None:+ return existing+ if getattr(model, "vllm_engine", None) is not None:+ raise ValueError(+ "Unsloth: block_swap_layers cannot be combined with fast_inference = True, "+ "since evicted weights would sync to vLLM as empty tensors."+ )+ if block_swap_layers == "auto":+ # Nothing to swap to without a discrete CUDA / ROCm card: auto means no swap there.+ if not torch.cuda.is_available() or is_integrated_unified_memory_gpu():+ return None+ if BlockSwap is None:+ _check_block_swap(model)+ block_swap_layers = _auto_block_swap_indices(model, prefetch_depth)+ if not block_swap_layers:+ return None+ _check_block_swap(model)+ layers = find_decoder_layers(model)+ # Spaced evenly, each copy hides behind several layers of compute instead of one.+ swapper = _new_block_swap(layers, block_swap_layers, prefetch_depth, placement = "spread")+ # On the layer list too: the fast decode loop only sees the inner model.+ layers._unsloth_block_swap = swapper+ model._unsloth_block_swap = swapper+ return swapper+++def _checkpoint_tensors(+ model_name,+ token = None,+ revision = None,+ cache_dir = None,+ local_files_only = False,+ subfolder = None,+ variant = None,+):+ from safetensors import safe_open+ import json++ def _named(filename):+ if variant is None:+ return filename+ stem, ext = filename.rsplit(".", 1)+ return f"{stem}.{variant}.{ext}"++ def _get(filename):+ if os.path.isdir(model_name):+ path = os.path.join(model_name, subfolder or "", filename)+ return path if os.path.exists(path) else None+ from huggingface_hub import hf_hub_download+ try:+ return hf_hub_download(+ model_name,+ filename,+ subfolder = subfolder,+ revision = revision,+ token = token,+ cache_dir = cache_dir,+ local_files_only = local_files_only,+ )+ except Exception:+ return None++ index = _get(_named("model.safetensors.index.json"))+ if index is not None:+ with open(index, "r", encoding = "utf-8") as f:+ shards = sorted(set(json.load(f)["weight_map"].values()))+ else:+ shards = [_named("model.safetensors")]+ handles, tensors = [], {}+ for shard in shards:+ path = _get(shard)+ if path is None:+ raise RuntimeError(+ f"Unsloth: block_swap_layers could not find {shard} for {model_name}."+ )+ h = safe_open(path, framework = "pt", device = "cpu")+ handles.append(h)+ for key in h.keys():+ tensors[key] = lambda h = h, key = key: h.get_tensor(key)+ return tensors, handles+++def trim_config_for_block_swap(config, block_swap_layers):+ """Trim the swapped tail off the config; returns the originals to restore, or None."""+ if not block_swap_layers or block_swap_layers <= 0:+ return None+ if build_host_layers is None:+ raise ImportError(+ "Unsloth: from_pretrained(block_swap_layers = ...) needs a newer unsloth_zoo. "+ "Run `pip install --upgrade unsloth_zoo`."+ )+ _check_block_swap(config)+ total = config.num_hidden_layers+ n = min(int(block_swap_layers), total - 1)+ saved = {"num_hidden_layers": total}+ for key, value in list(vars(config).items()):+ if isinstance(value, (list, tuple)) and len(value) == total:+ saved[key] = value+ setattr(config, key, type(value)(value[: total - n]))+ config.num_hidden_layers = total - n+ return saved+++_SWAPPED_LAYER_KEY = re.compile(r"(?:^|\.)layers\.(\d+)\.")+++def skip_swapped_checkpoint_keys(saved, kept):+ """Hide the swapped tail's checkpoint keys from the standard load; returns an undo callable.++ transformers 4.x still loads every key its renaming mapping keeps: the tail's bnb stats reach the+ quantizer (ModuleList has no attribute `N`) and plain weights log "were not used", which+ RaiseUninitialized turns into an error. 5.x has no such method and already drops them."""+ from transformers.modeling_utils import PreTrainedModel++ original = PreTrainedModel.__dict__.get("_get_key_renaming_mapping")+ if saved is None or original is None:+ return lambda: None++ def _get_key_renaming_mapping(self, checkpoint_keys, *args, **kwargs):+ keep = []+ for key in checkpoint_keys:+ m = _SWAPPED_LAYER_KEY.search(key)+ if m is None or int(m.group(1)) < kept:+ keep.append(key)+ return original(self, keep, *args, **kwargs)++ PreTrainedModel._get_key_renaming_mapping = _get_key_renaming_mapping++ def undo():+ PreTrainedModel._get_key_renaming_mapping = original++ return undo+++def attach_block_swap_layers(+ model,+ saved,+ model_name,+ dtype,+ load_in_4bit,+ skip_modules = (),+ prefetch_depth = 2,+ **hub_kwargs,+):+ if saved is None:+ return None+ layers = find_decoder_layers(model)+ first = len(layers)+ config = model.config+ for key, value in saved.items():+ setattr(config, key, value)+ count = config.num_hidden_layers - first+ layer_cls = type(layers[0])+ # The tail fetches onto the head's card, where a multi-GPU plan reserved its slot pool.+ head = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None+ weight = getattr(head, "weight", None)+ device = weight.device if weight is not None else next(layers[first - 1].parameters()).device+ if device.type != "cuda":+ device = torch.device("cuda", torch.cuda.current_device())+ tensors, handles = _checkpoint_tensors(model_name, **hub_kwargs)+ try:+ new = build_host_layers(+ lambda idx: layer_cls(config, idx),+ first,+ count,+ tensors,+ device = device,+ compute_dtype = dtype,+ quantize_4bit = load_in_4bit,+ skip_modules = skip_modules,+ )+ finally:+ del handles+ # Carry over plain attributes post_patch set on loaded layers (Gemma's norm variance_epsilon).+ reference = dict(layers[0].named_modules())+ for layer in new:+ for name, module in layer.named_modules():+ ref = reference.get(name)+ if ref is None:+ continue+ for key, value in vars(ref).items():+ if key.startswith("_") or key in vars(module):+ continue+ if not isinstance(value, (torch.Tensor, torch.nn.Module)):+ setattr(module, key, value)+ layers.append(layer)+ swapper = _new_block_swap(layers, count, prefetch_depth, device = device, placement = "tail")+ layers._unsloth_block_swap = swapper+ model._unsloth_block_swap = swapper+ return swapper++def is_moe_model(model) -> bool:"""Detect whether a model (or config) is a Mixture of Experts model."""config = getattr(model, "config", model)diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.pyindex fce9af39ef9..c81e30550d9 100644--- a/unsloth/models/cohere.py+++ b/unsloth/models/cohere.py@@ -445,7 +445,10 @@ def CohereModel_fast_forward_inference(attention_mask = Nonenext_decoder_cache = []+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)for idx, decoder_layer in enumerate(self.model.layers):+ if block_swap is not None:+ block_swap.enter(idx)layer_device, device_index = per_layer_device(decoder_layer)hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)residual = hidden_states@@ -465,6 +468,8 @@ def CohereModel_fast_forward_inference(residual += hidden_states_attentionresidual += hidden_states_mlphidden_states = residual+ if block_swap is not None:+ block_swap.leave(idx)next_decoder_cache.append(present_key_value)hidden_states = fast_layernorm_inference(diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.pyindex 543ecf762f2..2175a3fed64 100644--- a/unsloth/models/falcon_h1.py+++ b/unsloth/models/falcon_h1.py@@ -512,8 +512,11 @@ def _FalconH1_fast_forward_inference(attention_mask = Nonenext_decoder_cache = []+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)for idx, decoder_layer in enumerate(self.model.layers):+ if block_swap is not None:+ block_swap.enter(idx)residual.copy_(X)X = fast_rms_layernorm_inference(decoder_layer.input_layernorm,@@ -559,6 +562,8 @@ def _FalconH1_fast_forward_inference(down_multiplier = down_multiplier,)X += residual+ if block_swap is not None:+ block_swap.leave(idx)next_decoder_cache.append(present_key_value)X = fast_rms_layernorm_inference(diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.pyindex 4b227b5e5c0..2df1d8e6c3e 100644--- a/unsloth/models/gemma.py+++ b/unsloth/models/gemma.py@@ -190,7 +190,10 @@ def GemmaModel_fast_forward_inference(rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)next_decoder_cache = []+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)for idx, decoder_layer in enumerate(self.model.layers):+ if block_swap is not None:+ block_swap.enter(idx)layer_device, device_index = per_layer_device(decoder_layer)hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)if attention_mask is not None:@@ -220,6 +223,8 @@ def GemmaModel_fast_forward_inference(hidden_states = fast_geglu_inference(decoder_layer.mlp, hidden_states)hidden_states += residual+ if block_swap is not None:+ block_swap.leave(idx)next_decoder_cache.append(present_key_value)hidden_states = fast_rms_layernorm_inference_gemma(self.model.norm, hidden_states, out_weights[device_index]diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex a9bd96821b2..562328f6231 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -568,7 +568,10 @@ def Gemma2Model_fast_forward_inference(SWA = NoneGA = Nonenext_decoder_cache = []+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)for idx, decoder_layer in enumerate(self.model.layers):+ if block_swap is not None:+ block_swap.enter(idx)# For pipeline parallelism every tensor must be on the same device; this movement happens once per GPU in PP.layer_device, device_index = per_layer_device(decoder_layer)hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)@@ -617,6 +620,8 @@ def Gemma2Model_fast_forward_inference()hidden_states += residual+ if block_swap is not None:+ block_swap.leave(idx)next_decoder_cache.append(present_key_value)hidden_states = fast_rms_layernorm_inference_gemma(self.model.norm, hidden_states, out_weights[device_index]diff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex ff190178f9b..293378a5e73 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -451,7 +451,10 @@ def GraniteModel_fast_forward_inference()next_decoder_cache = []+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)for idx, decoder_layer in enumerate(self.model.layers):+ if block_swap is not None:+ block_swap.enter(idx)layer_device, _ = per_layer_device(decoder_layer)hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)@@ -475,6 +478,8 @@ def GraniteModel_fast_forward_inference()hidden_states = fast_swiglu_inference(decoder_layer.mlp, hidden_states)hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)+ if block_swap is not None:+ block_swap.leave(idx)next_decoder_cache.append(present_key_value)hidden_states = fast_rms_layernorm_inference(self.model.norm, hidden_states)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex ee135948bdb..8cd6fa87fef 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -26,6 +26,7 @@ from ._utils import per_layer_devicefrom ._utils import embedding_applies_scalefrom ._utils import (_get_inference_mode_context_manager,+ _offload_embedding_for_room,_prepare_model_for_qat,is_bfloat16_supported,get_quant_type,@@ -36,6 +37,7 @@ from ._utils import ()from .loader_utils import (DEFAULT_DEVICE_MAP,+ OFFLOAD_EMBEDDING_AUTO,_exclude_rope_inv_freq_from_ddp,_get_fp8_mode_and_check_settings,_restore_dropped_fp8_scales,@@ -51,6 +53,7 @@ from .loader_utils import (planner_quantization_kwargs,requested_device_map,resolve_unsloth_device_map,+ resolve_auto_block_swap,warn_if_bitsandbytes_quantized_nothing,)from ..utils.packing import (@@ -116,6 +119,12 @@ from ..kernels import *from ..kernels.utils import has_mxfp4_basefrom ..tokenizer_utils import *from .vision import FastBaseModel, _is_text_seq2seq_config+from .vision import (+ _offload_embedding_unsupported_platform,+ _resolve_offload_embedding,+ offload_input_embedding,+ restore_input_embedding,+)from transformers.models.llama.modeling_llama import (LlamaAttention,@@ -1373,8 +1382,11 @@ def _LlamaModel_fast_forward_inference(rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)next_decoder_cache = []+ block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)for idx, decoder_layer in enumerate(self.model.layers):+ if block_swap is not None:+ block_swap.enter(idx)layer_device, device_index = per_layer_device(decoder_layer)X, residual, position_ids = move_to_device(layer_device, X, residual, position_ids)# self_attn is called directly, so no accelerate hook moves the mask to a split layer's device.@@ -1414,6 +1426,8 @@ def _LlamaModel_fast_forward_inference(temp_up = temp_ups[device_index],)X += residual+ if block_swap is not None:+ block_swap.leave(idx)next_decoder_cache.append(present_key_value)X = fast_rms_layernorm_inference(@@ -2720,6 +2734,34 @@ class FastLlamaModel:# Respect a user-provided config so it is the single config object used everywhere below; else# HF gets it again through **kwargs alongside our config= and fails with a duplicate kwarg.user_config = kwargs.pop("config", None)+ block_swap_layers = kwargs.pop("block_swap_layers", 0)+ offload_embedding = kwargs.pop("offload_embedding", False)+ if offload_embedding and fast_inference:+ if offload_embedding != OFFLOAD_EMBEDDING_AUTO:+ print(+ "Unsloth: Not offloading embeddings; incompatible with fast_inference (vLLM)."+ )+ offload_embedding = False+ _offload_embedding_mode = offload_embedding+ # None: no memory plan; True / False: the block swap planner decided.+ _embedding_needed = None+ if block_swap_layers and kwargs.get("state_dict") is not None:+ block_swap_layers = refuse_block_swap_load(+ block_swap_layers,+ "does not take a state_dict; save it as a safetensors checkpoint and load that.",+ )+ if block_swap_layers and kwargs.get("quantization_config") is not None:+ block_swap_layers = refuse_block_swap_load(+ block_swap_layers,+ "does not take a quantization_config; pass load_in_4bit = True instead.",+ )+ if block_swap_layers and (+ kwargs.get("gguf_file") or kwargs.get("use_safetensors") is False+ ):+ block_swap_layers = refuse_block_swap_load(+ block_swap_layers,+ "needs a safetensors checkpoint; it does not support gguf_file or use_safetensors = False.",+ )if user_config is not None:model_config = user_config# model_name may have been remapped to a prequantized repo whose checkpoint needs its@@ -2901,6 +2943,16 @@ class FastLlamaModel:# checkpoint left unarmed (vLLM reads it itself) must not reach vLLM as a bitsandbytes load.if not (_explicit_bnb_4bit and _checked_4bit):load_in_4bit, load_in_8bit = _checked_4bit, _checked_8bit+ if block_swap_layers and load_in_8bit:+ block_swap_layers = refuse_block_swap_load(+ block_swap_layers, "supports 16-bit and 4-bit loads, not load_in_8bit."+ )+ if block_swap_layers and _ckpt_quant_method not in (None, "bitsandbytes"):+ # The host tail is rebuilt as dense or bnb 4-bit layers; packed formats would not survive it.+ block_swap_layers = refuse_block_swap_load(+ block_swap_layers,+ f"does not support {_ckpt_quant_method} checkpoints; use a bitsandbytes or 16-bit checkpoint.",+ )from .modelopt_fp8 import (keep_fp8_scale_names_on_save,move_config_overrides_onto_config,@@ -2968,6 +3020,42 @@ class FastLlamaModel:"load adds, so the plan would size dense weights at 4bit")+ if block_swap_layers and (fast_inference or num_labels is not None):+ block_swap_layers = refuse_block_swap_load(+ block_swap_layers, "does not support fast_inference or classification heads."+ )+ if block_swap_layers == "auto":+ block_swap_layers, device_map, _embedding_needed = resolve_auto_block_swap(+ requested_device_map(device_map),+ model_name,+ max_seq_length = max_seq_length,+ offload_embedding = bool(offload_embedding)+ and num_labels is None+ and _offload_embedding_unsupported_platform() is None,+ planner_kwargs = planner_kwargs_with_max_memory(device_map_planner_kwargs, kwargs),+ skip_reason = _planner_skip_reason,+ **planner_config_overrides(kwargs),+ token = token,+ trust_remote_code = trust_remote_code,+ **planner_hub_kwargs(kwargs),+ revision = revision,+ **add_dtype_kwargs(dtype),+ **planner_quantization_kwargs(+ **compressed_tensors_planner_quantization(+ model_config,+ load_in_4bit,+ load_in_8bit,+ kwargs.get("quantization_config", None),+ ),+ rewritten_quantization_config = modelopt_planner_quantization_config(model_config)+ if _modelopt_rewritten+ else fp8_to_nf4_planner_quantization_config(+ model_config,+ SKIP_QUANTIZATION_MODULES + (["out_proj"] if IS_FALCON_H1 else []),+ ),+ extra_skip_modules = ["out_proj"] if IS_FALCON_H1 else None,+ ),+ )# Here, not in loader.py: the mapper there can still substitute the repo (a -bnb-4bit name# resolving to its 16-bit twin), so a plan sized for the caller's name is the wrong plan.device_map = resolve_unsloth_device_map(@@ -3053,6 +3141,16 @@ class FastLlamaModel:kwargs = add_dtype_kwargs(dtype, kwargs)+ if block_swap_layers:+ import copy as _copy++ # Trim a copy: HF deep-copies config= anyway, so a trimmed caller config would stay short.+ model_config = _copy.deepcopy(model_config)+ _block_swap_saved = trim_config_for_block_swap(model_config, block_swap_layers)+ _undo_block_swap_keys = skip_swapped_checkpoint_keys(+ _block_swap_saved, model_config.num_hidden_layers+ )+raise_handler = RaiseUninitialized()try:if num_labels is not None:@@ -3133,7 +3231,13 @@ class FastLlamaModel:_ct_requant = (getattr(model_config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None) is not None)- if user_config is not None or _modelopt_rewritten or _ct_requant or _fp8_to_nf4:+ if (+ user_config is not None+ or _modelopt_rewritten+ or _ct_requant+ or _fp8_to_nf4+ or _block_swap_saved is not None+ ):# Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override# on the config and pass the single config object through.if max_position_embeddings is not None:@@ -3252,6 +3356,7 @@ class FastLlamaModel:model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)finally:raise_handler.remove()+ _undo_block_swap_keys()os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transferif _modelopt_rewritten:keep_fp8_scale_names_on_save(model)@@ -3276,6 +3381,38 @@ class FastLlamaModel:model, tokenizer = patch_tokenizer(model, tokenizer)model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)+ attach_block_swap_layers(+ model,+ _block_swap_saved,+ model_name,+ dtype,+ load_in_4bit,+ # The load's own skip list: Falcon-H1 adds out_proj, which its mamba kernels need unquantized.+ skip_modules = llm_int8_skip_modules if load_in_4bit else SKIP_QUANTIZATION_MODULES,+ prefetch_depth = planned_prefetch_depth(device_map_planner_kwargs),+ token = token,+ revision = revision,+ cache_dir = kwargs.get("cache_dir"),+ local_files_only = kwargs.get("local_files_only", False),+ subfolder = kwargs.get("subfolder"),+ variant = kwargs.get("variant"),+ )+ # "auto" moves it only when memory is tight (block swap plan or free VRAM vs the step reserve); True always does.+ model._unsloth_offload_embedding_mode = (+ _offload_embedding_mode if num_labels is None else False+ )+ if num_labels is not None:+ pass+ elif offload_embedding == OFFLOAD_EMBEDDING_AUTO:+ if _embedding_needed:+ _offload_embedding_for_room(model, require_frozen = False)+ elif _embedding_needed is None:+ offload_embedding_if_tight(model, max_seq_length, at_load = True)+ elif offload_embedding:+ if _resolve_offload_embedding(model, offload_embedding):+ offload_input_embedding(model)+ else:+ _offload_embedding_for_room(model, require_frozen = False)for idx, layer in enumerate(model.model.layers):layer.self_attn.apply_qkv = original_apply_qkv@@ -3514,6 +3651,8 @@ class FastLlamaModel:qat_scheme = None,target_parameters = None, # For MoE expert layers (nn.Parameter)ensure_weight_tying = None, # None = auto (tie when we redirect a tied pair)+ block_swap_layers = 0,+ checkpoint_skip_layers = 0,**kwargs,):# The flag reflects the LAST load, not this model.@@ -3551,6 +3690,8 @@ class FastLlamaModel:temporary_location = temporary_location,target_parameters = target_parameters,ensure_weight_tying = ensure_weight_tying,+ block_swap_layers = block_swap_layers,+ checkpoint_skip_layers = checkpoint_skip_layers,**kwargs,)if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":@@ -3677,6 +3818,10 @@ class FastLlamaModel:# apply_unsloth_gradient_checkpointing above already re-patched global state to match (#4735).model._unsloth_gradient_checkpointing = use_gradient_checkpointingmodel = _exclude_rope_inv_freq_from_ddp(model)+ install_block_swap(+ model, block_swap_layers, use_gradient_checkpointing = use_gradient_checkpointing+ )+ skip_checkpointing(model, checkpoint_skip_layers)return modelelse:raise TypeError(@@ -3934,6 +4079,8 @@ class FastLlamaModel:_saved_temp_tokenizer = model._saved_temp_tokenizerlora_config = LoraConfig(**arguments)+ if train_embed_tokens and restore_input_embedding(model):+ print("Unsloth: Moved the offloaded input embedding back to the GPU to train it.")input_embeddings_device = model.get_input_embeddings().weight.deviceif is_classification:output_embeddings_device = model.score.weight.device@@ -3978,6 +4125,11 @@ class FastLlamaModel:model._saved_temp_tokenizer = _saved_temp_tokenizermodel = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)+ offload_embedding_if_tight(model)+ install_block_swap(+ model, block_swap_layers, use_gradient_checkpointing = use_gradient_checkpointing+ )+ skip_checkpointing(model, checkpoint_skip_layers)if ensure_weight_tying:try:diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 456457165d4..5bc947eb8c5 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -1459,14 +1459,6 @@ class FastLanguageModel(FastLlamaModel):except Exception as e:print(f"Unsloth: Could not patch bitsandbytes for torch.compile - {e}")- # The optimized path never carried offload_embedding, so a request for one is dropped rather than honoured; say so instead of leaving the caller to infer it from memory use. "auto" stays quiet: it promises a decision, and off is one.- if offload_embedding != OFFLOAD_EMBEDDING_AUTO and offload_embedding:- print(- "Unsloth: Not offloading embeddings; the optimized path for this "- "architecture does not support it. Pass `device_map` or use FastModel "- "if you need the offload."- )-model, tokenizer = dispatch_model.from_pretrained(model_name = model_name,max_seq_length = max_seq_length,@@ -1491,6 +1483,12 @@ class FastLanguageModel(FastLlamaModel):max_lora_rank = max_lora_rank,disable_log_stats = disable_log_stats,load_in_fp8 = load_in_fp8,+ # resize_token_embeddings below replaces the embedding and its hooks: only an explicit request offloads.+ offload_embedding = (+ False+ if resize_model_vocab is not None and offload_embedding == OFFLOAD_EMBEDDING_AUTO+ else offload_embedding+ ),*args,**kwargs,)diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 296920ea38f..31d88838afc 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -643,6 +643,126 @@ def resolve_unsloth_device_map(return plan.device_map+_BLOCK_SWAP_PLANNER_KEYS = (+ "batch_size",+ "lora_rank",+ "reserve_bytes",+ "prefetch_depth",+ "rows_per_chunk",+ "retained_rows",+ "headroom_bytes",+ "safety_bytes",+ "free_space_policy",+ "no_split_module_classes",+)+++def resolve_auto_block_swap(+ device_map,+ model_name,+ *,+ max_seq_length,+ offload_embedding = False,+ planner_kwargs = None,+ skip_reason = None,+ placement = "tail",+ **config_kwargs,+):+ """`from_pretrained(block_swap_layers = "auto")`: `(layers, device_map, embedding)`, the trailing+ decoder layers to build in host RAM so the rest plus a training step's reserve fits, the map to load+ the rest with, and whether to move the input embedding to host RAM first (one GPU, when+ `offload_embedding` allows; None when nothing was planned). 0, the map unchanged and no move when+ everything fits, so nothing slows down. `device_map = "unsloth"` / `"unsloth_balanced"` sizes every+ card through the multi-GPU planner; anything else sizes the one card the load uses."""++ def _none(reason):+ print(f"Unsloth: block_swap_layers = 'auto' loads every layer onto the GPU: {reason}.")+ return 0, device_map, None++ if skip_reason is not None:+ return _none(skip_reason)+ if DEVICE_TYPE_TORCH != "cuda" or not torch.cuda.is_available():+ return _none("block swap needs a CUDA or ROCm GPU")+ if is_distributed():+ return _none("each rank of a distributed launch owns its own device")+ try:+ from unsloth_zoo.device_map_planner import plan_block_swap+ except ImportError:+ return _none("this unsloth_zoo cannot plan block swap")++ planner_kwargs = dict(planner_kwargs or {})+ requested_memory = planner_kwargs.pop("max_memory", None)+ devices = []+ if isinstance(device_map, str) and device_map in _PLANNED_DEVICE_MAPS:+ devices = (+ [d for d in requested_memory if isinstance(d, int) and not isinstance(d, bool)]+ if requested_memory+ else list(range(torch.cuda.device_count()))+ )+ multi = len(devices) > 1+ if multi:+ target = None+ elif devices:+ target = devices[0]+ else:+ if isinstance(device_map, dict) and set(device_map) == {""}:+ target = torch.device(device_map[""])+ if target.type != "cuda":+ return _none(f"the load places the model on {target}")+ target = target.index if target.index is not None else torch.cuda.current_device()+ elif device_map is None or (+ isinstance(device_map, str)+ and (device_map in TRANSFORMERS_PLACEMENT_STRATEGIES or device_map.startswith("cuda"))+ ):+ target = (+ torch.device(device_map).index+ if isinstance(device_map, str) and device_map.startswith("cuda:")+ else torch.cuda.current_device()+ )+ else:+ return _none(+ "it sizes one GPU or the `device_map = 'unsloth'` planner, not an explicit map"+ )+ devices = [target]+ from ._utils import usable_cuda_bytes++ max_memory = {}+ for d in devices:+ free = usable_cuda_bytes(d)+ cap = _as_bytes((requested_memory or {}).get(d)) if requested_memory else None+ max_memory[d] = free if cap is None else min(free, cap)++ options = {k: planner_kwargs[k] for k in _BLOCK_SWAP_PLANNER_KEYS if k in planner_kwargs}+ try:+ planner_parameters = inspect.signature(plan_block_swap).parameters+ except (TypeError, ValueError):+ planner_parameters = {}+ if offload_embedding and not multi and "offload_embedding" in planner_parameters:+ options["offload_embedding"] = True+ if placement != "tail" and not multi:+ if "placement" not in planner_parameters:+ return _none("this unsloth_zoo plans only trailing layers")+ options["placement"] = placement+ plan = plan_block_swap(+ model_name,+ max_memory = max_memory,+ seq_len = max_seq_length,+ **options,+ **config_kwargs,+ )+ print(+ "Unsloth: "+ + plan.describe().splitlines()[0].replace("block swap:", "block_swap_layers = 'auto':")+ )+ embedding = bool(getattr(plan, "offload_embedding", False))+ if not plan.layers:+ return 0, device_map, embedding+ if multi and plan.device_plan is not None:+ print(plan.device_plan.describe())+ return plan.layers, plan.device_plan.device_map, embedding+ return plan.layers, {"": target}, embedding++def __get_model_name(model_name,load_in_4bit = True,diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 21646b3be8a..82f683e7a52 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -98,7 +98,9 @@ from ..kernels import (from ._utils import (__version__,importlib_version,+ _offload_embedding_for_room,_prepare_model_for_qat,+ load_layers_to_host,resolve_model_class,resolve_remote_code_model_class,attention_class_for_load,@@ -123,6 +125,8 @@ from ._utils import (set_module_gradient_checkpointing,)from ._utils import *+from ._uma_safetensors import is_integrated_unified_memory_gpu+from ._utils import estimate_training_reserve_bytes as _zoo_reserve_estimatefrom ._remote_code_buffers import restore_remote_code_non_persistent_buffersfrom ._custom_dtype import resolve_dtype, trusted_custom_dtypefrom .remote_code_shims import apply_remote_code_shims@@ -146,6 +150,7 @@ from .loader_utils import (exclude_no_placement_params,planner_quantization_kwargs,requested_device_map,+ resolve_auto_block_swap,resolve_unsloth_device_map,warn_if_bitsandbytes_quantized_nothing,)@@ -705,6 +710,11 @@ def _install_offload_embedding_hooks(embed_tokens, output_embeddings, return_devreturn output.to(target)return output+ # Host lookups and cross-device copies cannot be traced: a compiled caller breaks around them.+ disable = getattr(getattr(torch, "compiler", None), "disable", None)+ if disable is not None:+ _unsloth_offload_pre_hook = disable(_unsloth_offload_pre_hook)+ _unsloth_offload_post_hook = disable(_unsloth_offload_post_hook)embed_tokens.register_forward_pre_hook(_unsloth_offload_pre_hook, prepend = True)embed_tokens.register_forward_hook(_unsloth_offload_post_hook, prepend = True)embed_tokens._unsloth_offload_hooks_installed = True@@ -788,9 +798,14 @@ def _embedding_is_worth_offloading(input_embeddings):return size >= _OFFLOAD_EMBEDDING_MIN_BYTES and size / total >= _OFFLOAD_EMBEDDING_MIN_FRACTION-def _resolve_offload_embedding(model, offload_embedding):+def _resolve_offload_embedding(+ model,+ offload_embedding,+ needed = False,+):"""Report `offload_embedding` as True only when the offload will really run. It is a VRAM optimisation, not a correctness switch, so turn it off where it cannot help instead of failing the load. It also gates `_attach_bnb_multidevice_hooks`, which must still run whenever no offload happens, so every "no offload" case has to answer False. `"auto"` (the default) decides from the size of the embedding, and the declines below stay silent for it: they explain why something a caller asked for is not happening, and nobody asked for a default."""automatic = offload_embedding == OFFLOAD_EMBEDDING_AUTO+ # `needed`: a memory plan wants the room, so "auto" skips the size rule.def _decline(reason):if not automatic:@@ -819,7 +834,132 @@ def _resolve_offload_embedding(model, offload_embedding):if is_distributed():# The offload leaves embed_tokens on the CPU while the rest of the rank stays on CUDA; under full finetuning it is trainable, and DDP with device_ids refuses a module whose trainable parameters span both.return _decline("a distributed launch cannot wrap a model split across CPU and GPU.")- return _embedding_is_worth_offloading(in_embed) if automatic else True+ if automatic and (+ getattr(in_embed, "weight", None) is None or in_embed.weight.device.type in ("cpu", "meta")+ ):+ return False+ if not automatic:+ return True+ # "auto" follows free VRAM; an unsloth_zoo without the reserve estimate keeps the size rule.+ return needed or (_zoo_reserve_estimate is None and _embedding_is_worth_offloading(in_embed))+++# Other large token tables (Gemma 3n / 4 per-layer embeddings) move too; small position tables stay.+_EXTRA_EMBEDDING_MIN_BYTES = 256 * 2**20+++def _input_side_embeddings(model):+ """The input embedding plus every other large nn.Embedding not shared with the output head."""+ main = model.get_input_embeddings()+ out_embed = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None+ head = getattr(out_embed, "weight", None)+ found = [main]+ for module in model.modules():+ weight = getattr(module, "weight", None)+ if (+ module is main+ or not isinstance(module, torch.nn.Embedding)+ or weight is None+ or weight is head+ or weight.numel() * weight.element_size() < _EXTRA_EMBEDDING_MIN_BYTES+ ):+ continue+ found.append(module)+ return found, out_embed+++def _accelerator_device(model):+ # Headless backbones (AutoModel) have no head to say where the decoder runs.+ return next(+ (p.device for p in model.parameters() if p.device.type not in ("cpu", "meta")), None+ )+++def offload_input_embedding(model, embeddings = None):+ """Move the input embeddings to host RAM: lookups run there and only the looked-up rows go back+ to the decoder's card. Returns the bytes moved."""+ found, out_embed = _input_side_embeddings(model)+ head = getattr(out_embed, "weight", None)+ nbytes = 0+ for embedding in found if embeddings is None else embeddings:+ weight = embedding.weight+ if weight.device.type == "cpu":+ # Streamed to host by the block swap load: hook it (home = head's card) and move its buffers (embed_scale).+ if getattr(embedding, "_unsloth_offload_hooks_installed", False):+ continue+ _embed_device = head.device if head is not None else _accelerator_device(model)+ if _embed_device is None:+ _embed_device = torch.device("cpu")+ embedding.to("cpu")+ else:+ _embed_device = weight.device # decoder device, before offload+ embedding.to("cpu")+ nbytes += weight.numel() * weight.element_size()+ _install_offload_embedding_hooks(embedding, out_embed, _embed_device)+ print(f"Unsloth: Offloading embeddings to RAM to save {round(nbytes / 1024**3, 2)} GB.")+ # The transformers model, not a PEFT wrapper: its `device` property is the one callers read.+ _pin_device_to_decoder(model.get_base_model() if hasattr(model, "get_base_model") else model)+ # GPU memory must be freed explicitly or it will not be freed.+ clean_gpu_cache()+ gc.collect()+ return nbytes+++def offload_spare_embeddings(model, require_frozen = True):+ """offload_embedding = "auto" under memory pressure: move every input-side embedding that+ lm_head does not share (and, after get_peft_model, that is frozen) to host RAM. A tied input+ embedding stays, but Gemma 3n / 4 per-layer tables still go. Returns the bytes moved."""+ if _offload_embedding_unsupported_platform() is not None or is_distributed():+ return 0+ try:+ found, out_embed = _input_side_embeddings(model)+ except Exception:+ return 0+ # Already on the host only via the block swap load (unhooked, decoder on an accelerator); CPU models have nothing to offload.+ head = getattr(out_embed, "weight", None)+ if head is not None:+ accelerated = head.device.type not in ("cpu", "meta")+ else:+ accelerated = _accelerator_device(model) is not None+ candidates = []+ for embedding in found:+ weight = getattr(embedding, "weight", None)+ if (+ weight is None+ or weight.device.type == "meta"+ or (+ weight.device.type == "cpu"+ and (+ not accelerated or getattr(embedding, "_unsloth_offload_hooks_installed", False)+ )+ )+ or _embeddings_are_tied(embedding, out_embed)+ or _embedding_dispatch_device(embedding) is not None+ or (require_frozen and any(p.requires_grad for p in embedding.parameters()))+ ):+ continue+ candidates.append(embedding)+ return offload_input_embedding(model, candidates) if candidates else 0+++def restore_input_embedding(model):+ """Undo `offload_input_embedding` for the input embedding (training it needs it on the card).+ Frozen extra tables stay on the host. The hooks stay and become no-ops."""+ base = model.get_base_model() if hasattr(model, "get_base_model") else model+ if not getattr(base, "_unsloth_embedding_offloaded", False):+ return False+ embedding = model.get_input_embeddings()+ out_embed = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None+ weight = getattr(out_embed, "weight", None)+ if weight is None or weight.device.type == "cpu":+ return False+ if embedding.weight.device.type != "cpu":+ return False+ embedding.to(weight.device)+ base._unsloth_embedding_offloaded = any(+ e.weight.device.type == "cpu" for e in _input_side_embeddings(model)[0]+ )+ return TrueVLLM_SUPPORTED_VLM = [@@ -2487,6 +2627,26 @@ class FastBaseModel:user_config = kwargs.pop("config", None)if auto_config is None and user_config is not None:auto_config = user_config+ _block_swap_layers = kwargs.pop("block_swap_layers", 0)+ if _block_swap_layers and load_layers_to_host is None:+ _block_swap_layers = refuse_block_swap_load(+ _block_swap_layers,+ "needs a newer unsloth_zoo (`pip install --upgrade unsloth_zoo`); "+ "get_peft_model(block_swap_layers = ...) still swaps once the model is on the card.",+ )+ if _block_swap_layers and (fast_inference or full_finetuning):+ _block_swap_layers = refuse_block_swap_load(+ _block_swap_layers,+ "streams frozen LoRA base weights, so it cannot be combined with "+ "fast_inference or full_finetuning.",+ )+ if _block_swap_layers and (+ not torch.cuda.is_available() or is_integrated_unified_memory_gpu()+ ):+ _block_swap_layers = refuse_block_swap_load(+ _block_swap_layers,+ "needs a discrete CUDA or ROCm GPU; unified memory has no separate RAM to load into.",+ )# Offline snapshot for the loads below; not popped, so the weight load still reads local_files_only from **kwargs. See _get_effective_local_files_only.local_files_only = _get_effective_local_files_only(kwargs)@@ -2932,6 +3092,54 @@ class FastBaseModel:raise RuntimeError("Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!")+ if _block_swap_layers and block_swap_load_device(device_map) is None:+ _block_swap_layers = refuse_block_swap_load(+ _block_swap_layers,+ "needs every layer on one GPU; pass block_swap_layers to get_peft_model instead.",+ )+ _embedding_needed = None+ if _block_swap_layers == "auto":+ _block_swap_layers, device_map, _embedding_needed = resolve_auto_block_swap(+ requested_device_map(device_map),+ model_name,+ max_seq_length = max_seq_length,+ # Windows / WSL decline the offload later: a plan counting on it would leave a streamed table unhooked.+ offload_embedding = bool(offload_embedding)+ and _offload_embedding_unsupported_platform() is None,+ planner_kwargs = planner_kwargs_with_max_memory(device_map_planner_kwargs, kwargs),+ placement = "spread",+ skip_reason = _planner_skip_reason+ or (+ "the planner cannot rebuild this checkpoint's prepared config"+ if _planner_config is not None+ else None+ ),+ **planner_config_overrides(kwargs),+ token = token,+ trust_remote_code = trust_remote_code,+ **planner_hub_kwargs(kwargs),+ revision = _revision,+ **add_dtype_kwargs(torch_dtype),+ **planner_quantization_kwargs(+ **compressed_tensors_planner_quantization(+ auto_config, load_in_4bit, load_in_8bit, user_quantization_config+ ),+ rewritten_quantization_config = modelopt_planner_quantization_config(+ auto_config, dequantize = load_in_16bit+ )+ if _modelopt_rewritten+ else fp8_to_nf4_planner_quantization_config(+ auto_config,+ SKIP_QUANTIZATION_MODULES + _architecture_skip_modules(model_types),+ ),+ extra_skip_modules = _architecture_skip_modules(model_types) or None,+ ),+ )+ _block_swap_device = block_swap_load_device(device_map)+ if (_block_swap_layers or _embedding_needed) and _block_swap_device is not None:+ # A placement strategy would spill the host-bound weights to the CPU, which 4-bit loads refuse.+ device_map = {"": _block_swap_device}+ _block_swap_state = None# Prefetch the repo (killable child) so the in-process load is a cache hit. vLLM owns the download only when actually available; if it was requested but is missing, the load falls through in-process, so warm the weights here._vllm_owns_weights = fast_inference and is_vLLM_available()@@ -3120,6 +3328,8 @@ class FastBaseModel:raise_handler = RaiseUninitialized()try:+ # get_peft_model(block_swap_layers = "auto") may move it later, unless the caller said no.+ _offload_embedding_mode = False if fast_inference else offload_embeddingif offload_embedding and fast_inference:if offload_embedding != OFFLOAD_EMBEDDING_AUTO:print(@@ -3149,14 +3359,17 @@ class FastBaseModel:)):try:- model = auto_model.from_pretrained(- model_name,- config = model_config,- device_map = device_map,- token = token,- trust_remote_code = trust_remote_code,- **kwargs,- )+ with begin_block_swap_load(+ _block_swap_layers, device_map, embeddings = bool(_embedding_needed)+ ) as (_block_swap_state):+ model = auto_model.from_pretrained(+ model_name,+ config = model_config,+ device_map = device_map,+ token = token,+ trust_remote_code = trust_remote_code,+ **kwargs,+ )finally:# The load deep-copied the config; give the caller's object its fp8 block back.disarm_fp8_to_nf4(model_config)@@ -3182,13 +3395,15 @@ class FastBaseModel:offload_embedding,)- _attach_bnb_multidevice_hooks(- model,- load_in_4bit = load_in_4bit,- load_in_8bit = load_in_8bit,- offload_embedding = offload_embedding,- fast_inference = fast_inference,- )+ # One card by construction: the layers in host RAM are not an offloaded dispatch.+ if _block_swap_state is None:+ _attach_bnb_multidevice_hooks(+ model,+ load_in_4bit = load_in_4bit,+ load_in_8bit = load_in_8bit,+ offload_embedding = offload_embedding,+ fast_inference = fast_inference,+ )_no_placement_hooked = _hook_no_placement_ancestors(model)if _no_placement_hooked:logger.info(@@ -3230,24 +3445,17 @@ class FastBaseModel:if hasattr(model, "generate"):model.fast_generate = make_fast_generate_wrapper(model.generate)model.fast_generate_batches = error_out_no_vllm+ # "auto" moves it only when memory is tight, here and again in get_peft_model.+ model._unsloth_offload_embedding_mode = _offload_embedding_modeif offload_embedding:- embed_tokens = model.get_input_embeddings()- out_embed = (- model.get_output_embeddings()- if hasattr(model, "get_output_embeddings")- else None- )- nbytes = embed_tokens.weight.numel() * embed_tokens.weight.itemsize- ngb = round(nbytes / 1024 / 1024 / 1024, 2)- print(f"Unsloth: Offloading embeddings to RAM to save {ngb} GB.")- _embed_device = embed_tokens.weight.device # decoder device, before offload- embed_tokens.to("cpu")-- _install_offload_embedding_hooks(embed_tokens, out_embed, _embed_device)- _pin_device_to_decoder(model)- # GPU memory must be freed explicitly or it will not be freed.- clean_gpu_cache()- gc.collect()+ offload_input_embedding(model)+ elif _offload_embedding_mode == OFFLOAD_EMBEDDING_AUTO and _embedding_needed:+ _offload_embedding_for_room(model, require_frozen = False)+ elif _offload_embedding_mode == OFFLOAD_EMBEDDING_AUTO:+ offload_embedding_if_tight(model, max_seq_length, at_load = True)+ elif _offload_embedding_mode:+ # A tied input embedding stays; untied per-layer tables still move.+ offload_spare_embeddings(model, require_frozen = False)else:from unsloth_zoo.vllm_utils import (load_vllm,@@ -3695,9 +3903,20 @@ class FastBaseModel:_mark_loaded_revision(tokenizer, _tokenizer_revision)model = _mark_forced_float32(model, do_forced_float32)model = _mark_full_finetuning(model, full_finetuning)+ # Last, so the host copies carry the fp32 recasts the passes above make.+ finish_block_swap_load(+ model, _block_swap_state, planned_prefetch_depth(device_map_planner_kwargs)+ )# LAST, like the llama loader: patch_model_and_tokenizer below REPLACES the embedding and lm_head with fresh modules carrying the weights but not the _hf_hook, so a model split across cards would still meet the original cross-device index_select. Idempotent.- if not fast_inference and not offload_embedding:+ # Embeddings offloaded later sit on the host on purpose: a dispatch hook would pull them back.+ _base = model.get_base_model() if hasattr(model, "get_base_model") else model+ if (+ not fast_inference+ and not offload_embedding+ and not getattr(_base, "_unsloth_embedding_offloaded", False)+ and _block_swap_state is None+ ):try:_repaired = _repair_dispatch_hooks(model)if _repaired:@@ -3740,6 +3959,8 @@ class FastBaseModel:target_parameters = None, # For MoE expert layers (nn.Parameter)ensure_weight_tying = None, # None = auto (tie when we redirect a tied pair)finetune_audio_layers = False, # placed last to preserve existing positional argument order+ block_swap_layers = 0,+ checkpoint_skip_layers = 0,**kwargs,):if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":@@ -3842,6 +4063,9 @@ class FastBaseModel:)modules_to_save = _scope_modules_to_save_to_core(model, modules_to_save)_raise_if_fast_inference_modules_to_save(model, modules_to_save)+ if any(str(m).split(".")[-1] == "embed_tokens" for m in (modules_to_save or ())):+ if restore_input_embedding(model):+ print("Unsloth: Moved the offloaded input embedding back to the GPU to train it.")# Only a regex generated here may be widened to expert submodules below._target_modules_auto_regex = False@@ -4107,6 +4331,11 @@ class FastBaseModel:model.max_seq_length = max_seq_lengthfor module in model.modules():module.max_seq_length = max_seq_length+ offload_embedding_if_tight(model)+ install_block_swap(+ model, block_swap_layers, use_gradient_checkpointing = use_gradient_checkpointing+ )+ skip_checkpointing(model, checkpoint_skip_layers)for _ in range(3):gc.collect()clean_gpu_cache()