Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep offloaded embeddings in host RAM when Trainer and accelerate place the model (#13320)

Daniel Han7c4061ea614a
diff --git a/tests/test_offload_embedding_trainer_move.py b/tests/test_offload_embedding_trainer_move.py
new file mode 100644
index 00000000000..8c03fbe957b
--- /dev/null
+++ b/tests/test_offload_embedding_trainer_move.py
@@ -0,0 +1,124 @@
+"""An offloaded embedding must stay in host RAM through the model.to(device) that Trainer.train() and
+Accelerator.prepare_model issue, else the first training step pulls it back onto the card
+(unslothai/unsloth#7055: the 4.4 GB Gemma 4 E2B per-layer table on an 8 GB GPU). "meta" stands in for
+the accelerator so this runs on CPU."""
+
+import ast, functools, os, types
+import torch
+import torch.nn as nn
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+VISION = os.path.join(HERE, "unsloth", "models", "vision.py")
+NAMES = (
+ "_host_offloaded_embeddings",
+ "_wrap_move_keeps_host_embeddings",
+ "_patch_trainer_move_keeps_host_embeddings",
+)
+
+
+def _load():
+ src = open(VISION, encoding = "utf-8").read()
+ ns = {"torch": torch, "functools": functools}
+ found = set()
+ for node in ast.parse(src).body:
+ if isinstance(node, ast.FunctionDef) and node.name in NAMES:
+ exec(ast.get_source_segment(src, node), ns)
+ found.add(node.name)
+ assert found == set(NAMES), set(NAMES) - found
+ return ns
+
+
+NS = _load()
+
+
+def _model(offloaded = True):
+ model = nn.Module()
+ model.embed = nn.Embedding(16, 4)
+ model.inner = nn.Module()
+ model.inner.per_layer = nn.Embedding(16, 4)
+ model.proj = nn.Linear(4, 4)
+ if offloaded:
+ model.inner.per_layer._unsloth_offload_hooks_installed = True
+ return model
+
+
+def _owner_from_trainer():
+ from transformers.trainer import Trainer
+ class Owner:
+ _move_model_to_device = Trainer._move_model_to_device
+
+ return Owner
+
+
+def _trainer_self():
+ from transformers.training_args import ParallelMode
+ return types.SimpleNamespace(
+ args = types.SimpleNamespace(parallel_mode = ParallelMode.NOT_DISTRIBUTED)
+ )
+
+
+def test_trainer_move_keeps_offloaded_table_on_host():
+ Owner = _owner_from_trainer()
+ NS["_wrap_move_keeps_host_embeddings"](Owner, "_move_model_to_device")
+ model = _model()
+ Owner._move_model_to_device(_trainer_self(), model, torch.device("meta"))
+ assert model.proj.weight.device.type == "meta"
+ assert model.embed.weight.device.type == "meta"
+ assert model.inner.per_layer.weight.device.type == "cpu"
+ # Only that call is affected: the module keeps its own _apply afterwards.
+ assert "_apply" not in model.inner.per_layer.__dict__
+ model.to("meta")
+ assert model.inner.per_layer.weight.device.type == "meta"
+
+
+def test_unwrapped_trainer_move_pulls_the_table_back():
+ # Negative control: what Trainer does without the wrapper.
+ Owner = _owner_from_trainer()
+ model = _model()
+ Owner._move_model_to_device(_trainer_self(), model, torch.device("meta"))
+ assert model.inner.per_layer.weight.device.type == "meta"
+
+
+def test_unmarked_or_restored_embedding_moves():
+ class Owner:
+ def prepare_model(
+ self,
+ model,
+ device_placement = None,
+ ):
+ return model.to("meta")
+
+ NS["_wrap_move_keeps_host_embeddings"](Owner, "prepare_model")
+ model = _model(offloaded = False)
+ Owner().prepare_model(model)
+ assert model.inner.per_layer.weight.device.type == "meta"
+ # Marked but back on an accelerator (restore_input_embedding) is not pinned either.
+ model = _model()
+ model.inner.per_layer.to("meta")
+ assert NS["_host_offloaded_embeddings"](model) == []
+
+
+def test_wrap_is_idempotent_and_restores_on_error():
+ class Owner:
+ def prepare_model(self, model):
+ raise RuntimeError("boom")
+
+ NS["_wrap_move_keeps_host_embeddings"](Owner, "prepare_model")
+ first = Owner.prepare_model
+ NS["_wrap_move_keeps_host_embeddings"](Owner, "prepare_model")
+ assert Owner.prepare_model is first
+ model = _model()
+ try:
+ Owner().prepare_model(model)
+ except RuntimeError:
+ pass
+ assert "_apply" not in model.inner.per_layer.__dict__
+
+
+def test_patch_targets_exist():
+ # Drift guard: the methods that issue the move must still exist under these names.
+ from transformers.trainer import Trainer
+ from accelerate import Accelerator
+
+ assert callable(getattr(Trainer, "_move_model_to_device", None))
+ assert callable(getattr(Accelerator, "prepare_model", None))
diff --git a/tests/test_offload_tied_autodisable.py b/tests/test_offload_tied_autodisable.py
index e247b5c7cf7..cde03a15d87 100644
--- a/tests/test_offload_tied_autodisable.py
+++ b/tests/test_offload_tied_autodisable.py
@@ -516,6 +516,7 @@ def test_headless_streamed_table_returns_rows_to_the_decoder_card():
hooked = []
ns["_install_offload_embedding_hooks"] = lambda emb, out, device: hooked.append((emb, device))
ns["_pin_device_to_decoder"] = ns["clean_gpu_cache"] = lambda *a: None
+ ns["_patch_trainer_move_keeps_host_embeddings"] = lambda: None
ns["gc"] = types.SimpleNamespace(collect = lambda: None)
model = _HeadlessPerLayerModel("cuda").requires_grad_(False)
model.per_layer.to("cpu")
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index ad43500772e..756607997aa 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -846,6 +846,53 @@ def _install_offload_embedding_hooks(embed_tokens, output_embeddings, return_dev
return True
+def _host_offloaded_embeddings(model):
+ return [
+ module
+ for module in model.modules()
+ if getattr(module, "_unsloth_offload_hooks_installed", False)
+ and getattr(getattr(module, "weight", None), "device", None) is not None
+ and module.weight.device.type == "cpu"
+ ]
+
+
+def _wrap_move_keeps_host_embeddings(owner, name):
+ original = getattr(owner, name, None)
+ if original is None or getattr(original, "_unsloth_keeps_host_embeddings", False):
+ return
+
+ @functools.wraps(original)
+ def wrapped(self, model, *args, **kwargs):
+ pinned = _host_offloaded_embeddings(model) if isinstance(model, torch.nn.Module) else []
+ for module in pinned:
+ # nn.Module._apply recurses through module._apply, so an instance attribute skips just this subtree.
+ module._apply = lambda fn, recurse = True, _module = module: _module
+ try:
+ return original(self, model, *args, **kwargs)
+ finally:
+ for module in pinned:
+ module.__dict__.pop("_apply", None)
+
+ wrapped._unsloth_keeps_host_embeddings = True
+ setattr(owner, name, wrapped)
+
+
+def _patch_trainer_move_keeps_host_embeddings():
+ # Trainer.train() (under bf16/fp16 full eval, Unsloth's default) and Accelerator.prepare_model both call
+ # model.to(device) when no hf_device_map is set, as on a block swap load; that pulled offloaded tables back onto
+ # the card at the first step (the 4.4 GB Gemma 4 E2B per-layer table, unslothai/unsloth#7055).
+ try:
+ from transformers.trainer import Trainer
+ _wrap_move_keeps_host_embeddings(Trainer, "_move_model_to_device")
+ except Exception:
+ pass
+ try:
+ from accelerate import Accelerator
+ _wrap_move_keeps_host_embeddings(Accelerator, "prepare_model")
+ except Exception:
+ pass
+
+
def _pin_device_to_decoder(model):
# `model.device` is the device of the first parameter, and the embedding we offload is it, so
# the documented `inputs.to(model.device)` hands a CUDA model CPU ids. The lookup itself still
@@ -1027,6 +1074,7 @@ def offload_input_embedding(model, embeddings = None):
embedding.to("cpu")
nbytes += weight.numel() * weight.element_size()
_install_offload_embedding_hooks(embedding, out_embed, _embed_device)
+ _patch_trainer_move_keeps_host_embeddings()
print(f"Unsloth: Offloading embeddings to RAM to save {round(nbytes / 1024**3, 2)} GB.")
# The transformers model, not a PEFT wrapper: its `device` property is the one callers read.
_pin_device_to_decoder(model.get_base_model() if hasattr(model, "get_base_model") else model)