Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep offloaded embeddings in host RAM when Trainer and accelerate place the model (#13320)
Daniel Han7c4061ea614a
diff --git a/tests/test_offload_embedding_trainer_move.py b/tests/test_offload_embedding_trainer_move.pynew file mode 100644index 00000000000..8c03fbe957b--- /dev/null+++ b/tests/test_offload_embedding_trainer_move.py@@ -0,0 +1,124 @@+"""An offloaded embedding must stay in host RAM through the model.to(device) that Trainer.train() and+Accelerator.prepare_model issue, else the first training step pulls it back onto the card+(unslothai/unsloth#7055: the 4.4 GB Gemma 4 E2B per-layer table on an 8 GB GPU). "meta" stands in for+the accelerator so this runs on CPU."""++import ast, functools, os, types+import torch+import torch.nn as nn++HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+VISION = os.path.join(HERE, "unsloth", "models", "vision.py")+NAMES = (+ "_host_offloaded_embeddings",+ "_wrap_move_keeps_host_embeddings",+ "_patch_trainer_move_keeps_host_embeddings",+)+++def _load():+ src = open(VISION, encoding = "utf-8").read()+ ns = {"torch": torch, "functools": functools}+ found = set()+ for node in ast.parse(src).body:+ if isinstance(node, ast.FunctionDef) and node.name in NAMES:+ exec(ast.get_source_segment(src, node), ns)+ found.add(node.name)+ assert found == set(NAMES), set(NAMES) - found+ return ns+++NS = _load()+++def _model(offloaded = True):+ model = nn.Module()+ model.embed = nn.Embedding(16, 4)+ model.inner = nn.Module()+ model.inner.per_layer = nn.Embedding(16, 4)+ model.proj = nn.Linear(4, 4)+ if offloaded:+ model.inner.per_layer._unsloth_offload_hooks_installed = True+ return model+++def _owner_from_trainer():+ from transformers.trainer import Trainer+ class Owner:+ _move_model_to_device = Trainer._move_model_to_device++ return Owner+++def _trainer_self():+ from transformers.training_args import ParallelMode+ return types.SimpleNamespace(+ args = types.SimpleNamespace(parallel_mode = ParallelMode.NOT_DISTRIBUTED)+ )+++def test_trainer_move_keeps_offloaded_table_on_host():+ Owner = _owner_from_trainer()+ NS["_wrap_move_keeps_host_embeddings"](Owner, "_move_model_to_device")+ model = _model()+ Owner._move_model_to_device(_trainer_self(), model, torch.device("meta"))+ assert model.proj.weight.device.type == "meta"+ assert model.embed.weight.device.type == "meta"+ assert model.inner.per_layer.weight.device.type == "cpu"+ # Only that call is affected: the module keeps its own _apply afterwards.+ assert "_apply" not in model.inner.per_layer.__dict__+ model.to("meta")+ assert model.inner.per_layer.weight.device.type == "meta"+++def test_unwrapped_trainer_move_pulls_the_table_back():+ # Negative control: what Trainer does without the wrapper.+ Owner = _owner_from_trainer()+ model = _model()+ Owner._move_model_to_device(_trainer_self(), model, torch.device("meta"))+ assert model.inner.per_layer.weight.device.type == "meta"+++def test_unmarked_or_restored_embedding_moves():+ class Owner:+ def prepare_model(+ self,+ model,+ device_placement = None,+ ):+ return model.to("meta")++ NS["_wrap_move_keeps_host_embeddings"](Owner, "prepare_model")+ model = _model(offloaded = False)+ Owner().prepare_model(model)+ assert model.inner.per_layer.weight.device.type == "meta"+ # Marked but back on an accelerator (restore_input_embedding) is not pinned either.+ model = _model()+ model.inner.per_layer.to("meta")+ assert NS["_host_offloaded_embeddings"](model) == []+++def test_wrap_is_idempotent_and_restores_on_error():+ class Owner:+ def prepare_model(self, model):+ raise RuntimeError("boom")++ NS["_wrap_move_keeps_host_embeddings"](Owner, "prepare_model")+ first = Owner.prepare_model+ NS["_wrap_move_keeps_host_embeddings"](Owner, "prepare_model")+ assert Owner.prepare_model is first+ model = _model()+ try:+ Owner().prepare_model(model)+ except RuntimeError:+ pass+ assert "_apply" not in model.inner.per_layer.__dict__+++def test_patch_targets_exist():+ # Drift guard: the methods that issue the move must still exist under these names.+ from transformers.trainer import Trainer+ from accelerate import Accelerator++ assert callable(getattr(Trainer, "_move_model_to_device", None))+ assert callable(getattr(Accelerator, "prepare_model", None))diff --git a/tests/test_offload_tied_autodisable.py b/tests/test_offload_tied_autodisable.pyindex e247b5c7cf7..cde03a15d87 100644--- a/tests/test_offload_tied_autodisable.py+++ b/tests/test_offload_tied_autodisable.py@@ -516,6 +516,7 @@ def test_headless_streamed_table_returns_rows_to_the_decoder_card():hooked = []ns["_install_offload_embedding_hooks"] = lambda emb, out, device: hooked.append((emb, device))ns["_pin_device_to_decoder"] = ns["clean_gpu_cache"] = lambda *a: None+ ns["_patch_trainer_move_keeps_host_embeddings"] = lambda: Nonens["gc"] = types.SimpleNamespace(collect = lambda: None)model = _HeadlessPerLayerModel("cuda").requires_grad_(False)model.per_layer.to("cpu")diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex ad43500772e..756607997aa 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -846,6 +846,53 @@ def _install_offload_embedding_hooks(embed_tokens, output_embeddings, return_devreturn True+def _host_offloaded_embeddings(model):+ return [+ module+ for module in model.modules()+ if getattr(module, "_unsloth_offload_hooks_installed", False)+ and getattr(getattr(module, "weight", None), "device", None) is not None+ and module.weight.device.type == "cpu"+ ]+++def _wrap_move_keeps_host_embeddings(owner, name):+ original = getattr(owner, name, None)+ if original is None or getattr(original, "_unsloth_keeps_host_embeddings", False):+ return++ @functools.wraps(original)+ def wrapped(self, model, *args, **kwargs):+ pinned = _host_offloaded_embeddings(model) if isinstance(model, torch.nn.Module) else []+ for module in pinned:+ # nn.Module._apply recurses through module._apply, so an instance attribute skips just this subtree.+ module._apply = lambda fn, recurse = True, _module = module: _module+ try:+ return original(self, model, *args, **kwargs)+ finally:+ for module in pinned:+ module.__dict__.pop("_apply", None)++ wrapped._unsloth_keeps_host_embeddings = True+ setattr(owner, name, wrapped)+++def _patch_trainer_move_keeps_host_embeddings():+ # Trainer.train() (under bf16/fp16 full eval, Unsloth's default) and Accelerator.prepare_model both call+ # model.to(device) when no hf_device_map is set, as on a block swap load; that pulled offloaded tables back onto+ # the card at the first step (the 4.4 GB Gemma 4 E2B per-layer table, unslothai/unsloth#7055).+ try:+ from transformers.trainer import Trainer+ _wrap_move_keeps_host_embeddings(Trainer, "_move_model_to_device")+ except Exception:+ pass+ try:+ from accelerate import Accelerator+ _wrap_move_keeps_host_embeddings(Accelerator, "prepare_model")+ except Exception:+ pass++def _pin_device_to_decoder(model):# `model.device` is the device of the first parameter, and the embedding we offload is it, so# the documented `inputs.to(model.device)` hands a CUDA model CPU ids. The lookup itself still@@ -1027,6 +1074,7 @@ def offload_input_embedding(model, embeddings = None):embedding.to("cpu")nbytes += weight.numel() * weight.element_size()_install_offload_embedding_hooks(embedding, out_embed, _embed_device)+ _patch_trainer_move_keeps_host_embeddings()print(f"Unsloth: Offloading embeddings to RAM to save {round(nbytes / 1024**3, 2)} GB.")# The transformers model, not a PEFT wrapper: its `device` property is the one callers read._pin_device_to_decoder(model.get_base_model() if hasattr(model, "get_base_model") else model)