Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep model.device on the decoder while the embedding is offloaded to RAM (#10569)

Daniel Han5f9f4c50fd66
diff --git a/tests/test_offload_device_property.py b/tests/test_offload_device_property.py
new file mode 100644
index 00000000000..30e4ca234ae
--- /dev/null
+++ b/tests/test_offload_device_property.py
@@ -0,0 +1,128 @@
+"""Tests _pin_device_to_decoder in vision.py: while the embedding is offloaded to RAM,
+`model.device` must report the decoder device, because `inputs.to(model.device)` is the
+documented idiom and CPU ids make generation build position_ids on the wrong device.
+No GPU needed: a meta parameter stands in for the accelerator."""
+
+import ast, os
+import torch
+import torch.nn as nn
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+VISION = os.path.join(HERE, "unsloth", "models", "vision.py")
+
+
+def _load_pinner():
+ src = open(VISION, encoding = "utf-8").read()
+ mod = ast.parse(src)
+ for node in mod.body:
+ if isinstance(node, ast.FunctionDef) and node.name == "_pin_device_to_decoder":
+ ns = {"torch": torch}
+ exec(ast.get_source_segment(src, node), ns)
+ return ns["_pin_device_to_decoder"]
+ raise AssertionError("_pin_device_to_decoder not found in vision.py")
+
+
+pin = _load_pinner()
+DECODER = "meta" if not torch.cuda.is_available() else "cuda"
+
+
+class _Base(nn.Module):
+ """Stands in for transformers' ModuleUtilsMixin.device: first parameter wins."""
+
+ @property
+ def device(self):
+ return next(self.parameters()).device
+
+
+def _model(cls, decoder_device = DECODER):
+ model = cls()
+ model.embed_tokens = nn.Embedding(32, 8)
+ model.layer = nn.Linear(8, 8, bias = False)
+ model.layer.to(decoder_device)
+ return model
+
+
+def test_reports_decoder_device_while_offloaded():
+ class _M(_Base):
+ pass
+
+ model = _model(_M)
+ assert model.device.type == "cpu", "embedding must be first, or the repro is not reproduced"
+ assert pin(model) is True
+ assert model.device.type == DECODER, model.device
+
+
+def test_untouched_model_of_same_class_is_unaffected():
+ # The property lives on the class, so a second instance that never offloaded must
+ # keep the stock answer.
+ class _M(_Base):
+ pass
+
+ pin(_model(_M))
+ assert _model(_M).device.type == "cpu"
+
+
+def test_whole_model_on_cpu_falls_back():
+ # model.to("cpu") after an offload: there is no accelerator parameter left to report.
+ class _M(_Base):
+ pass
+
+ model = _model(_M, decoder_device = "cpu")
+ pin(model)
+ assert model.device.type == "cpu", model.device
+
+
+def test_follows_a_later_move():
+ class _M(_Base):
+ pass
+
+ model = _model(_M, decoder_device = "cpu")
+ pin(model)
+ model.layer.to(DECODER)
+ assert model.device.type == DECODER, model.device
+
+
+def test_flag_adds_no_parameters_or_state():
+ class _M(_Base):
+ pass
+
+ model = _model(_M)
+ before = list(model.state_dict())
+ pin(model)
+ assert list(model.state_dict()) == before
+ assert [n for n, _ in model.named_parameters()] == ["embed_tokens.weight", "layer.weight"]
+
+
+def test_repeated_pin_is_idempotent():
+ class _M(_Base):
+ pass
+
+ model = _model(_M)
+ pin(model)
+ pin(model)
+ assert model.device.type == DECODER, model.device
+
+
+def test_class_without_device_property_is_left_alone():
+ class _M(nn.Module):
+ pass
+
+ assert pin(_model(_M)) is False
+
+
+if __name__ == "__main__":
+ test_reports_decoder_device_while_offloaded()
+ print("[PASS] offloaded model reports the decoder device")
+ test_untouched_model_of_same_class_is_unaffected()
+ print("[PASS] non-offloaded instance of the same class unaffected")
+ test_whole_model_on_cpu_falls_back()
+ print("[PASS] all-cpu model falls back to the stock answer")
+ test_follows_a_later_move()
+ print("[PASS] a later .to() move is followed")
+ test_flag_adds_no_parameters_or_state()
+ print("[PASS] no extra parameters or state_dict keys")
+ test_repeated_pin_is_idempotent()
+ print("[PASS] repeated pin is idempotent")
+ test_class_without_device_property_is_left_alone()
+ print("[PASS] a class with no device property is left alone")
+ print("OK: model.device reports the decoder device while the embedding is offloaded")
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index c2d8968bb73..e17ffd384d5 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -464,6 +464,34 @@ def _install_offload_embedding_hooks(embed_tokens, output_embeddings, return_dev
return True
+def _pin_device_to_decoder(model):
+ # `model.device` is the device of the first parameter, and the embedding we offload is it, so
+ # the documented `inputs.to(model.device)` hands a CUDA model CPU ids. The lookup itself still
+ # works through the hooks above, but `cache_position` is built from `input_ids.device`, so
+ # `position_ids` stays on the CPU while the hidden states are already on CUDA and the rotary
+ # embedding dies on a cpu/cuda matmul. Report the first parameter still on an accelerator
+ # instead, read live so `model.to()` moves are followed and a whole-model move to the CPU
+ # falls back to the original answer.
+ cls = type(model)
+ if not cls.__dict__.get("_unsloth_device_skips_offload", False):
+ original = getattr(cls, "device", None)
+ if not isinstance(original, property):
+ return False
+
+ def _unsloth_device(self):
+ if getattr(self, "_unsloth_embedding_offloaded", False):
+ for param in self.parameters():
+ if param.device.type != "cpu":
+ return param.device
+ return original.fget(self)
+
+ cls.device = property(_unsloth_device)
+ cls._unsloth_device_skips_offload = True
+ # A plain bool, so nn.Module.__setattr__ leaves it off the parameter and module registries.
+ model._unsloth_embedding_offloaded = True
+ return True
+
+
def _embeddings_are_tied(input_embeddings, output_embeddings):
# A tied lm_head reuses this weight, so offloading to CPU would strand the output projection.
if input_embeddings is None or output_embeddings is None:
@@ -1709,6 +1737,7 @@ class FastBaseModel:
# Device-safe embedding offload.
_install_offload_embedding_hooks(embed_tokens, out_embed, _embed_device)
+ _pin_device_to_decoder(model)
# GPU memory must be freed explicitly or it will not be freed.
clean_gpu_cache()
gc.collect()