Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Attach dispatch hooks on a num_labels load that asked for fast_inference (#10509)

Daniel Han82ee481a2803
diff --git a/tests/test_dispatch_hook_repair.py b/tests/test_dispatch_hook_repair.py
index 246dfecccaa..b9df80b98fe 100644
--- a/tests/test_dispatch_hook_repair.py
+++ b/tests/test_dispatch_hook_repair.py
@@ -144,6 +144,37 @@ def test_a_torch_device_cpu_entry_is_never_hooked_either():
)
+def _guards_of(tree, callee):
+ """The `if` conditions that decide whether `callee` runs, as expressions."""
+ import ast
+ return [
+ node.test
+ for node in ast.walk(tree)
+ if isinstance(node, ast.If)
+ and any(
+ isinstance(c, ast.Call) and getattr(c.func, "id", None) == callee
+ for c in ast.walk(node)
+ )
+ ]
+
+
+def _evaluate(expression, **names):
+ """Run a condition lifted out of the loader, with vLLM ownership stubbed to its contract."""
+ import ast
+
+ def _vllm_will_load_weights(fast_inference, num_labels = None):
+ # The real one probes the GPU and the vLLM install, neither of which a CPU runner has. Its
+ # one rule that matters here is asserted against the real function below.
+ return bool(fast_inference) and num_labels is None
+
+ scope = dict(names)
+ scope["_vllm_will_load_weights"] = _vllm_will_load_weights
+ return eval(
+ compile(ast.fix_missing_locations(ast.Expression(body = expression)), "<guard>", "eval"),
+ scope,
+ )
+
+
def test_the_llama_loader_stands_aside_under_vllm():
"""vLLM owns the weights; the HF tree this would hook is not what runs."""
import ast
@@ -162,24 +193,264 @@ def test_the_llama_loader_stands_aside_under_vllm():
]
assert calls, "the llama loader no longer repairs dispatch hooks at all"
- guarded = [
+ guarded = _guards_of(tree, "_repair_dispatch_hooks")
+ assert guarded, "the repair is no longer behind a condition at all"
+ # Evaluated, not matched by name: the guard is allowed to ask a predicate rather than read the raw
+ # flag, and either spelling has to keep vLLM out.
+ assert not any(_evaluate(guard, fast_inference = True, num_labels = None) for guard in guarded), (
+ "the repair runs on a real vLLM load, so a vLLM load gets accelerate hooks "
+ "on a module tree vLLM does not execute"
+ )
+
+
+def test_a_num_labels_load_is_hooked_even_when_fast_inference_was_asked_for():
+ """vLLM has no classification head, so `fast_inference` there is a request it never honours.
+
+ `AutoModelForSequenceClassification` is loaded in-process and can be split across cards, so
+ passing the raw flag on leaves it with no dispatch hooks and no end-of-load repair, and it dies
+ with `index is on cuda:0, different from other tensors on cuda:1`.
+ """
+ import ast
+ import inspect
+ import textwrap
+
+ from unsloth.models.llama import FastLlamaModel, _vllm_will_load_weights
+
+ # Ties the stub in _evaluate to the real predicate: vLLM never owns a num_labels load.
+ assert _vllm_will_load_weights(True, 2) is False
+
+ tree = ast.parse(textwrap.dedent(inspect.getsource(FastLlamaModel.from_pretrained)))
+ branches = [
node
for node in ast.walk(tree)
if isinstance(node, ast.If)
- and isinstance(node.test, ast.UnaryOp)
- and isinstance(node.test.op, ast.Not)
- and getattr(node.test.operand, "id", None) == "fast_inference"
+ and "num_labels" in ast.dump(node.test)
and any(
- isinstance(c, ast.Call) and getattr(c.func, "id", None) == "_repair_dispatch_hooks"
+ isinstance(c, ast.Call)
+ and getattr(c.func, "id", None) == "_attach_bnb_multidevice_hooks"
for c in ast.walk(node)
)
]
- assert guarded, (
- "the repair is no longer behind `if not fast_inference`, so a vLLM load "
- "gets accelerate hooks on a module tree vLLM does not execute"
+ assert len(branches) == 1, "no single `num_labels` branch attaches the hooks; this guard has gone vacuous"
+
+ passed = [
+ keyword.value
+ for call in ast.walk(branches[0])
+ if isinstance(call, ast.Call)
+ and getattr(call.func, "id", None) == "_attach_bnb_multidevice_hooks"
+ for keyword in call.keywords
+ if keyword.arg == "fast_inference"
+ ]
+ assert passed, "the classification load no longer says whether vLLM owns its weights"
+ assert not any(
+ _evaluate(value, fast_inference = True, num_labels = 2) for value in passed
+ ), (
+ "the classification load hands _attach_bnb_multidevice_hooks a truthy fast_inference, "
+ "which returns early, so a split bnb model gets no dispatch hooks"
+ )
+
+ guarded = _guards_of(tree, "_repair_dispatch_hooks")
+ assert guarded, "the repair is no longer behind a condition at all"
+ assert all(_evaluate(guard, fast_inference = True, num_labels = 2) for guard in guarded), (
+ "the end-of-load repair is skipped for a classification load, so the modules "
+ "post_patch rebuilt keep no hook and the split model still crosses devices"
+ )
+
+
+def _normalisation_block():
+ """The top-of-`from_pretrained` `if fast_inference:` block, as something runnable.
+
+ That block, not the caller, decides what `fast_inference` holds by the time the end-of-load
+ guard reads it: it clears the flag when vLLM is missing or the card is too old, and turns it back
+ on for hip. Lifting it keeps the tests below honest about which states are reachable instead of
+ asserting over states `from_pretrained` never produces.
+ """
+ import ast
+ import inspect
+ import textwrap
+
+ from unsloth.models.llama import FastLlamaModel
+
+ tree = ast.parse(textwrap.dedent(inspect.getsource(FastLlamaModel.from_pretrained)))
+ blocks = [
+ node
+ for node in tree.body[0].body
+ if isinstance(node, ast.If)
+ and isinstance(node.test, ast.Name)
+ and node.test.id == "fast_inference"
+ ]
+ assert len(blocks) == 1, (
+ "from_pretrained no longer normalises fast_inference in one top-level block, so what "
+ "reaches the end-of-load guard is not what this test models"
+ )
+ return ast.fix_missing_locations(ast.Module(body = blocks, type_ignores = []))
+
+
+def _normalise(fast_inference, num_labels = None):
+ """What `from_pretrained` leaves in `fast_inference` on the host the caller monkeypatched."""
+ import unsloth.models.llama as llama
+
+ scope = {
+ "os": __import__("os"),
+ "torch": torch,
+ "print": lambda *a, **k: None,
+ "logger": types.SimpleNamespace(warning_once = lambda *a, **k: None),
+ "DEVICE_TYPE": llama.DEVICE_TYPE,
+ "is_vLLM_available": llama.is_vLLM_available,
+ "_vllm_will_load_weights": llama._vllm_will_load_weights,
+ "fast_inference": fast_inference,
+ "num_labels": num_labels,
+ "revision": None,
+ "tokenizer_revision": None,
+ "unsloth_vllm_standby": False,
+ }
+ exec(compile(_normalisation_block(), "<normalise>", "exec"), scope)
+ return scope["fast_inference"]
+
+
+def _host(monkeypatch, device_type, vllm_installed, capability):
+ """Install one (accelerator, vLLM present?, compute capability) machine on the loader."""
+ import unsloth.models.llama as llama
+
+ monkeypatch.setattr(llama, "DEVICE_TYPE", device_type)
+ monkeypatch.setattr(llama, "is_vLLM_available", lambda: vllm_installed)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (capability, 0))
+
+
[email protected]("device_type", ["cuda", "hip", "xpu", "mlx"])
[email protected]("vllm_installed", [True, False])
[email protected]("capability", [6, 9])
[email protected]("fast_inference", [True, False])
+def test_asking_the_predicate_is_the_raw_flag_on_anything_but_a_classification_load(
+ monkeypatch, device_type, vllm_installed, capability, fast_inference
+):
+ """The end-of-load guard reads a predicate now, and that must change nothing else.
+
+ `from_pretrained` already cleared `fast_inference` for every reason the predicate would clear
+ it, so on a `num_labels = None` load the two spellings have to agree on every machine. If they
+ ever stop agreeing, a load whose weights came in through transformers gets no hook repair (or a
+ vLLM load gets hooks on a tree vLLM does not execute), and neither shows up as a failure until
+ someone splits a model across two cards.
+ """
+ import unsloth.models.llama as llama
+
+ _host(monkeypatch, device_type, vllm_installed, capability)
+ reached = _normalise(fast_inference, num_labels = None)
+ assert bool(llama._vllm_will_load_weights(reached, None)) == bool(reached), (
+ f"on {device_type} (vLLM installed = {vllm_installed}, capability = {capability}) a "
+ f"fast_inference = {fast_inference} load reaches the end of the load with "
+ f"fast_inference = {reached}, but the predicate answers "
+ f"{llama._vllm_will_load_weights(reached, None)}"
)
[email protected]("device_type", ["cuda", "hip", "xpu", "mlx"])
[email protected]("vllm_installed", [True, False])
[email protected]("capability", [6, 9])
+def test_a_classification_load_is_never_vllms_on_any_machine(
+ monkeypatch, device_type, vllm_installed, capability
+):
+ """The whole fix rests on this: there is no host where vLLM owns a `num_labels` load."""
+ import unsloth.models.llama as llama
+
+ _host(monkeypatch, device_type, vllm_installed, capability)
+ reached = _normalise(True, num_labels = 2)
+ assert llama._vllm_will_load_weights(reached, 2) is False
+ # num_labels = 0 is a real (if odd) classification request; `is not None` is the rule, not truth.
+ assert llama._vllm_will_load_weights(reached, 0) is False
+
+
+def test_the_predicate_probes_nothing_when_it_short_circuits(monkeypatch):
+ """The new call sites must not add a probe to loads that previously did none.
+
+ Both edited sites can be reached with `fast_inference` false or `num_labels` set, and neither
+ asked the vLLM install or the driver anything before. `import vllm`'s spec lookup and
+ `get_device_capability` are cheap but neither is free, and the second raises outright on a host
+ that reports DEVICE_TYPE "cuda" with no driver (UNSLOTH_ALLOW_CPU=1).
+ """
+ import unsloth.models.llama as llama
+
+ def _no(*args, **kwargs):
+ raise AssertionError("the predicate probed the machine after short-circuiting")
+
+ monkeypatch.setattr(llama, "DEVICE_TYPE", "cuda")
+ monkeypatch.setattr(llama, "is_vLLM_available", _no)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", _no)
+
+ assert llama._vllm_will_load_weights(False, None) is False
+ assert llama._vllm_will_load_weights(False, 2) is False
+ assert llama._vllm_will_load_weights(True, 2) is False
+
+
+class _Classifier(torch.nn.Module):
+ """A `...ForSequenceClassification` shape: a trunk, a `score` head, no output embedding."""
+
+ def __init__(self, device_map):
+ super().__init__()
+ self.model = torch.nn.Module()
+ self.model.embed_tokens = torch.nn.Embedding(8, 4)
+ self.model.layer = torch.nn.Linear(4, 4)
+ self.score = torch.nn.Linear(4, 2, bias = False)
+ self.hf_device_map = dict(device_map)
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def get_output_embeddings(self):
+ return None # a classification head is not an output embedding
+
+ def dispatch(self):
+ """What `dispatch_model` leaves behind: every far map entry carries a hook."""
+ for name, device in self.hf_device_map.items():
+ if str(device) in ("cpu", "disk"):
+ continue
+ add_hook_to_module(self.get_submodule(name), AlignDevicesHook(execution_device = device))
+ return self
+
+ def post_patch(self):
+ """What `patch_model_and_tokenizer` does: a NEW Embedding over the same weight."""
+ old = self.model.embed_tokens
+ self.model.embed_tokens = torch.nn.Embedding(8, 4, _weight = old.weight, _freeze = False)
+ return self
+
+
+def test_a_split_classification_model_gets_its_rebuilt_embedding_hooked():
+ """The end of the load is the only place that can fix this, which is why the guard matters.
+
+ A classification model answers None for its output embedding, so the input embedding is the
+ whole repair, and `score` sits on the near card with nothing to give back.
+ """
+ model = _Classifier(
+ {"model.embed_tokens": FAR, "model.layer": NEAR, "score": NEAR}
+ ).dispatch()
+ assert hasattr(model.model.embed_tokens, "_hf_hook"), "fixture never dispatched"
+
+ model.post_patch()
+ assert not hasattr(model.model.embed_tokens, "_hf_hook"), "fixture is not the broken state"
+
+ assert _repair()(model) == 1
+ hook = model.model.embed_tokens._hf_hook
+ assert str(hook.execution_device) == FAR, hook.execution_device
+ assert not hasattr(model.score, "_hf_hook"), "the near-card head has no hook to give back"
+
+
+ "device_map",
+ [
+ {"": NEAR},
+ {"model": NEAR, "score": NEAR},
+ {"model.embed_tokens": "disk", "model.layer": NEAR},
+ ],
+ ids = ["single_device", "two_entries_one_device", "far_entry_is_offload"],
+)
+def test_running_the_repair_on_an_unsplit_classification_load_costs_it_nothing(device_map):
+ """Turning the repair on for `num_labels` loads must be free for everyone not split."""
+ model = _Classifier(device_map).post_patch()
+ before = _hooked(model)
+ assert _repair()(model) == 0
+ assert _hooked(model) == before
+
+
def test_a_name_the_model_does_not_have_is_skipped_not_invented():
model = _Model({"embed_tokens": FAR, "does.not.exist": FAR, "layer": NEAR})
assert _repair()(model) == 1, "the one real far-device name was not repaired"
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 4e8f0e1b78d..91643ece053 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2655,7 +2655,9 @@ class FastLlamaModel:
and not _head.weight.is_floating_point()
):
_head.to(dtype)
- # Attach dispatch hooks for bnb multi-device loads.
+ # Attach dispatch hooks for bnb multi-device loads. The hooks stand aside only when vLLM
+ # owns the weights, which it never does here: vLLM has no classification head, so this
+ # branch loaded the weights in-process even though the caller asked for fast_inference.
from unsloth.models.vision import _attach_bnb_multidevice_hooks
_attach_bnb_multidevice_hooks(
@@ -2663,7 +2665,7 @@ class FastLlamaModel:
load_in_4bit = load_in_4bit,
load_in_8bit = kwargs.get("load_in_8bit", False),
offload_embedding = False,
- fast_inference = fast_inference,
+ fast_inference = _vllm_will_load_weights(fast_inference, num_labels),
)
# Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200), reading
# scales from the same revision as the weights.
@@ -2987,8 +2989,9 @@ class FastLlamaModel:
# LAST: post_patch replaces the embedding modules and the QKV/MLP patching below replaces the
# forwards a hook wraps, so an earlier attach is lost. Skipped under vLLM, which owns the
- # weights.
- if not fast_inference:
+ # weights. Not the raw flag: a num_labels load stayed in-process above, so the weights this
+ # repairs are the weights that run.
+ if not _vllm_will_load_weights(fast_inference, num_labels):
try:
from unsloth.models.vision import _repair_dispatch_hooks
_repaired = _repair_dispatch_hooks(model)