The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Fix MTPHead unpack crash in prefetch queue cleanup (CORE-644) (#17180)
Christian Byrnea9ba0f1a7e16
diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.pyindex 6cfe9921f..75dfaf786 100644--- a/comfy/model_prefetch.py+++ b/comfy/model_prefetch.py@@ -123,20 +123,35 @@ def pin_modules(comfy_modules, device, dtype=None):return offload_stream, fully_faulteddef cleanup_prefetched_modules(module, comfy_modules):+ # Always drop the _prefetch marker, even if unpinning fails: a stale marker makes cast_bias_weight+ # skip the fresh cast and read the old transfer state. Clean up every module, then raise the first error.+ first_error = Nonefor s in comfy_modules:prefetch = getattr(s, "_prefetch", None)if prefetch is None:continue- for param_key in ("weight", "bias"):- lowvram_fn = getattr(s, param_key + "_lowvram_function", None)- if lowvram_fn is not None:- lowvram_fn.clear_prepared()- if prefetch["signature"] is not None:- comfy_aimdo.model_vbar.vbar_unpin(s._v)- delattr(s, "_prefetch")+ try:+ for param_key in ("weight", "bias"):+ lowvram_fn = getattr(s, param_key + "_lowvram_function", None)+ if lowvram_fn is not None:+ lowvram_fn.clear_prepared()+ if prefetch["signature"] is not None:+ comfy_aimdo.model_vbar.vbar_unpin(s._v)+ except Exception as e:+ if first_error is None:+ first_error = e+ finally:+ delattr(s, "_prefetch")if getattr(module, "_v_block_faulted", False):- comfy_aimdo.model_vbar.vbar_unpin(module._v_block)- del module._v_block_faulted+ try:+ comfy_aimdo.model_vbar.vbar_unpin(module._v_block)+ except Exception as e:+ if first_error is None:+ first_error = e+ finally:+ del module._v_block_faulted+ if first_error is not None:+ raise first_errordef _drop_graph(module):graph = getattr(module, "_comfy_graph", None)@@ -156,19 +171,35 @@ def cleanup_prefetch_queues():global MALLOC_GRAPH_ROGUESglobal MALLOC_GRAPH_USED- cleanup_malloc_graph()- for queue in PREFETCH_QUEUES:- # Only the head has been prefetched; later entries are raw module groups.- if queue[0] is not None:- _, (prefetched_module, comfy_modules) = queue[0]- cleanup_prefetched_modules(prefetched_module, comfy_modules)+ queues = PREFETCH_QUEUES+ # Reset before walking so a failing entry can't leave stale queues behind for every later node.PREFETCH_QUEUES = []+ first_error = None+ try:+ cleanup_malloc_graph()+ except Exception as e:+ first_error = e+ for queue in queues:+ # Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later+ # entries, and a head that was not reached yet, still hold the raw module or module group.+ head = queue[0] if queue else None+ if not isinstance(head, tuple) or len(head) != 2 or not isinstance(head[1], tuple):+ continue+ prefetched_module, comfy_modules = head[1]+ try:+ cleanup_prefetched_modules(prefetched_module, comfy_modules)+ except Exception as e:+ # Keep unpinning the other queues; report the first failure once they are done.+ if first_error is None:+ first_error = eGRAPH_WARMED_MODULES.clear()if MALLOC_GRAPH_USED:logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES)MALLOC_GRAPH_BREAKS = 0MALLOC_GRAPH_ROGUES = 0MALLOC_GRAPH_USED = False+ if first_error is not None:+ raise first_errordef prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_graph=False, generator=None, malloc_scope=None):malloc_graph = MALLOC_GRAPHS.get(threading.get_ident())diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.pyindex 4719c52dd..d4f241c5f 100644--- a/comfy/text_encoders/qwen35.py+++ b/comfy/text_encoders/qwen35.py@@ -998,7 +998,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):draft_freqs.copy_(freqs_at(p))host_rows = hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed)queue = comfy.model_prefetch.make_prefetch_queue(- [None if host_rows else embed, (head, self.mtp)], device,+ [None if host_rows else embed, [head, self.mtp]], device,{"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars})def embed_core():diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.pynew file mode 100644index 000000000..4ccaa7147--- /dev/null+++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py@@ -0,0 +1,128 @@+import pytest+import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.model_prefetch as model_prefetch # noqa: E402+++class MTPHead(torch.nn.Module):+ pass++[email protected](autouse=True)+def _reset_queues():+ model_prefetch.PREFETCH_QUEUES = []+ yield+ model_prefetch.PREFETCH_QUEUES = []+++def _prefetched(module):+ module._prefetch = {"signature": None}+ return (None, (module, [module]))++[email protected]("pending_unit", [tuple, list])+def test_cleanup_skips_unconsumed_grouped_unit(pending_unit):+ # Qwen3.5 MTP draft queue interrupted after the embed pop: the embed slot has+ # been pinned, the grouped (head, mtp) slot has not been reached yet.+ embed = torch.nn.Embedding(4, 4)+ head = torch.nn.Linear(4, 4)+ queue = [_prefetched(embed), pending_unit((head, MTPHead())), None]+ model_prefetch.PREFETCH_QUEUES = [queue]++ model_prefetch.cleanup_prefetch_queues()++ assert model_prefetch.PREFETCH_QUEUES == []+ assert not hasattr(embed, "_prefetch")+++def test_cleanup_resets_queues_when_entry_cleanup_fails(monkeypatch):+ def boom(module, comfy_modules):+ raise RuntimeError("unpin failed")++ monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", boom)+ model_prefetch.PREFETCH_QUEUES = [[_prefetched(torch.nn.Linear(4, 4)), None]]++ with pytest.raises(RuntimeError):+ model_prefetch.cleanup_prefetch_queues()+ # A failed cleanup must not poison every later node on the worker.+ assert model_prefetch.PREFETCH_QUEUES == []+ model_prefetch.cleanup_prefetch_queues()++[email protected]("pending_unit", [tuple, list])+def test_cleanup_skips_unreached_grouped_head(pending_unit):+ # Interrupted between popping the consumed slot and pinning the next one: the+ # head of the queue is still the raw (head, mtp) group, not a pinned slot.+ queue = [pending_unit((torch.nn.Linear(4, 4), MTPHead())), None]+ model_prefetch.PREFETCH_QUEUES = [queue]++ model_prefetch.cleanup_prefetch_queues()++ assert model_prefetch.PREFETCH_QUEUES == []+++def test_cleanup_unpins_later_queues_when_an_earlier_one_fails(monkeypatch):+ real_cleanup = model_prefetch.cleanup_prefetched_modules+ failing = torch.nn.Linear(4, 4)+ later = torch.nn.Linear(4, 4)++ def cleanup(module, comfy_modules):+ if module is failing:+ raise RuntimeError("unpin failed")+ real_cleanup(module, comfy_modules)++ monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", cleanup)+ model_prefetch.PREFETCH_QUEUES = [[_prefetched(failing), None], [_prefetched(later), None]]++ with pytest.raises(RuntimeError, match="unpin failed"):+ model_prefetch.cleanup_prefetch_queues()+ # The first failure is re-raised, but only after every other queue was cleaned up.+ assert not hasattr(later, "_prefetch")+ assert model_prefetch.PREFETCH_QUEUES == []+++def test_failed_unpin_still_clears_prefetch_markers(monkeypatch):+ # A module whose unpin raised must not keep _prefetch, or a later cast would skip+ # the fresh cast and reuse the stale transfer state. The rest of the group is still cleaned.+ calls = []++ def vbar_unpin(v):+ calls.append(v)+ if v == "first":+ raise RuntimeError("unpin failed")++ monkeypatch.setattr(model_prefetch.comfy_aimdo.model_vbar, "vbar_unpin", vbar_unpin)+ first = torch.nn.Linear(4, 4)+ second = torch.nn.Linear(4, 4)+ for module, v in ((first, "first"), (second, "second")):+ module._v = v+ module._prefetch = {"signature": "sig"}+ model_prefetch.PREFETCH_QUEUES = [[(None, (first, [first, second])), None]]++ with pytest.raises(RuntimeError, match="unpin failed"):+ model_prefetch.cleanup_prefetch_queues()+ assert calls == ["first", "second"]+ assert not hasattr(first, "_prefetch")+ assert not hasattr(second, "_prefetch")+++def test_malloc_graph_cleanup_failure_still_cleans_queues(monkeypatch):+ # After a GPU failure the graph abort can raise too. Queues and _prefetch markers must not+ # survive into the next node, and the graph error still reaches the caller.+ def boom():+ raise RuntimeError("graph abort failed")++ monkeypatch.setattr(model_prefetch, "cleanup_malloc_graph", boom)+ module = torch.nn.Linear(4, 4)+ model_prefetch.PREFETCH_QUEUES = [[_prefetched(module), None]]++ with pytest.raises(RuntimeError, match="graph abort failed"):+ model_prefetch.cleanup_prefetch_queues()++ assert model_prefetch.PREFETCH_QUEUES == []+ assert not hasattr(module, "_prefetch")