Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Fix MTPHead unpack crash in prefetch queue cleanup (CORE-644) (#17180)

Christian Byrnea9ba0f1a7e16
diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py
index 6cfe9921f..75dfaf786 100644
--- a/comfy/model_prefetch.py
+++ b/comfy/model_prefetch.py
@@ -123,20 +123,35 @@ def pin_modules(comfy_modules, device, dtype=None):
return offload_stream, fully_faulted
def cleanup_prefetched_modules(module, comfy_modules):
+ # Always drop the _prefetch marker, even if unpinning fails: a stale marker makes cast_bias_weight
+ # skip the fresh cast and read the old transfer state. Clean up every module, then raise the first error.
+ first_error = None
for s in comfy_modules:
prefetch = getattr(s, "_prefetch", None)
if prefetch is None:
continue
- for param_key in ("weight", "bias"):
- lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
- if lowvram_fn is not None:
- lowvram_fn.clear_prepared()
- if prefetch["signature"] is not None:
- comfy_aimdo.model_vbar.vbar_unpin(s._v)
- delattr(s, "_prefetch")
+ try:
+ for param_key in ("weight", "bias"):
+ lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
+ if lowvram_fn is not None:
+ lowvram_fn.clear_prepared()
+ if prefetch["signature"] is not None:
+ comfy_aimdo.model_vbar.vbar_unpin(s._v)
+ except Exception as e:
+ if first_error is None:
+ first_error = e
+ finally:
+ delattr(s, "_prefetch")
if getattr(module, "_v_block_faulted", False):
- comfy_aimdo.model_vbar.vbar_unpin(module._v_block)
- del module._v_block_faulted
+ try:
+ comfy_aimdo.model_vbar.vbar_unpin(module._v_block)
+ except Exception as e:
+ if first_error is None:
+ first_error = e
+ finally:
+ del module._v_block_faulted
+ if first_error is not None:
+ raise first_error
def _drop_graph(module):
graph = getattr(module, "_comfy_graph", None)
@@ -156,19 +171,35 @@ def cleanup_prefetch_queues():
global MALLOC_GRAPH_ROGUES
global MALLOC_GRAPH_USED
- cleanup_malloc_graph()
- for queue in PREFETCH_QUEUES:
- # Only the head has been prefetched; later entries are raw module groups.
- if queue[0] is not None:
- _, (prefetched_module, comfy_modules) = queue[0]
- cleanup_prefetched_modules(prefetched_module, comfy_modules)
+ queues = PREFETCH_QUEUES
+ # Reset before walking so a failing entry can't leave stale queues behind for every later node.
PREFETCH_QUEUES = []
+ first_error = None
+ try:
+ cleanup_malloc_graph()
+ except Exception as e:
+ first_error = e
+ for queue in queues:
+ # Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later
+ # entries, and a head that was not reached yet, still hold the raw module or module group.
+ head = queue[0] if queue else None
+ if not isinstance(head, tuple) or len(head) != 2 or not isinstance(head[1], tuple):
+ continue
+ prefetched_module, comfy_modules = head[1]
+ try:
+ cleanup_prefetched_modules(prefetched_module, comfy_modules)
+ except Exception as e:
+ # Keep unpinning the other queues; report the first failure once they are done.
+ if first_error is None:
+ first_error = e
GRAPH_WARMED_MODULES.clear()
if MALLOC_GRAPH_USED:
logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES)
MALLOC_GRAPH_BREAKS = 0
MALLOC_GRAPH_ROGUES = 0
MALLOC_GRAPH_USED = False
+ if first_error is not None:
+ raise first_error
def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_graph=False, generator=None, malloc_scope=None):
malloc_graph = MALLOC_GRAPHS.get(threading.get_ident())
diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py
index 4719c52dd..d4f241c5f 100644
--- a/comfy/text_encoders/qwen35.py
+++ b/comfy/text_encoders/qwen35.py
@@ -998,7 +998,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
draft_freqs.copy_(freqs_at(p))
host_rows = hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed)
queue = comfy.model_prefetch.make_prefetch_queue(
- [None if host_rows else embed, (head, self.mtp)], device,
+ [None if host_rows else embed, [head, self.mtp]], device,
{"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars})
def embed_core():
diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.py
new file mode 100644
index 000000000..4ccaa7147
--- /dev/null
+++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py
@@ -0,0 +1,128 @@
+import pytest
+import torch
+
+from comfy.cli_args import args as cli_args
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+import comfy.model_prefetch as model_prefetch # noqa: E402
+
+
+class MTPHead(torch.nn.Module):
+ pass
+
+
[email protected](autouse=True)
+def _reset_queues():
+ model_prefetch.PREFETCH_QUEUES = []
+ yield
+ model_prefetch.PREFETCH_QUEUES = []
+
+
+def _prefetched(module):
+ module._prefetch = {"signature": None}
+ return (None, (module, [module]))
+
+
[email protected]("pending_unit", [tuple, list])
+def test_cleanup_skips_unconsumed_grouped_unit(pending_unit):
+ # Qwen3.5 MTP draft queue interrupted after the embed pop: the embed slot has
+ # been pinned, the grouped (head, mtp) slot has not been reached yet.
+ embed = torch.nn.Embedding(4, 4)
+ head = torch.nn.Linear(4, 4)
+ queue = [_prefetched(embed), pending_unit((head, MTPHead())), None]
+ model_prefetch.PREFETCH_QUEUES = [queue]
+
+ model_prefetch.cleanup_prefetch_queues()
+
+ assert model_prefetch.PREFETCH_QUEUES == []
+ assert not hasattr(embed, "_prefetch")
+
+
+def test_cleanup_resets_queues_when_entry_cleanup_fails(monkeypatch):
+ def boom(module, comfy_modules):
+ raise RuntimeError("unpin failed")
+
+ monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", boom)
+ model_prefetch.PREFETCH_QUEUES = [[_prefetched(torch.nn.Linear(4, 4)), None]]
+
+ with pytest.raises(RuntimeError):
+ model_prefetch.cleanup_prefetch_queues()
+ # A failed cleanup must not poison every later node on the worker.
+ assert model_prefetch.PREFETCH_QUEUES == []
+ model_prefetch.cleanup_prefetch_queues()
+
+
[email protected]("pending_unit", [tuple, list])
+def test_cleanup_skips_unreached_grouped_head(pending_unit):
+ # Interrupted between popping the consumed slot and pinning the next one: the
+ # head of the queue is still the raw (head, mtp) group, not a pinned slot.
+ queue = [pending_unit((torch.nn.Linear(4, 4), MTPHead())), None]
+ model_prefetch.PREFETCH_QUEUES = [queue]
+
+ model_prefetch.cleanup_prefetch_queues()
+
+ assert model_prefetch.PREFETCH_QUEUES == []
+
+
+def test_cleanup_unpins_later_queues_when_an_earlier_one_fails(monkeypatch):
+ real_cleanup = model_prefetch.cleanup_prefetched_modules
+ failing = torch.nn.Linear(4, 4)
+ later = torch.nn.Linear(4, 4)
+
+ def cleanup(module, comfy_modules):
+ if module is failing:
+ raise RuntimeError("unpin failed")
+ real_cleanup(module, comfy_modules)
+
+ monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", cleanup)
+ model_prefetch.PREFETCH_QUEUES = [[_prefetched(failing), None], [_prefetched(later), None]]
+
+ with pytest.raises(RuntimeError, match="unpin failed"):
+ model_prefetch.cleanup_prefetch_queues()
+ # The first failure is re-raised, but only after every other queue was cleaned up.
+ assert not hasattr(later, "_prefetch")
+ assert model_prefetch.PREFETCH_QUEUES == []
+
+
+def test_failed_unpin_still_clears_prefetch_markers(monkeypatch):
+ # A module whose unpin raised must not keep _prefetch, or a later cast would skip
+ # the fresh cast and reuse the stale transfer state. The rest of the group is still cleaned.
+ calls = []
+
+ def vbar_unpin(v):
+ calls.append(v)
+ if v == "first":
+ raise RuntimeError("unpin failed")
+
+ monkeypatch.setattr(model_prefetch.comfy_aimdo.model_vbar, "vbar_unpin", vbar_unpin)
+ first = torch.nn.Linear(4, 4)
+ second = torch.nn.Linear(4, 4)
+ for module, v in ((first, "first"), (second, "second")):
+ module._v = v
+ module._prefetch = {"signature": "sig"}
+ model_prefetch.PREFETCH_QUEUES = [[(None, (first, [first, second])), None]]
+
+ with pytest.raises(RuntimeError, match="unpin failed"):
+ model_prefetch.cleanup_prefetch_queues()
+ assert calls == ["first", "second"]
+ assert not hasattr(first, "_prefetch")
+ assert not hasattr(second, "_prefetch")
+
+
+def test_malloc_graph_cleanup_failure_still_cleans_queues(monkeypatch):
+ # After a GPU failure the graph abort can raise too. Queues and _prefetch markers must not
+ # survive into the next node, and the graph error still reaches the caller.
+ def boom():
+ raise RuntimeError("graph abort failed")
+
+ monkeypatch.setattr(model_prefetch, "cleanup_malloc_graph", boom)
+ module = torch.nn.Linear(4, 4)
+ model_prefetch.PREFETCH_QUEUES = [[_prefetched(module), None]]
+
+ with pytest.raises(RuntimeError, match="graph abort failed"):
+ model_prefetch.cleanup_prefetch_queues()
+
+ assert model_prefetch.PREFETCH_QUEUES == []
+ assert not hasattr(module, "_prefetch")