Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add LTXVAddLatentGuide for pinning a pre-encoded latent as a guide (#16176)

Alex Artyomov0d0b6b5694df
diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.py
index ef622481c..e9e5d5c51 100644
--- a/comfy_extras/nodes_lt.py
+++ b/comfy_extras/nodes_lt.py
@@ -428,6 +428,43 @@ class LTXVAddGuide(io.ComfyNode):
return latent_image, noise_mask
+ @classmethod
+ def attach_guide_latent(cls, positive, negative, latent_image, noise_mask, guide_latent, frame_idx, strength,
+ scale_factors, latent_downscale_factor=1, causal_fix=None, attention_mask=None):
+ """Dilate a guide latent onto the canvas, pin it, and record its attention entry.
+
+ Keeps the three values that have to agree in one place: the pre-dilation
+ shape recorded on the attention entry, the post-dilation token count, and
+ the downscale factor handed to append_keyframe. context_windows re-derives
+ the factor from the first two, so they must not drift apart.
+ """
+ guide_latent_shape = list(guide_latent.shape[2:]) # pre-dilation [F, H, W] for spatial-mask downsampling
+ guide_mask = None
+ if latent_downscale_factor > 1:
+ guide_latent, guide_mask = cls.dilate_latent(guide_latent, latent_downscale_factor)
+
+ positive, negative, latent_image, noise_mask = cls.append_keyframe(
+ positive,
+ negative,
+ frame_idx,
+ latent_image,
+ noise_mask,
+ guide_latent,
+ strength,
+ scale_factors,
+ guide_mask=guide_mask,
+ latent_downscale_factor=latent_downscale_factor,
+ causal_fix=causal_fix,
+ )
+
+ # Track this guide for per-reference attention control.
+ pre_filter_count = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
+ positive, negative = _append_guide_attention_entry(
+ positive, negative, pre_filter_count, guide_latent_shape, strength=strength,
+ attention_mask=attention_mask,
+ )
+ return positive, negative, latent_image, noise_mask
+
@classmethod
def execute(cls, positive, negative, vae, latent, image, frame_idx, strength, attention_mask=None, iclora_parameters=None) -> io.NodeOutput:
scale_factors = vae.downscale_index_formula
@@ -462,32 +499,20 @@ class LTXVAddGuide(io.ComfyNode):
t = t[:, :, 1:, :, :]
image = image[1:]
- guide_latent_shape = list(t.shape[2:]) # pre-dilation [F, H, W] for spatial-mask downsampling
- guide_mask = None
- if latent_downscale_factor > 1:
- t, guide_mask = cls.dilate_latent(t, latent_downscale_factor)
-
frame_idx, latent_idx = cls.get_latent_index(positive, latent_length, len(image), frame_idx, scale_factors, latent_shape=latent_image.shape)
assert latent_idx + t.shape[2] <= latent_length, "Conditioning frames exceed the length of the latent sequence."
- positive, negative, latent_image, noise_mask = cls.append_keyframe(
+ positive, negative, latent_image, noise_mask = cls.attach_guide_latent(
positive,
negative,
- frame_idx,
latent_image,
noise_mask,
t,
+ frame_idx,
strength,
scale_factors,
- guide_mask=guide_mask,
latent_downscale_factor=latent_downscale_factor,
causal_fix=causal_fix,
- )
-
- # Track this guide for per-reference attention control.
- pre_filter_count = t.shape[2] * t.shape[3] * t.shape[4]
- positive, negative = _append_guide_attention_entry(
- positive, negative, pre_filter_count, guide_latent_shape, strength=strength,
attention_mask=attention_mask,
)
@@ -496,6 +521,130 @@ class LTXVAddGuide(io.ComfyNode):
generate = execute # TODO: remove
+class LTXVAddLatentGuide(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="LTXVAddLatentGuide",
+ display_name="LTXV Add Latent Guide",
+ category="model/conditioning/ltxv",
+ description="Pins an already-encoded latent as a guide, for when the guide comes out of an "
+ "earlier stage rather than an image. Same effect as LTXV Add Guide without the "
+ "VAE decode/encode round trip. A guide that is spatially smaller than the target "
+ "(an IC-LoRA or detailing reference) is dilated onto a sparse grid, and its RoPE "
+ "end positions are expanded by the same ratio so it covers the target canvas "
+ "instead of addressing only the top-left corner of it.",
+ search_aliases=["latent guide", "add latent guide", "guide latent"],
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Latent.Input("latent", tooltip="Target video latent the guide is pinned onto."),
+ io.Latent.Input(
+ "guiding_latent",
+ tooltip="Guide latent. Its spatial size must divide the target's by the same whole "
+ "number on both axes; equal size pins it as-is, half size is treated as an "
+ "x2 IC-LoRA reference.",
+ ),
+ io.Int.Input(
+ "latent_idx",
+ default=0,
+ min=-9999,
+ max=9999,
+ tooltip="Latent frame index to start the guide at, counted in latent frames rather "
+ "than pixel frames. Negative values place the guide on frames before the "
+ "start of the latent, not counted back from its end.",
+ ),
+ io.Float.Input(
+ "strength",
+ default=1.0,
+ min=0.0,
+ max=1.0,
+ step=0.01,
+ tooltip="Capped at 1.0. A dilated guide marks its padding positions with a "
+ "negative denoise mask so the model drops them; above 1.0 the kept "
+ "positions would go negative too and the whole guide would be dropped. "
+ "Amplify beyond 1.0 with attention_mask instead.",
+ ),
+ io.Mask.Input(
+ "attention_mask",
+ optional=True,
+ tooltip="Optional pixel-space spatial mask. Controls per-region "
+ "conditioning influence via self-attention, multiplied by strength.",
+ ),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, positive, negative, vae, latent, guiding_latent, latent_idx, strength, attention_mask=None) -> io.NodeOutput:
+ scale_factors = vae.downscale_index_formula
+ latent_image = latent["samples"]
+ noise_mask = get_noise_mask(latent)
+ guide_latent = guiding_latent["samples"]
+
+ for name, samples in (("latent", latent_image), ("guiding_latent", guide_latent)):
+ if samples.ndim != 5:
+ raise ValueError(
+ f"{name} must be a 5D video latent (batch, channels, frames, height, width), "
+ f"got shape {list(samples.shape)}."
+ )
+
+ guide_frames = guide_latent.shape[2]
+ latent_frames = latent_image.shape[2]
+ if latent_idx + guide_frames > latent_frames:
+ raise ValueError(
+ f"Guide of {guide_frames} latent frame(s) at latent_idx {latent_idx} runs past the "
+ f"end of the {latent_frames}-frame latent. Negative values are allowed and place the "
+ f"guide before the start of the latent."
+ )
+
+ if latent_image.shape[3] % guide_latent.shape[3] != 0 or latent_image.shape[4] % guide_latent.shape[4] != 0:
+ raise ValueError(
+ f"Guiding latent spatial size {guide_latent.shape[3]}x{guide_latent.shape[4]} must divide "
+ f"the target size {latent_image.shape[3]}x{latent_image.shape[4]} by a whole number."
+ )
+
+ height_scale = latent_image.shape[3] // guide_latent.shape[3]
+ width_scale = latent_image.shape[4] // guide_latent.shape[4]
+ # dilate_latent and append_keyframe take a single IC-LoRA downscale factor for both
+ # axes, so a non-square ratio would mis-place RoPE on one of them.
+ if height_scale != width_scale:
+ raise ValueError(
+ f"Guiding latent spatial ratio must be square, got height x{height_scale} and "
+ f"width x{width_scale} ({guide_latent.shape[3]}x{guide_latent.shape[4]} -> "
+ f"{latent_image.shape[3]}x{latent_image.shape[4]})."
+ )
+
+ time_scale_factor = scale_factors[0]
+ if latent_idx <= 0:
+ frame_idx = latent_idx * time_scale_factor
+ else:
+ frame_idx = 1 + (latent_idx - 1) * time_scale_factor
+
+ positive, negative, latent_image, noise_mask = LTXVAddGuide.attach_guide_latent(
+ positive,
+ negative,
+ latent_image,
+ noise_mask,
+ guide_latent,
+ frame_idx,
+ strength,
+ scale_factors,
+ latent_downscale_factor=width_scale,
+ attention_mask=attention_mask,
+ )
+
+ out = latent.copy()
+ out["samples"] = latent_image
+ out["noise_mask"] = noise_mask
+ return io.NodeOutput(positive, negative, out)
+
+
class LTXVCropGuides(io.ComfyNode):
@classmethod
def define_schema(cls):
@@ -1186,6 +1335,7 @@ class LtxvExtension(ComfyExtension):
LTXVScheduler,
GetICLoRAParameters,
LTXVAddGuide,
+ LTXVAddLatentGuide,
LTXVPreprocess,
LTXVCropGuides,
LTXVConcatAVLatent,
diff --git a/tests-unit/comfy_extras_test/nodes_lt_keyframes_test.py b/tests-unit/comfy_extras_test/nodes_lt_keyframes_test.py
index b8a49cf77..ace8f3c6c 100644
--- a/tests-unit/comfy_extras_test/nodes_lt_keyframes_test.py
+++ b/tests-unit/comfy_extras_test/nodes_lt_keyframes_test.py
@@ -5,9 +5,10 @@ They cover keyframe placement, conditioning metadata, guide conversion, and free
from __future__ import annotations
+import sys
from contextlib import contextmanager
from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock
import pytest
import torch
@@ -132,15 +133,39 @@ _nodes_lt_stub.get_keyframe_idxs = _get_keyframe_idxs
_nodes_lt_stub._append_guide_attention_entry = _append_guide_attention_entry
_nodes_lt_stub.LTXVAddGuide = _StubAddGuide
-with patch.dict(
- "sys.modules",
- {
+def _import_keyframes_against_stub():
+ """Import the module under test with comfy_extras.nodes_lt stubbed, then put it back.
+
+ Only the stubbed keys are restored, not the whole of sys.modules: patch.dict
+ restores the entire dict on exit, which evicts every module imported inside the
+ block and forces a later re-import. Re-importing torch internals raises on
+ duplicate TORCH_LIBRARY registration, which broke running this file alongside
+ nodes_lt_test.py.
+ """
+ stubs = {
"nodes": mock_nodes,
"server": mock_server,
"comfy_extras.nodes_lt": _nodes_lt_stub,
- },
-):
- import comfy_extras.nodes_lt_keyframes as keyframes
+ }
+ saved = {name: sys.modules.get(name) for name in stubs}
+ sys.modules.update(stubs)
+ try:
+ import comfy_extras.nodes_lt_keyframes as module
+
+ return module
+ finally:
+ for name, original in saved.items():
+ if original is None:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = original
+ # The imported module stays bound to the stubs above, so drop it from the cache
+ # rather than let a later import pick up a stub-backed copy. The reference
+ # returned to this module keeps working.
+ sys.modules.pop("comfy_extras.nodes_lt_keyframes", None)
+
+
+keyframes = _import_keyframes_against_stub()
def _zeros(shape):
diff --git a/tests-unit/comfy_extras_test/nodes_lt_test.py b/tests-unit/comfy_extras_test/nodes_lt_test.py
new file mode 100644
index 000000000..44087d944
--- /dev/null
+++ b/tests-unit/comfy_extras_test/nodes_lt_test.py
@@ -0,0 +1,242 @@
+"""Unit tests for LTXVAddLatentGuide and the guide-attachment path it shares with LTXVAddGuide.
+
+The RoPE arithmetic runs for real here: only ``nodes`` and ``server`` are stubbed, so
+``append_keyframe``, ``dilate_latent`` and ``_append_guide_attention_entry`` are the
+real implementations and the assertions are on actual keyframe coordinates.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+
+# Stub nodes/server for the import only, then restore exactly those keys. patch.dict is
+# not used because it restores the whole of sys.modules on exit, which evicts everything
+# imported inside the block and forces a re-import that trips duplicate TORCH_LIBRARY
+# registration. Leaving the stubs installed is equally wrong: pytest imports every test
+# module at collection time, so a lingering MagicMock "nodes" breaks later modules that
+# use the real one. Same shape as tests-unit/comfy_extras_test/image_stitch_test.py.
+_stubs = {"nodes": MagicMock(MAX_RESOLUTION=16384), "server": MagicMock()}
+_saved = {name: sys.modules.get(name) for name in _stubs}
+sys.modules.update(_stubs)
+try:
+ import comfy_extras.nodes_lt as nodes_lt
+finally:
+ for _name, _original in _saved.items():
+ if _original is None:
+ sys.modules.pop(_name, None)
+ else:
+ sys.modules[_name] = _original
+
+LATENT_CHANNELS = 128
+SCALE_FACTORS = (8, 32, 32)
+TIME, HEIGHT, WIDTH = 0, 1, 2
+START, END = 0, 1
+
+
+def _vae():
+ return SimpleNamespace(downscale_index_formula=SCALE_FACTORS)
+
+
+def _latent(frames, height, width):
+ return {"samples": torch.zeros((1, LATENT_CHANNELS, frames, height, width))}
+
+
+def _cond():
+ return [({}, {})]
+
+
+def _add_latent_guide(guide_hw, latent_hw=(4, 4), guide_frames=1, latent_frames=3, latent_idx=0):
+ positive, negative, latent = nodes_lt.LTXVAddLatentGuide.execute(
+ _cond(),
+ _cond(),
+ _vae(),
+ _latent(latent_frames, *latent_hw),
+ _latent(guide_frames, *guide_hw),
+ latent_idx,
+ 1.0,
+ )
+ metadata = positive[0][1]
+ return metadata["keyframe_idxs"], metadata["guide_attention_entries"], latent
+
+
+def _axis(keyframe_idxs, axis, bound):
+ return keyframe_idxs[0, axis, :, bound].tolist()
+
+
+def test_same_size_guide_spans_one_patch_per_token():
+ """A 1:1 guide gets no offset: each token's end is one scale factor past its start.
+
+ The inverse of the case below, so a factor derived wrongly at 1:1 is caught too.
+ """
+ keyframe_idxs, entries, _ = _add_latent_guide(guide_hw=(4, 4))
+
+ for axis in (HEIGHT, WIDTH):
+ starts = _axis(keyframe_idxs, axis, START)
+ assert _axis(keyframe_idxs, axis, END) == [s + SCALE_FACTORS[axis] for s in starts]
+
+ assert entries[0]["latent_shape"] == [1, 4, 4]
+
+
+def test_half_size_guide_expands_only_the_end_positions():
+ """An x2 guide keeps its start positions and pushes each end out by one scale factor.
+
+ That is what makes the dilated reference cover the whole canvas. Leaving the factor
+ at 1 keeps same-size coordinates while each token encodes a larger patch, so the
+ reference addresses only the top-left corner of the target.
+ """
+ same_size, _, _ = _add_latent_guide(guide_hw=(4, 4))
+ downscaled, entries, _ = _add_latent_guide(guide_hw=(2, 2))
+
+ # Dilation puts the small guide on the same sparse grid, so token count is unchanged.
+ assert downscaled.shape == same_size.shape
+
+ for axis in (HEIGHT, WIDTH):
+ assert _axis(downscaled, axis, START) == _axis(same_size, axis, START)
+ expected = [e + SCALE_FACTORS[axis] for e in _axis(same_size, axis, END)]
+ assert _axis(downscaled, axis, END) == expected
+
+ # Time is never touched by the spatial offset.
+ assert _axis(downscaled, TIME, START) == _axis(same_size, TIME, START)
+ assert _axis(downscaled, TIME, END) == _axis(same_size, TIME, END)
+
+ assert entries[0]["latent_shape"] == [1, 2, 2]
+
+
+def test_attention_entry_lets_context_windows_rederive_the_factor():
+ """The entry keeps the pre-dilation shape while the token count is post-dilation.
+
+ ``context_windows`` divides the post-dilation guide height by the entry's
+ ``latent_shape`` height to recover the downscale factor, so these two must not drift
+ apart or windowed and non-windowed sampling disagree on the guide's RoPE.
+ """
+ _, entries, latent = _add_latent_guide(guide_hw=(2, 2))
+
+ entry = entries[0]
+ assert entry["latent_shape"] == [1, 2, 2] # pre-dilation
+ assert entry["pre_filter_count"] == 1 * 4 * 4 # post-dilation
+ assert latent["samples"].shape[3] // entry["latent_shape"][1] == 2
+
+
+ "latent_idx, expected_start", [(-1, -8), (0, 0), (1, 1), (2, 9)]
+)
+def test_latent_idx_maps_onto_pixel_frames(latent_idx, expected_start):
+ """latent_idx is in latent frames, and negatives sit before the start of the latent.
+
+ The first latent frame covers a single pixel frame, so the mapping is 0 -> 0, 1 -> 1,
+ then 8 apart. Negative values are not counted back from the end.
+ """
+ keyframe_idxs, _, _ = _add_latent_guide(
+ guide_hw=(4, 4), latent_frames=8, latent_idx=latent_idx
+ )
+
+ assert set(_axis(keyframe_idxs, TIME, START)) == {expected_start}
+
+
+ "kwargs, message",
+ [
+ (dict(guide_hw=(2, 4)), "square"),
+ (dict(guide_hw=(3, 3)), "whole number"),
+ (dict(guide_hw=(4, 4), latent_idx=99), "runs past the end"),
+ (dict(guide_hw=(4, 4), guide_frames=5), "runs past the end"),
+ ],
+)
+def test_unusable_guides_are_rejected(kwargs, message):
+ with pytest.raises(ValueError, match=message):
+ _add_latent_guide(**kwargs)
+
+
+def test_non_5d_guiding_latent_is_rejected():
+ """An image-model latent would otherwise fail with a bare IndexError on shape[4]."""
+ with pytest.raises(ValueError, match="5D video latent"):
+ nodes_lt.LTXVAddLatentGuide.execute(
+ _cond(),
+ _cond(),
+ _vae(),
+ _latent(3, 4, 4),
+ {"samples": torch.zeros((1, LATENT_CHANNELS, 4, 4))},
+ 0,
+ 1.0,
+ )
+
+
+def test_attention_mask_reaches_the_guide_entry():
+ """Only coverage that the optional input is forwarded to the guide entry."""
+ mask = torch.full((1, 128, 128), 0.5)
+ positive, negative, _ = nodes_lt.LTXVAddLatentGuide.execute(
+ _cond(), _cond(), _vae(), _latent(3, 4, 4), _latent(1, 2, 2), 0, 1.0, attention_mask=mask
+ )
+
+ # Stored as (1, 1, F, H, W) for downstream self-attention masking.
+ assert positive[0][1]["guide_attention_entries"][0]["pixel_mask"].shape == (1, 1, 1, 128, 128)
+ # Positive and negative each get their own entry, so neither leaks into the other.
+ assert len(negative[0][1]["guide_attention_entries"]) == 1
+
+
+def test_node_is_registered_with_a_loadable_schema():
+ """Both failure modes here are invisible to every other test in this file.
+
+ A bad io.Schema keyword only surfaces when the node is registered, and a node left
+ out of the extension list simply does not exist in ComfyUI. The strength cap is
+ asserted here because raising it re-exposes the missing clamp in append_keyframe's
+ guide_mask branch, where a dilated guide is dropped above 1.0.
+ """
+ schema = nodes_lt.LTXVAddLatentGuide.define_schema()
+ inputs = {inp.id: inp for inp in schema.inputs}
+
+ assert schema.node_id == "LTXVAddLatentGuide"
+ assert inputs["strength"].max == 1.0
+ assert inputs["attention_mask"].optional is True
+
+ node_list = asyncio.run(nodes_lt.LtxvExtension().get_node_list())
+ assert nodes_lt.LTXVAddLatentGuide in node_list
+
+
+ "iclora_parameters, expected_shape, expected_extra_end",
+ [
+ (None, [1, 4, 4], 0),
+ ({"reference_downscale_factor": 2}, [1, 2, 2], SCALE_FACTORS[HEIGHT]),
+ ],
+)
+def test_add_guide_image_path_still_routes_through_the_shared_helper(
+ iclora_parameters, expected_shape, expected_extra_end
+):
+ """LTXVAddGuide must be unchanged by sharing attach_guide_latent with the latent node."""
+
+ class _Vae:
+ downscale_index_formula = SCALE_FACTORS
+
+ def encode(self, pixels):
+ frames, height, width, _ = pixels.shape
+ return torch.zeros(
+ (1, LATENT_CHANNELS, (frames - 1) // SCALE_FACTORS[TIME] + 1, height // 32, width // 32)
+ )
+
+ positive, _, _ = nodes_lt.LTXVAddGuide.execute(
+ _cond(),
+ _cond(),
+ _Vae(),
+ _latent(3, 4, 4),
+ torch.zeros((1, 4 * 32, 4 * 32, 3)),
+ 0,
+ 1.0,
+ iclora_parameters=iclora_parameters,
+ )
+
+ metadata = positive[0][1]
+ entry = metadata["guide_attention_entries"][0]
+ assert entry["latent_shape"] == expected_shape
+ assert entry["pre_filter_count"] == 1 * 4 * 4
+
+ keyframe_idxs = metadata["keyframe_idxs"]
+ starts = _axis(keyframe_idxs, HEIGHT, START)
+ ends = _axis(keyframe_idxs, HEIGHT, END)
+ assert ends == [s + SCALE_FACTORS[HEIGHT] + expected_extra_end for s in starts]