The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add LTXVAddLatentGuide for pinning a pre-encoded latent as a guide (#16176)
Alex Artyomov0d0b6b5694df
diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.pyindex ef622481c..e9e5d5c51 100644--- a/comfy_extras/nodes_lt.py+++ b/comfy_extras/nodes_lt.py@@ -428,6 +428,43 @@ class LTXVAddGuide(io.ComfyNode):return latent_image, noise_mask+ @classmethod+ def attach_guide_latent(cls, positive, negative, latent_image, noise_mask, guide_latent, frame_idx, strength,+ scale_factors, latent_downscale_factor=1, causal_fix=None, attention_mask=None):+ """Dilate a guide latent onto the canvas, pin it, and record its attention entry.++ Keeps the three values that have to agree in one place: the pre-dilation+ shape recorded on the attention entry, the post-dilation token count, and+ the downscale factor handed to append_keyframe. context_windows re-derives+ the factor from the first two, so they must not drift apart.+ """+ guide_latent_shape = list(guide_latent.shape[2:]) # pre-dilation [F, H, W] for spatial-mask downsampling+ guide_mask = None+ if latent_downscale_factor > 1:+ guide_latent, guide_mask = cls.dilate_latent(guide_latent, latent_downscale_factor)++ positive, negative, latent_image, noise_mask = cls.append_keyframe(+ positive,+ negative,+ frame_idx,+ latent_image,+ noise_mask,+ guide_latent,+ strength,+ scale_factors,+ guide_mask=guide_mask,+ latent_downscale_factor=latent_downscale_factor,+ causal_fix=causal_fix,+ )++ # Track this guide for per-reference attention control.+ pre_filter_count = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]+ positive, negative = _append_guide_attention_entry(+ positive, negative, pre_filter_count, guide_latent_shape, strength=strength,+ attention_mask=attention_mask,+ )+ return positive, negative, latent_image, noise_mask+@classmethoddef execute(cls, positive, negative, vae, latent, image, frame_idx, strength, attention_mask=None, iclora_parameters=None) -> io.NodeOutput:scale_factors = vae.downscale_index_formula@@ -462,32 +499,20 @@ class LTXVAddGuide(io.ComfyNode):t = t[:, :, 1:, :, :]image = image[1:]- guide_latent_shape = list(t.shape[2:]) # pre-dilation [F, H, W] for spatial-mask downsampling- guide_mask = None- if latent_downscale_factor > 1:- t, guide_mask = cls.dilate_latent(t, latent_downscale_factor)-frame_idx, latent_idx = cls.get_latent_index(positive, latent_length, len(image), frame_idx, scale_factors, latent_shape=latent_image.shape)assert latent_idx + t.shape[2] <= latent_length, "Conditioning frames exceed the length of the latent sequence."- positive, negative, latent_image, noise_mask = cls.append_keyframe(+ positive, negative, latent_image, noise_mask = cls.attach_guide_latent(positive,negative,- frame_idx,latent_image,noise_mask,t,+ frame_idx,strength,scale_factors,- guide_mask=guide_mask,latent_downscale_factor=latent_downscale_factor,causal_fix=causal_fix,- )-- # Track this guide for per-reference attention control.- pre_filter_count = t.shape[2] * t.shape[3] * t.shape[4]- positive, negative = _append_guide_attention_entry(- positive, negative, pre_filter_count, guide_latent_shape, strength=strength,attention_mask=attention_mask,)@@ -496,6 +521,130 @@ class LTXVAddGuide(io.ComfyNode):generate = execute # TODO: remove+class LTXVAddLatentGuide(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="LTXVAddLatentGuide",+ display_name="LTXV Add Latent Guide",+ category="model/conditioning/ltxv",+ description="Pins an already-encoded latent as a guide, for when the guide comes out of an "+ "earlier stage rather than an image. Same effect as LTXV Add Guide without the "+ "VAE decode/encode round trip. A guide that is spatially smaller than the target "+ "(an IC-LoRA or detailing reference) is dilated onto a sparse grid, and its RoPE "+ "end positions are expanded by the same ratio so it covers the target canvas "+ "instead of addressing only the top-left corner of it.",+ search_aliases=["latent guide", "add latent guide", "guide latent"],+ inputs=[+ io.Conditioning.Input("positive"),+ io.Conditioning.Input("negative"),+ io.Vae.Input("vae"),+ io.Latent.Input("latent", tooltip="Target video latent the guide is pinned onto."),+ io.Latent.Input(+ "guiding_latent",+ tooltip="Guide latent. Its spatial size must divide the target's by the same whole "+ "number on both axes; equal size pins it as-is, half size is treated as an "+ "x2 IC-LoRA reference.",+ ),+ io.Int.Input(+ "latent_idx",+ default=0,+ min=-9999,+ max=9999,+ tooltip="Latent frame index to start the guide at, counted in latent frames rather "+ "than pixel frames. Negative values place the guide on frames before the "+ "start of the latent, not counted back from its end.",+ ),+ io.Float.Input(+ "strength",+ default=1.0,+ min=0.0,+ max=1.0,+ step=0.01,+ tooltip="Capped at 1.0. A dilated guide marks its padding positions with a "+ "negative denoise mask so the model drops them; above 1.0 the kept "+ "positions would go negative too and the whole guide would be dropped. "+ "Amplify beyond 1.0 with attention_mask instead.",+ ),+ io.Mask.Input(+ "attention_mask",+ optional=True,+ tooltip="Optional pixel-space spatial mask. Controls per-region "+ "conditioning influence via self-attention, multiplied by strength.",+ ),+ ],+ outputs=[+ io.Conditioning.Output(display_name="positive"),+ io.Conditioning.Output(display_name="negative"),+ io.Latent.Output(display_name="latent"),+ ],+ )++ @classmethod+ def execute(cls, positive, negative, vae, latent, guiding_latent, latent_idx, strength, attention_mask=None) -> io.NodeOutput:+ scale_factors = vae.downscale_index_formula+ latent_image = latent["samples"]+ noise_mask = get_noise_mask(latent)+ guide_latent = guiding_latent["samples"]++ for name, samples in (("latent", latent_image), ("guiding_latent", guide_latent)):+ if samples.ndim != 5:+ raise ValueError(+ f"{name} must be a 5D video latent (batch, channels, frames, height, width), "+ f"got shape {list(samples.shape)}."+ )++ guide_frames = guide_latent.shape[2]+ latent_frames = latent_image.shape[2]+ if latent_idx + guide_frames > latent_frames:+ raise ValueError(+ f"Guide of {guide_frames} latent frame(s) at latent_idx {latent_idx} runs past the "+ f"end of the {latent_frames}-frame latent. Negative values are allowed and place the "+ f"guide before the start of the latent."+ )++ if latent_image.shape[3] % guide_latent.shape[3] != 0 or latent_image.shape[4] % guide_latent.shape[4] != 0:+ raise ValueError(+ f"Guiding latent spatial size {guide_latent.shape[3]}x{guide_latent.shape[4]} must divide "+ f"the target size {latent_image.shape[3]}x{latent_image.shape[4]} by a whole number."+ )++ height_scale = latent_image.shape[3] // guide_latent.shape[3]+ width_scale = latent_image.shape[4] // guide_latent.shape[4]+ # dilate_latent and append_keyframe take a single IC-LoRA downscale factor for both+ # axes, so a non-square ratio would mis-place RoPE on one of them.+ if height_scale != width_scale:+ raise ValueError(+ f"Guiding latent spatial ratio must be square, got height x{height_scale} and "+ f"width x{width_scale} ({guide_latent.shape[3]}x{guide_latent.shape[4]} -> "+ f"{latent_image.shape[3]}x{latent_image.shape[4]})."+ )++ time_scale_factor = scale_factors[0]+ if latent_idx <= 0:+ frame_idx = latent_idx * time_scale_factor+ else:+ frame_idx = 1 + (latent_idx - 1) * time_scale_factor++ positive, negative, latent_image, noise_mask = LTXVAddGuide.attach_guide_latent(+ positive,+ negative,+ latent_image,+ noise_mask,+ guide_latent,+ frame_idx,+ strength,+ scale_factors,+ latent_downscale_factor=width_scale,+ attention_mask=attention_mask,+ )++ out = latent.copy()+ out["samples"] = latent_image+ out["noise_mask"] = noise_mask+ return io.NodeOutput(positive, negative, out)++class LTXVCropGuides(io.ComfyNode):@classmethoddef define_schema(cls):@@ -1186,6 +1335,7 @@ class LtxvExtension(ComfyExtension):LTXVScheduler,GetICLoRAParameters,LTXVAddGuide,+ LTXVAddLatentGuide,LTXVPreprocess,LTXVCropGuides,LTXVConcatAVLatent,diff --git a/tests-unit/comfy_extras_test/nodes_lt_keyframes_test.py b/tests-unit/comfy_extras_test/nodes_lt_keyframes_test.pyindex b8a49cf77..ace8f3c6c 100644--- a/tests-unit/comfy_extras_test/nodes_lt_keyframes_test.py+++ b/tests-unit/comfy_extras_test/nodes_lt_keyframes_test.py@@ -5,9 +5,10 @@ They cover keyframe placement, conditioning metadata, guide conversion, and freefrom __future__ import annotations+import sysfrom contextlib import contextmanagerfrom types import SimpleNamespace-from unittest.mock import MagicMock, patch+from unittest.mock import MagicMockimport pytestimport torch@@ -132,15 +133,39 @@ _nodes_lt_stub.get_keyframe_idxs = _get_keyframe_idxs_nodes_lt_stub._append_guide_attention_entry = _append_guide_attention_entry_nodes_lt_stub.LTXVAddGuide = _StubAddGuide-with patch.dict(- "sys.modules",- {+def _import_keyframes_against_stub():+ """Import the module under test with comfy_extras.nodes_lt stubbed, then put it back.++ Only the stubbed keys are restored, not the whole of sys.modules: patch.dict+ restores the entire dict on exit, which evicts every module imported inside the+ block and forces a later re-import. Re-importing torch internals raises on+ duplicate TORCH_LIBRARY registration, which broke running this file alongside+ nodes_lt_test.py.+ """+ stubs = {"nodes": mock_nodes,"server": mock_server,"comfy_extras.nodes_lt": _nodes_lt_stub,- },-):- import comfy_extras.nodes_lt_keyframes as keyframes+ }+ saved = {name: sys.modules.get(name) for name in stubs}+ sys.modules.update(stubs)+ try:+ import comfy_extras.nodes_lt_keyframes as module++ return module+ finally:+ for name, original in saved.items():+ if original is None:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = original+ # The imported module stays bound to the stubs above, so drop it from the cache+ # rather than let a later import pick up a stub-backed copy. The reference+ # returned to this module keeps working.+ sys.modules.pop("comfy_extras.nodes_lt_keyframes", None)+++keyframes = _import_keyframes_against_stub()def _zeros(shape):diff --git a/tests-unit/comfy_extras_test/nodes_lt_test.py b/tests-unit/comfy_extras_test/nodes_lt_test.pynew file mode 100644index 000000000..44087d944--- /dev/null+++ b/tests-unit/comfy_extras_test/nodes_lt_test.py@@ -0,0 +1,242 @@+"""Unit tests for LTXVAddLatentGuide and the guide-attachment path it shares with LTXVAddGuide.++The RoPE arithmetic runs for real here: only ``nodes`` and ``server`` are stubbed, so+``append_keyframe``, ``dilate_latent`` and ``_append_guide_attention_entry`` are the+real implementations and the assertions are on actual keyframe coordinates.+"""++from __future__ import annotations++import asyncio+import sys+from types import SimpleNamespace+from unittest.mock import MagicMock++import pytest+import torch++# Stub nodes/server for the import only, then restore exactly those keys. patch.dict is+# not used because it restores the whole of sys.modules on exit, which evicts everything+# imported inside the block and forces a re-import that trips duplicate TORCH_LIBRARY+# registration. Leaving the stubs installed is equally wrong: pytest imports every test+# module at collection time, so a lingering MagicMock "nodes" breaks later modules that+# use the real one. Same shape as tests-unit/comfy_extras_test/image_stitch_test.py.+_stubs = {"nodes": MagicMock(MAX_RESOLUTION=16384), "server": MagicMock()}+_saved = {name: sys.modules.get(name) for name in _stubs}+sys.modules.update(_stubs)+try:+ import comfy_extras.nodes_lt as nodes_lt+finally:+ for _name, _original in _saved.items():+ if _original is None:+ sys.modules.pop(_name, None)+ else:+ sys.modules[_name] = _original++LATENT_CHANNELS = 128+SCALE_FACTORS = (8, 32, 32)+TIME, HEIGHT, WIDTH = 0, 1, 2+START, END = 0, 1+++def _vae():+ return SimpleNamespace(downscale_index_formula=SCALE_FACTORS)+++def _latent(frames, height, width):+ return {"samples": torch.zeros((1, LATENT_CHANNELS, frames, height, width))}+++def _cond():+ return [({}, {})]+++def _add_latent_guide(guide_hw, latent_hw=(4, 4), guide_frames=1, latent_frames=3, latent_idx=0):+ positive, negative, latent = nodes_lt.LTXVAddLatentGuide.execute(+ _cond(),+ _cond(),+ _vae(),+ _latent(latent_frames, *latent_hw),+ _latent(guide_frames, *guide_hw),+ latent_idx,+ 1.0,+ )+ metadata = positive[0][1]+ return metadata["keyframe_idxs"], metadata["guide_attention_entries"], latent+++def _axis(keyframe_idxs, axis, bound):+ return keyframe_idxs[0, axis, :, bound].tolist()+++def test_same_size_guide_spans_one_patch_per_token():+ """A 1:1 guide gets no offset: each token's end is one scale factor past its start.++ The inverse of the case below, so a factor derived wrongly at 1:1 is caught too.+ """+ keyframe_idxs, entries, _ = _add_latent_guide(guide_hw=(4, 4))++ for axis in (HEIGHT, WIDTH):+ starts = _axis(keyframe_idxs, axis, START)+ assert _axis(keyframe_idxs, axis, END) == [s + SCALE_FACTORS[axis] for s in starts]++ assert entries[0]["latent_shape"] == [1, 4, 4]+++def test_half_size_guide_expands_only_the_end_positions():+ """An x2 guide keeps its start positions and pushes each end out by one scale factor.++ That is what makes the dilated reference cover the whole canvas. Leaving the factor+ at 1 keeps same-size coordinates while each token encodes a larger patch, so the+ reference addresses only the top-left corner of the target.+ """+ same_size, _, _ = _add_latent_guide(guide_hw=(4, 4))+ downscaled, entries, _ = _add_latent_guide(guide_hw=(2, 2))++ # Dilation puts the small guide on the same sparse grid, so token count is unchanged.+ assert downscaled.shape == same_size.shape++ for axis in (HEIGHT, WIDTH):+ assert _axis(downscaled, axis, START) == _axis(same_size, axis, START)+ expected = [e + SCALE_FACTORS[axis] for e in _axis(same_size, axis, END)]+ assert _axis(downscaled, axis, END) == expected++ # Time is never touched by the spatial offset.+ assert _axis(downscaled, TIME, START) == _axis(same_size, TIME, START)+ assert _axis(downscaled, TIME, END) == _axis(same_size, TIME, END)++ assert entries[0]["latent_shape"] == [1, 2, 2]+++def test_attention_entry_lets_context_windows_rederive_the_factor():+ """The entry keeps the pre-dilation shape while the token count is post-dilation.++ ``context_windows`` divides the post-dilation guide height by the entry's+ ``latent_shape`` height to recover the downscale factor, so these two must not drift+ apart or windowed and non-windowed sampling disagree on the guide's RoPE.+ """+ _, entries, latent = _add_latent_guide(guide_hw=(2, 2))++ entry = entries[0]+ assert entry["latent_shape"] == [1, 2, 2] # pre-dilation+ assert entry["pre_filter_count"] == 1 * 4 * 4 # post-dilation+ assert latent["samples"].shape[3] // entry["latent_shape"][1] == 2+++ "latent_idx, expected_start", [(-1, -8), (0, 0), (1, 1), (2, 9)]+)+def test_latent_idx_maps_onto_pixel_frames(latent_idx, expected_start):+ """latent_idx is in latent frames, and negatives sit before the start of the latent.++ The first latent frame covers a single pixel frame, so the mapping is 0 -> 0, 1 -> 1,+ then 8 apart. Negative values are not counted back from the end.+ """+ keyframe_idxs, _, _ = _add_latent_guide(+ guide_hw=(4, 4), latent_frames=8, latent_idx=latent_idx+ )++ assert set(_axis(keyframe_idxs, TIME, START)) == {expected_start}+++ "kwargs, message",+ [+ (dict(guide_hw=(2, 4)), "square"),+ (dict(guide_hw=(3, 3)), "whole number"),+ (dict(guide_hw=(4, 4), latent_idx=99), "runs past the end"),+ (dict(guide_hw=(4, 4), guide_frames=5), "runs past the end"),+ ],+)+def test_unusable_guides_are_rejected(kwargs, message):+ with pytest.raises(ValueError, match=message):+ _add_latent_guide(**kwargs)+++def test_non_5d_guiding_latent_is_rejected():+ """An image-model latent would otherwise fail with a bare IndexError on shape[4]."""+ with pytest.raises(ValueError, match="5D video latent"):+ nodes_lt.LTXVAddLatentGuide.execute(+ _cond(),+ _cond(),+ _vae(),+ _latent(3, 4, 4),+ {"samples": torch.zeros((1, LATENT_CHANNELS, 4, 4))},+ 0,+ 1.0,+ )+++def test_attention_mask_reaches_the_guide_entry():+ """Only coverage that the optional input is forwarded to the guide entry."""+ mask = torch.full((1, 128, 128), 0.5)+ positive, negative, _ = nodes_lt.LTXVAddLatentGuide.execute(+ _cond(), _cond(), _vae(), _latent(3, 4, 4), _latent(1, 2, 2), 0, 1.0, attention_mask=mask+ )++ # Stored as (1, 1, F, H, W) for downstream self-attention masking.+ assert positive[0][1]["guide_attention_entries"][0]["pixel_mask"].shape == (1, 1, 1, 128, 128)+ # Positive and negative each get their own entry, so neither leaks into the other.+ assert len(negative[0][1]["guide_attention_entries"]) == 1+++def test_node_is_registered_with_a_loadable_schema():+ """Both failure modes here are invisible to every other test in this file.++ A bad io.Schema keyword only surfaces when the node is registered, and a node left+ out of the extension list simply does not exist in ComfyUI. The strength cap is+ asserted here because raising it re-exposes the missing clamp in append_keyframe's+ guide_mask branch, where a dilated guide is dropped above 1.0.+ """+ schema = nodes_lt.LTXVAddLatentGuide.define_schema()+ inputs = {inp.id: inp for inp in schema.inputs}++ assert schema.node_id == "LTXVAddLatentGuide"+ assert inputs["strength"].max == 1.0+ assert inputs["attention_mask"].optional is True++ node_list = asyncio.run(nodes_lt.LtxvExtension().get_node_list())+ assert nodes_lt.LTXVAddLatentGuide in node_list+++ "iclora_parameters, expected_shape, expected_extra_end",+ [+ (None, [1, 4, 4], 0),+ ({"reference_downscale_factor": 2}, [1, 2, 2], SCALE_FACTORS[HEIGHT]),+ ],+)+def test_add_guide_image_path_still_routes_through_the_shared_helper(+ iclora_parameters, expected_shape, expected_extra_end+):+ """LTXVAddGuide must be unchanged by sharing attach_guide_latent with the latent node."""++ class _Vae:+ downscale_index_formula = SCALE_FACTORS++ def encode(self, pixels):+ frames, height, width, _ = pixels.shape+ return torch.zeros(+ (1, LATENT_CHANNELS, (frames - 1) // SCALE_FACTORS[TIME] + 1, height // 32, width // 32)+ )++ positive, _, _ = nodes_lt.LTXVAddGuide.execute(+ _cond(),+ _cond(),+ _Vae(),+ _latent(3, 4, 4),+ torch.zeros((1, 4 * 32, 4 * 32, 3)),+ 0,+ 1.0,+ iclora_parameters=iclora_parameters,+ )++ metadata = positive[0][1]+ entry = metadata["guide_attention_entries"][0]+ assert entry["latent_shape"] == expected_shape+ assert entry["pre_filter_count"] == 1 * 4 * 4++ keyframe_idxs = metadata["keyframe_idxs"]+ starts = _axis(keyframe_idxs, HEIGHT, START)+ ends = _axis(keyframe_idxs, HEIGHT, END)+ assert ends == [s + SCALE_FACTORS[HEIGHT] + expected_extra_end for s in starts]