Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Fix yue2 AMD issues and add more controls to the generate ABC node. (#16293)

comfyanonymous98c7334e2d0a
diff --git a/comfy/audio_encoders/sheetsage2.py b/comfy/audio_encoders/sheetsage2.py
index 222135d8a..fa75acf82 100644
--- a/comfy/audio_encoders/sheetsage2.py
+++ b/comfy/audio_encoders/sheetsage2.py
@@ -87,7 +87,7 @@ class Decoder(nn.Module):
DecoderLayer(dim, intermediate, heads, device=device, dtype=dtype, operations=operations) for _ in range(layers)
])
- def forward(self, x, positions, cache):
+ def forward(self, x, positions, cache, decode_buffer=None):
length = x.shape[1]
fixed = isinstance(cache[0][0], FixedKV)
index = cache[0][0].index if fixed else cache[0][0][2]
@@ -95,9 +95,10 @@ class Decoder(nn.Module):
if length > 1:
mask = torch.full((length, index + length), -torch.inf, device=x.device, dtype=x.dtype).triu_(index + 1)
x = self.layernorm_embedding(x + self.embed_positions(positions, out_dtype=x.dtype))
- graph = fixed and length == 1 and index > 0
+ graph = fixed and length == 1 and index > 0 and decode_buffer is not None
if graph:
- x = x.clone()
+ decode_buffer.copy_(x)
+ x = decode_buffer
attention = optimized_attention_for_device(x.device, mask=mask is not None or graph, small_input=True)
queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), x.device, {"prefetch_dynamic_vbars": True})
for i, layer in enumerate(self.layers):
@@ -167,9 +168,9 @@ class SheetSage2(nn.Module):
cache.append((self_cache, (cross.project(memory, cross.k_proj), cross.project(memory, cross.v_proj))))
return cache
- def decode(self, ids, positions, cache):
+ def decode(self, ids, positions, cache, decode_buffer=None):
x = self.token_embedding(ids, out_dtype=self.dtype)
- return self.output_projection(self.decoder(x, positions, cache)[:, -1:])
+ return self.output_projection(self.decoder(x, positions, cache, decode_buffer=decode_buffer)[:, -1:])
def generate_tokens(self, memory, stop_seconds, prefix=None):
tokenizer = self.tokenizer
@@ -185,6 +186,8 @@ class SheetSage2(nn.Module):
ids = torch.empty((1, 1), device=device, dtype=torch.long)
positions = torch.full((1, 1), len(tokens) + 2, device=device, dtype=torch.long)
fixed = isinstance(cache[0][0], FixedKV)
+ # Captured decoder layers must share the same input address on every replay.
+ decode_buffer = memory.new_empty((memory.shape[0], 1, memory.shape[-1])) if fixed else None
progress = comfy.utils.ProgressBar(self.max_tokens - len(tokens))
try:
for step in comfy.utils.model_trange(self.max_tokens - len(tokens), desc="SheetSage2 transcription", unit="token"):
@@ -206,7 +209,7 @@ class SheetSage2(nn.Module):
ids.copy_(next_id)
if fixed:
comfy.model_prefetch.malloc_graph_begin(device)
- logits.copy_(self.decode(ids, positions, cache))
+ logits.copy_(self.decode(ids, positions, cache, decode_buffer=decode_buffer))
if fixed:
comfy.model_prefetch.malloc_graph_end()
positions.add_(1)
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index c7147904b..eda2dfb39 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -822,7 +822,7 @@ class Llama2_(nn.Module):
device=device)
def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True,
- dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None):
+ dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None, decode_buffers=None):
if embeds is not None:
x = embeds
else:
@@ -858,7 +858,15 @@ class Llama2_(nn.Module):
enable_graph = self.graph_dynamic_vbar_blocks and fixed_kv_decode
if enable_graph:
- x = x.clone()
+ if decode_buffers is None:
+ x = x.clone()
+ else:
+ hidden_buffer, rotary_buffers = decode_buffers
+ hidden_buffer.copy_(x)
+ x = hidden_buffer
+ for buffer, value in zip(rotary_buffers, freqs_cis):
+ buffer.copy_(value)
+ freqs_cis = rotary_buffers
intermediate = None
all_intermediate = None
diff --git a/comfy/text_encoders/yue2.py b/comfy/text_encoders/yue2.py
index 2c01e3ee8..85f0e9944 100644
--- a/comfy/text_encoders/yue2.py
+++ b/comfy/text_encoders/yue2.py
@@ -88,6 +88,7 @@ class YuE2Tokenizer:
"top_p": kwargs.get("top_p", 0.95),
"top_k": kwargs.get("top_k", 100),
"repetition_penalty": kwargs.get("repetition_penalty", 1.2),
+ "penalty_window": kwargs.get("penalty_window", 100),
"cfg_scale": kwargs.get("cfg_scale", 1.01 if cot == "off" else 1.0),
}
@@ -160,6 +161,11 @@ class YuE2TEModel(torch.nn.Module):
fixed_kv = isinstance(cache[0], FixedKV)
decode_tokens = torch.empty((len(prefixes), 1), device=device, dtype=torch.long)
positions = torch.tensor([[len(p)] for p in prefixes], device=device, dtype=torch.long)
+ # Decoder inputs and rotary tensors must keep their addresses across graph replays.
+ decode_buffers = None
+ if fixed_kv:
+ decode_buffers = (torch.empty((len(prefixes), 1, self.config.hidden_size), device=device, dtype=dtype),
+ self.model.compute_freqs_cis(positions, device))
history = []
end = ABC_END if phase == "abc" else MUSIC_END
progress = comfy.utils.ProgressBar(max_tokens)
@@ -184,7 +190,8 @@ class YuE2TEModel(torch.nn.Module):
if fixed_kv:
comfy.model_prefetch.malloc_graph_begin(device)
output = self.model(decode_tokens, past_key_values=cache, dtype=dtype, position_ids=positions,
- attention_mask=mask[:, :prefix_length + step + 1] if mask is not None and not fixed_kv else None)
+ attention_mask=mask[:, :prefix_length + step + 1] if mask is not None and not fixed_kv else None,
+ decode_buffers=decode_buffers)
logits.copy_(self.model.lm_head(output[0][:, -1]))
cache = output[2]
del output
@@ -228,7 +235,7 @@ class YuE2TEModel(torch.nn.Module):
ids, _ = self._generate(
tokens["prefix"], tokens["seed"] if seed is None else seed, max_length, "abc", dtype,
temperature=temperature if do_sample else 0, top_p=top_p, top_k=top_k,
- repetition_penalty=repetition_penalty, penalty_window=100, min_tokens=min(32, max_length),
+ repetition_penalty=repetition_penalty, penalty_window=tokens.get("penalty_window", 100), min_tokens=min(32, max_length),
)
return ids
diff --git a/comfy_extras/nodes_yue2.py b/comfy_extras/nodes_yue2.py
index b32ae7957..5c2606cf6 100644
--- a/comfy_extras/nodes_yue2.py
+++ b/comfy_extras/nodes_yue2.py
@@ -21,14 +21,20 @@ class YuE2GenerateABC(io.ComfyNode):
io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff, control_after_generate=True),
io.Combo.Input("mode", options=["full", "melody"], tooltip="full: generates melody and chords; melody: generates melody only, recommended for covers."),
io.Int.Input("max_abc_tokens", default=8192, min=1, max=20000, advanced=True),
+ io.Float.Input("temperature", default=0.7, min=0.0, max=5.0, step=0.05, advanced=True),
+ io.Float.Input("top_p", default=0.9, min=0.01, max=1.0, step=0.01, advanced=True),
+ io.Int.Input("top_k", default=30, min=1, max=32768, advanced=True),
+ io.Float.Input("repetition_penalty", default=1.005, min=0.01, max=10.0, step=0.005, advanced=True),
+ io.Int.Input("penalty_window", default=100, min=1, max=20000, advanced=True, tooltip="Number of recent ABC tokens used to penalize repetition."),
],
outputs=[io.String.Output(display_name="abc")],
)
@classmethod
- def execute(cls, clip, style, lyrics, seed, mode, max_abc_tokens):
- tokens = clip.tokenize(style, lyrics=lyrics, cot=mode, seed=seed, max_tokens=max_abc_tokens)
- ids = clip.generate(tokens, max_length=max_abc_tokens, temperature=0.7, top_p=0.9, top_k=30, repetition_penalty=1.005, seed=seed)
+ def execute(cls, clip, style, lyrics, seed, mode, max_abc_tokens, temperature=0.7, top_p=0.9, top_k=30, repetition_penalty=1.005, penalty_window=100):
+ tokens = clip.tokenize(style, lyrics=lyrics, cot=mode, seed=seed, max_tokens=max_abc_tokens, penalty_window=penalty_window)
+ ids = clip.generate(tokens, max_length=max_abc_tokens, temperature=temperature, top_p=top_p, top_k=top_k,
+ repetition_penalty=repetition_penalty, seed=seed)
return io.NodeOutput(clip.decode(ids))