The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Fix yue2 AMD issues and add more controls to the generate ABC node. (#16293)
comfyanonymous98c7334e2d0a
diff --git a/comfy/audio_encoders/sheetsage2.py b/comfy/audio_encoders/sheetsage2.pyindex 222135d8a..fa75acf82 100644--- a/comfy/audio_encoders/sheetsage2.py+++ b/comfy/audio_encoders/sheetsage2.py@@ -87,7 +87,7 @@ class Decoder(nn.Module):DecoderLayer(dim, intermediate, heads, device=device, dtype=dtype, operations=operations) for _ in range(layers)])- def forward(self, x, positions, cache):+ def forward(self, x, positions, cache, decode_buffer=None):length = x.shape[1]fixed = isinstance(cache[0][0], FixedKV)index = cache[0][0].index if fixed else cache[0][0][2]@@ -95,9 +95,10 @@ class Decoder(nn.Module):if length > 1:mask = torch.full((length, index + length), -torch.inf, device=x.device, dtype=x.dtype).triu_(index + 1)x = self.layernorm_embedding(x + self.embed_positions(positions, out_dtype=x.dtype))- graph = fixed and length == 1 and index > 0+ graph = fixed and length == 1 and index > 0 and decode_buffer is not Noneif graph:- x = x.clone()+ decode_buffer.copy_(x)+ x = decode_bufferattention = optimized_attention_for_device(x.device, mask=mask is not None or graph, small_input=True)queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), x.device, {"prefetch_dynamic_vbars": True})for i, layer in enumerate(self.layers):@@ -167,9 +168,9 @@ class SheetSage2(nn.Module):cache.append((self_cache, (cross.project(memory, cross.k_proj), cross.project(memory, cross.v_proj))))return cache- def decode(self, ids, positions, cache):+ def decode(self, ids, positions, cache, decode_buffer=None):x = self.token_embedding(ids, out_dtype=self.dtype)- return self.output_projection(self.decoder(x, positions, cache)[:, -1:])+ return self.output_projection(self.decoder(x, positions, cache, decode_buffer=decode_buffer)[:, -1:])def generate_tokens(self, memory, stop_seconds, prefix=None):tokenizer = self.tokenizer@@ -185,6 +186,8 @@ class SheetSage2(nn.Module):ids = torch.empty((1, 1), device=device, dtype=torch.long)positions = torch.full((1, 1), len(tokens) + 2, device=device, dtype=torch.long)fixed = isinstance(cache[0][0], FixedKV)+ # Captured decoder layers must share the same input address on every replay.+ decode_buffer = memory.new_empty((memory.shape[0], 1, memory.shape[-1])) if fixed else Noneprogress = comfy.utils.ProgressBar(self.max_tokens - len(tokens))try:for step in comfy.utils.model_trange(self.max_tokens - len(tokens), desc="SheetSage2 transcription", unit="token"):@@ -206,7 +209,7 @@ class SheetSage2(nn.Module):ids.copy_(next_id)if fixed:comfy.model_prefetch.malloc_graph_begin(device)- logits.copy_(self.decode(ids, positions, cache))+ logits.copy_(self.decode(ids, positions, cache, decode_buffer=decode_buffer))if fixed:comfy.model_prefetch.malloc_graph_end()positions.add_(1)diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex c7147904b..eda2dfb39 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -822,7 +822,7 @@ class Llama2_(nn.Module):device=device)def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True,- dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None):+ dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None, decode_buffers=None):if embeds is not None:x = embedselse:@@ -858,7 +858,15 @@ class Llama2_(nn.Module):enable_graph = self.graph_dynamic_vbar_blocks and fixed_kv_decodeif enable_graph:- x = x.clone()+ if decode_buffers is None:+ x = x.clone()+ else:+ hidden_buffer, rotary_buffers = decode_buffers+ hidden_buffer.copy_(x)+ x = hidden_buffer+ for buffer, value in zip(rotary_buffers, freqs_cis):+ buffer.copy_(value)+ freqs_cis = rotary_buffersintermediate = Noneall_intermediate = Nonediff --git a/comfy/text_encoders/yue2.py b/comfy/text_encoders/yue2.pyindex 2c01e3ee8..85f0e9944 100644--- a/comfy/text_encoders/yue2.py+++ b/comfy/text_encoders/yue2.py@@ -88,6 +88,7 @@ class YuE2Tokenizer:"top_p": kwargs.get("top_p", 0.95),"top_k": kwargs.get("top_k", 100),"repetition_penalty": kwargs.get("repetition_penalty", 1.2),+ "penalty_window": kwargs.get("penalty_window", 100),"cfg_scale": kwargs.get("cfg_scale", 1.01 if cot == "off" else 1.0),}@@ -160,6 +161,11 @@ class YuE2TEModel(torch.nn.Module):fixed_kv = isinstance(cache[0], FixedKV)decode_tokens = torch.empty((len(prefixes), 1), device=device, dtype=torch.long)positions = torch.tensor([[len(p)] for p in prefixes], device=device, dtype=torch.long)+ # Decoder inputs and rotary tensors must keep their addresses across graph replays.+ decode_buffers = None+ if fixed_kv:+ decode_buffers = (torch.empty((len(prefixes), 1, self.config.hidden_size), device=device, dtype=dtype),+ self.model.compute_freqs_cis(positions, device))history = []end = ABC_END if phase == "abc" else MUSIC_ENDprogress = comfy.utils.ProgressBar(max_tokens)@@ -184,7 +190,8 @@ class YuE2TEModel(torch.nn.Module):if fixed_kv:comfy.model_prefetch.malloc_graph_begin(device)output = self.model(decode_tokens, past_key_values=cache, dtype=dtype, position_ids=positions,- attention_mask=mask[:, :prefix_length + step + 1] if mask is not None and not fixed_kv else None)+ attention_mask=mask[:, :prefix_length + step + 1] if mask is not None and not fixed_kv else None,+ decode_buffers=decode_buffers)logits.copy_(self.model.lm_head(output[0][:, -1]))cache = output[2]del output@@ -228,7 +235,7 @@ class YuE2TEModel(torch.nn.Module):ids, _ = self._generate(tokens["prefix"], tokens["seed"] if seed is None else seed, max_length, "abc", dtype,temperature=temperature if do_sample else 0, top_p=top_p, top_k=top_k,- repetition_penalty=repetition_penalty, penalty_window=100, min_tokens=min(32, max_length),+ repetition_penalty=repetition_penalty, penalty_window=tokens.get("penalty_window", 100), min_tokens=min(32, max_length),)return idsdiff --git a/comfy_extras/nodes_yue2.py b/comfy_extras/nodes_yue2.pyindex b32ae7957..5c2606cf6 100644--- a/comfy_extras/nodes_yue2.py+++ b/comfy_extras/nodes_yue2.py@@ -21,14 +21,20 @@ class YuE2GenerateABC(io.ComfyNode):io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff, control_after_generate=True),io.Combo.Input("mode", options=["full", "melody"], tooltip="full: generates melody and chords; melody: generates melody only, recommended for covers."),io.Int.Input("max_abc_tokens", default=8192, min=1, max=20000, advanced=True),+ io.Float.Input("temperature", default=0.7, min=0.0, max=5.0, step=0.05, advanced=True),+ io.Float.Input("top_p", default=0.9, min=0.01, max=1.0, step=0.01, advanced=True),+ io.Int.Input("top_k", default=30, min=1, max=32768, advanced=True),+ io.Float.Input("repetition_penalty", default=1.005, min=0.01, max=10.0, step=0.005, advanced=True),+ io.Int.Input("penalty_window", default=100, min=1, max=20000, advanced=True, tooltip="Number of recent ABC tokens used to penalize repetition."),],outputs=[io.String.Output(display_name="abc")],)@classmethod- def execute(cls, clip, style, lyrics, seed, mode, max_abc_tokens):- tokens = clip.tokenize(style, lyrics=lyrics, cot=mode, seed=seed, max_tokens=max_abc_tokens)- ids = clip.generate(tokens, max_length=max_abc_tokens, temperature=0.7, top_p=0.9, top_k=30, repetition_penalty=1.005, seed=seed)+ def execute(cls, clip, style, lyrics, seed, mode, max_abc_tokens, temperature=0.7, top_p=0.9, top_k=30, repetition_penalty=1.005, penalty_window=100):+ tokens = clip.tokenize(style, lyrics=lyrics, cot=mode, seed=seed, max_tokens=max_abc_tokens, penalty_window=penalty_window)+ ids = clip.generate(tokens, max_length=max_abc_tokens, temperature=temperature, top_p=top_p, top_k=top_k,+ repetition_penalty=repetition_penalty, seed=seed)return io.NodeOutput(clip.decode(ids))