Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Speedup Yue2 AR (#16626)

rattus7fbcfa8be9a8
diff --git a/comfy/text_encoders/yue2.py b/comfy/text_encoders/yue2.py
index 4df5d8c28..de6759672 100644
--- a/comfy/text_encoders/yue2.py
+++ b/comfy/text_encoders/yue2.py
@@ -26,7 +26,7 @@ INSTRUCTIONS = {
}
-def distribution(logits, history, step, phase, temperature, top_p, top_k, repetition_penalty, penalty_window, min_tokens, legacy_off=False):
+def distribution(logits, recent, step, phase, temperature, top_p, top_k, repetition_penalty, min_tokens, legacy_off=False):
scores = logits.clone() if legacy_off else logits.float().clone()
end = ABC_END if phase == "abc" else MUSIC_END
allowed = torch.full_like(scores, -torch.inf)
@@ -38,8 +38,7 @@ def distribution(logits, history, step, phase, temperature, top_p, top_k, repeti
scores += allowed
if step < min_tokens:
scores[..., end] = -torch.inf
- if repetition_penalty != 1.0 and history:
- recent = torch.tensor([history[-penalty_window:]], dtype=torch.long, device=scores.device)
+ if repetition_penalty != 1.0:
counts = torch.zeros_like(scores)
counts.scatter_add_(-1, recent, torch.ones_like(recent, dtype=scores.dtype))
penalty = repetition_penalty ** counts
@@ -114,7 +113,9 @@ class YuE2TEModel(torch.nn.Module):
self.execution_device = device
def get_dynamic_vram__units(self):
- return self.model.get_dynamic_vram__units()
+ units, last_units = self.model.get_dynamic_vram__units()
+ # the sampling step is graphed as the embedding's unit
+ return [self.model.embed_tokens, *units], last_units
def set_clip_options(self, options):
self.execution_device = options.get("execution_device", self.execution_device)
@@ -159,45 +160,59 @@ class YuE2TEModel(torch.nn.Module):
prefix_length = max(map(len, prefixes))
logits, cache, mask = self._prefill(prefixes, prefix_length + max_tokens, dtype)
fixed_kv = isinstance(cache[0], FixedKV)
- decode_tokens = torch.empty((len(prefixes), 1), device=device, dtype=torch.long)
positions = torch.tensor([[len(p)] for p in prefixes], device=device, dtype=torch.long)
# Decoder inputs and rotary tensors must keep their addresses across graph replays.
- decode_buffers = None
- if fixed_kv:
- decode_buffers = (torch.empty((len(prefixes), 1, self.config.hidden_size), device=device, dtype=dtype),
- rope_matrix(self.model.compute_freqs_cis(positions, device)))
+ decode_buffers = (torch.empty((len(prefixes), 1, self.config.hidden_size), device=device, dtype=dtype),
+ rope_matrix(self.model.compute_freqs_cis(positions, device)))
history = []
end = ABC_END if phase == "abc" else MUSIC_END
progress = comfy.utils.ProgressBar(max_tokens)
+ # Sampling stays on the device (EOD is masked in both phases, so it pads the penalty window)
+ # and the host reads tokens back in batches instead of syncing on every step.
+ penalty_window = sampling.pop("penalty_window")
+ recent = torch.full((1, penalty_window), EOD, device=device, dtype=torch.long)
+ tokens = torch.empty((max_tokens,), device=device, dtype=torch.long)
+ step_index = torch.zeros((), device=device, dtype=torch.long)
+
+ def head():
+ guided = logits if cfg_scale == 1.0 else logits[1:] + cfg_scale * (logits[:1] - logits[1:])
+ scores = distribution(guided, recent, step, phase, legacy_off=legacy_off, **sampling)
+ if sampling["temperature"] == 0:
+ next_id = scores.argmax(-1, keepdim=True)
+ else:
+ probabilities = scores.softmax(-1).to(rng_device)
+ next_id = torch.multinomial(probabilities, 1, generator=generator).to(device)
+ tokens.index_copy_(0, step_index.view(1), next_id.view(1))
+ recent.scatter_(1, (step_index % penalty_window).view(1, 1), next_id)
+ step_index.add_(1)
+ decode_buffers[0].copy_(self.model.embed_tokens(next_id.expand(len(prefixes), 1), out_dtype=dtype))
+
try:
for step in comfy.utils.model_trange(max_tokens, desc="YuE2 ABC sampling" if phase == "abc" else "YuE2 music sampling", unit="token"):
comfy.model_management.throw_exception_if_processing_interrupted()
- guided = logits if cfg_scale == 1.0 else logits[1:] + cfg_scale * (logits[:1] - logits[1:])
- scores = distribution(guided, history, step, phase, legacy_off=legacy_off, **sampling)
- if sampling["temperature"] == 0:
- next_id = scores.argmax(-1, keepdim=True)
- else:
- probabilities = scores.softmax(-1).to(rng_device)
- next_id = torch.multinomial(probabilities, 1, generator=generator).to(device)
- decode_tokens.copy_(next_id)
- token = next_id.item()
+ # Keep decode allocations stable.
+ if fixed_kv:
+ comfy.model_prefetch.malloc_graph_begin(device)
+ # `step` is baked into the head graph, so capture only once the end token is no longer blocked.
+ head_queue = comfy.model_prefetch.make_prefetch_queue([self.model.embed_tokens], device, {"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars})
+ comfy.model_prefetch.prefetch_queue_pop(head_queue, device, self.model.embed_tokens, dtype, core=head,
+ enable_graph=step >= sampling["min_tokens"], generator=generator, malloc_scope="head")
+ comfy.model_prefetch.prefetch_queue_pop(head_queue, device, None, malloc_scope="head")
progress.update_absolute(step + 1)
- if token == end:
- return history, False
- history.append(token)
- if step + 1 < max_tokens:
- # Keep decode allocations stable; sampling has a changing history window.
- if fixed_kv:
- comfy.model_prefetch.malloc_graph_begin(device)
- output = self.model(decode_tokens, past_key_values=cache, dtype=dtype, position_ids=positions,
- attention_mask=mask[:, :prefix_length + step + 1] if mask is not None and not fixed_kv else None,
- decode_buffers=decode_buffers)
- logits.copy_(self.model.lm_head(output[0][:, -1]))
- cache = output[2]
- del output
- if fixed_kv:
- comfy.model_prefetch.malloc_graph_end()
- positions.add_(1)
+ output = self.model(None, embeds=decode_buffers[0], past_key_values=cache, dtype=dtype, position_ids=positions,
+ attention_mask=mask[:, :prefix_length + step + 1] if mask is not None and not fixed_kv else None,
+ decode_buffers=decode_buffers)
+ logits.copy_(self.model.lm_head(output[0][:, -1]))
+ cache = output[2]
+ del output
+ if fixed_kv:
+ comfy.model_prefetch.malloc_graph_end()
+ positions.add_(1)
+ if (step + 1) % 16 == 0 or step + 1 == max_tokens:
+ for token in tokens[len(history):step + 1].tolist():
+ if token == end:
+ return history, False
+ history.append(token)
finally:
# Each phase has different KV buffers and may change the CFG batch size.
comfy.model_prefetch.cleanup_prefetch_queues()