Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Cuda graphs + memory compiler on ace step 1.5 autoregressive model. (#16461)

comfyanonymousd15842095092
diff --git a/comfy/text_encoders/ace15.py b/comfy/text_encoders/ace15.py
index ec8fa313e..52d0a6f01 100644
--- a/comfy/text_encoders/ace15.py
+++ b/comfy/text_encoders/ace15.py
@@ -4,6 +4,7 @@ from comfy import sd1_clip
import torch
import math
import yaml
+import comfy.model_prefetch
import comfy.ops
import comfy.utils
@@ -27,6 +28,61 @@ def _audio_logits(model, x, audio_start, audio_end, eos_token=None):
return logits, eos_logits
+def _sample_audio_token(model, x, cfg_scale, temperature, top_p, top_k, min_p, generator, audio_start_id, audio_end_id, eos_token_id):
+ use_eos_score = eos_token_id is not None
+ audio_logits, eos_logits = _audio_logits(model, x, audio_start_id, audio_end_id, eos_token_id)
+ if cfg_scale != 1.0:
+ cfg_logits = audio_logits[1:2] + cfg_scale * (audio_logits[0:1] - audio_logits[1:2])
+ if use_eos_score:
+ cond_eos = eos_logits[0:1, 0]
+ uncond_eos = eos_logits[1:2, 0]
+ eos_score = uncond_eos + cfg_scale * (cond_eos - uncond_eos)
+ else:
+ cfg_logits = audio_logits[0:1]
+ if use_eos_score:
+ eos_score = eos_logits[0:1, 0]
+
+ remove_logit_value = torch.finfo(cfg_logits.dtype).min
+ if use_eos_score:
+ cfg_logits = torch.cat((eos_score.unsqueeze(1), cfg_logits), dim=1)
+
+ if top_k is not None and top_k > 0:
+ top_k_values = torch.topk(cfg_logits, min(top_k, cfg_logits.shape[-1])).values
+ cfg_logits[cfg_logits < top_k_values[..., -1, None]] = remove_logit_value
+
+ if min_p is not None and min_p > 0:
+ probs = torch.softmax(cfg_logits, dim=-1)
+ p_max = probs.max(dim=-1, keepdim=True).values
+ indices_to_remove = probs < (min_p * p_max)
+ cfg_logits[indices_to_remove] = remove_logit_value
+
+ if top_p is not None and top_p < 1.0:
+ sorted_logits, sorted_indices = torch.sort(cfg_logits, descending=True)
+ cumulative_probs = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)
+ sorted_indices_to_remove = cumulative_probs > top_p
+ sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
+ sorted_indices_to_remove[..., 0] = 0
+ indices_to_remove = torch.zeros_like(cfg_logits, dtype=torch.bool)
+ indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove)
+ cfg_logits[indices_to_remove] = remove_logit_value
+
+ if temperature > 0:
+ cfg_logits = cfg_logits / temperature
+ sampling_logits = cfg_logits.new_full((cfg_logits.shape[0], model.vocab_size), remove_logit_value)
+ if use_eos_score:
+ sampling_logits[:, eos_token_id] = cfg_logits[:, 0]
+ cfg_logits = cfg_logits[:, 1:]
+ sampling_logits[:, audio_start_id:audio_end_id] = cfg_logits
+ next_token = torch.multinomial(torch.softmax(sampling_logits, dim=-1), num_samples=1, generator=generator).squeeze(1)
+ else:
+ next_token = torch.argmax(cfg_logits, dim=-1)
+ if use_eos_score:
+ next_token = torch.where(next_token == 0, eos_token_id, next_token + audio_start_id - 1)
+ else:
+ next_token += audio_start_id
+ return next_token
+
+
def sample_manual_loop_no_classes(
model,
ids=None,
@@ -64,78 +120,39 @@ def sample_manual_loop_no_classes(
past_key_values = model.transformer.model.init_kv_cache(embeds_batch, embeds.shape[1] + max_new_tokens, device, execution_dtype)
fixed_kv = isinstance(past_key_values[0], comfy.text_encoders.llama.FixedKV)
- progress_bar = comfy.utils.ProgressBar(max_new_tokens)
- sampling_logits = None
-
- for step in comfy.utils.model_trange(max_new_tokens, desc="LM sampling"):
- outputs = model.transformer(None, attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=None, dtype=execution_dtype, embeds_info=embeds_info, past_key_values=past_key_values)
- past_key_values = outputs[2]
-
- use_eos_score = eos_token_id is not None and eos_token_id < audio_start_id and min_tokens < step
- audio_logits, eos_logits = _audio_logits(model.transformer.model, outputs[0], audio_start_id, audio_end_id, eos_token_id if use_eos_score else None)
- if cfg_scale != 1.0:
- cfg_logits = audio_logits[1:2] + cfg_scale * (audio_logits[0:1] - audio_logits[1:2])
- if use_eos_score:
- cond_eos = eos_logits[0:1, 0]
- uncond_eos = eos_logits[1:2, 0]
- eos_score = uncond_eos + cfg_scale * (cond_eos - uncond_eos)
- else:
- cfg_logits = audio_logits[0:1]
- if use_eos_score:
- eos_score = eos_logits[0:1, 0]
+ decode_tokens = torch.empty((embeds_batch, 1), device=device, dtype=torch.long)
+ decode_buffers = None
+ if model.transformer.model.graph_dynamic_vbar_blocks and comfy.model_prefetch.malloc_graph_enabled(device) and not comfy.model_management.args.disable_cuda_graphs:
+ decode_buffers = model.transformer.model.init_decode_buffers(embeds_batch, device, execution_dtype)
- remove_logit_value = torch.finfo(cfg_logits.dtype).min
- if use_eos_score:
- cfg_logits = torch.cat((eos_score.unsqueeze(1), cfg_logits), dim=1)
-
- if top_k is not None and top_k > 0:
- top_k_values = torch.topk(cfg_logits, min(top_k, cfg_logits.shape[-1])).values
- cfg_logits[cfg_logits < top_k_values[..., -1, None]] = remove_logit_value
-
- if min_p is not None and min_p > 0:
- probs = torch.softmax(cfg_logits, dim=-1)
- p_max = probs.max(dim=-1, keepdim=True).values
- indices_to_remove = probs < (min_p * p_max)
- cfg_logits[indices_to_remove] = remove_logit_value
-
- if top_p is not None and top_p < 1.0:
- sorted_logits, sorted_indices = torch.sort(cfg_logits, descending=True)
- cumulative_probs = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)
- sorted_indices_to_remove = cumulative_probs > top_p
- sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
- sorted_indices_to_remove[..., 0] = 0
- indices_to_remove = torch.zeros_like(cfg_logits, dtype=torch.bool)
- indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove)
- cfg_logits[indices_to_remove] = remove_logit_value
-
- if temperature > 0:
- cfg_logits = cfg_logits / temperature
- if sampling_logits is None:
- sampling_logits = cfg_logits.new_empty((cfg_logits.shape[0], model.transformer.model.vocab_size))
- sampling_logits.fill_(remove_logit_value)
- if use_eos_score:
- sampling_logits[:, eos_token_id] = cfg_logits[:, 0]
- cfg_logits = cfg_logits[:, 1:]
- sampling_logits[:, audio_start_id:audio_end_id] = cfg_logits
- next_token = torch.multinomial(torch.softmax(sampling_logits, dim=-1), num_samples=1, generator=generator).squeeze(1)
- else:
- next_token = torch.argmax(cfg_logits, dim=-1)
- if use_eos_score:
- next_token = torch.where(next_token == 0, eos_token_id, next_token + audio_start_id - 1)
- else:
- next_token += audio_start_id
-
- if eos_token_id is not None and next_token.item() == eos_token_id:
- break
-
- input_ids = next_token.repeat(embeds_batch).unsqueeze(1)
- embeds = model.transformer.get_input_embeddings()(input_ids, out_dtype=execution_dtype)
- if not fixed_kv:
- attention_mask = torch.cat([attention_mask, torch.ones((embeds_batch, 1), device=device, dtype=attention_mask.dtype)], dim=1)
-
- output_audio_codes[generated_tokens] = next_token[0] - audio_start_id
- generated_tokens += 1
- progress_bar.update_absolute(step)
+ progress_bar = comfy.utils.ProgressBar(max_new_tokens)
+ try:
+ for step in comfy.utils.model_trange(max_new_tokens, desc="LM sampling"):
+ comfy.model_management.throw_exception_if_processing_interrupted()
+ if step > 0:
+ comfy.model_prefetch.malloc_graph_begin(device)
+ embeds = model.transformer.get_input_embeddings()(decode_tokens, out_dtype=execution_dtype)
+
+ outputs = model.transformer(None, attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=None, dtype=execution_dtype, embeds_info=embeds_info, past_key_values=past_key_values, decode_buffers=decode_buffers)
+ past_key_values = outputs[2]
+ use_eos_score = eos_token_id is not None and eos_token_id < audio_start_id and min_tokens < step
+ next_token = _sample_audio_token(model.transformer.model, outputs[0], cfg_scale, temperature, top_p, top_k, min_p, generator, audio_start_id, audio_end_id, eos_token_id if use_eos_score else None)
+ decode_tokens.copy_(next_token.unsqueeze(1))
+ output_audio_codes[step] = next_token[0] - audio_start_id
+ del outputs, next_token, embeds
+ if step > 0:
+ comfy.model_prefetch.malloc_graph_end()
+
+ if eos_token_id is not None and decode_tokens[0, 0].item() == eos_token_id:
+ break
+
+ if not fixed_kv:
+ attention_mask = torch.cat([attention_mask, torch.ones((embeds_batch, 1), device=device, dtype=attention_mask.dtype)], dim=1)
+
+ generated_tokens += 1
+ progress_bar.update_absolute(step)
+ finally:
+ comfy.model_prefetch.cleanup_prefetch_queues()
return output_audio_codes[:generated_tokens].tolist()