The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Cuda graphs + memory compiler on ace step 1.5 autoregressive model. (#16461)
comfyanonymousd15842095092
diff --git a/comfy/text_encoders/ace15.py b/comfy/text_encoders/ace15.pyindex ec8fa313e..52d0a6f01 100644--- a/comfy/text_encoders/ace15.py+++ b/comfy/text_encoders/ace15.py@@ -4,6 +4,7 @@ from comfy import sd1_clipimport torchimport mathimport yaml+import comfy.model_prefetchimport comfy.opsimport comfy.utils@@ -27,6 +28,61 @@ def _audio_logits(model, x, audio_start, audio_end, eos_token=None):return logits, eos_logits+def _sample_audio_token(model, x, cfg_scale, temperature, top_p, top_k, min_p, generator, audio_start_id, audio_end_id, eos_token_id):+ use_eos_score = eos_token_id is not None+ audio_logits, eos_logits = _audio_logits(model, x, audio_start_id, audio_end_id, eos_token_id)+ if cfg_scale != 1.0:+ cfg_logits = audio_logits[1:2] + cfg_scale * (audio_logits[0:1] - audio_logits[1:2])+ if use_eos_score:+ cond_eos = eos_logits[0:1, 0]+ uncond_eos = eos_logits[1:2, 0]+ eos_score = uncond_eos + cfg_scale * (cond_eos - uncond_eos)+ else:+ cfg_logits = audio_logits[0:1]+ if use_eos_score:+ eos_score = eos_logits[0:1, 0]++ remove_logit_value = torch.finfo(cfg_logits.dtype).min+ if use_eos_score:+ cfg_logits = torch.cat((eos_score.unsqueeze(1), cfg_logits), dim=1)++ if top_k is not None and top_k > 0:+ top_k_values = torch.topk(cfg_logits, min(top_k, cfg_logits.shape[-1])).values+ cfg_logits[cfg_logits < top_k_values[..., -1, None]] = remove_logit_value++ if min_p is not None and min_p > 0:+ probs = torch.softmax(cfg_logits, dim=-1)+ p_max = probs.max(dim=-1, keepdim=True).values+ indices_to_remove = probs < (min_p * p_max)+ cfg_logits[indices_to_remove] = remove_logit_value++ if top_p is not None and top_p < 1.0:+ sorted_logits, sorted_indices = torch.sort(cfg_logits, descending=True)+ cumulative_probs = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)+ sorted_indices_to_remove = cumulative_probs > top_p+ sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()+ sorted_indices_to_remove[..., 0] = 0+ indices_to_remove = torch.zeros_like(cfg_logits, dtype=torch.bool)+ indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove)+ cfg_logits[indices_to_remove] = remove_logit_value++ if temperature > 0:+ cfg_logits = cfg_logits / temperature+ sampling_logits = cfg_logits.new_full((cfg_logits.shape[0], model.vocab_size), remove_logit_value)+ if use_eos_score:+ sampling_logits[:, eos_token_id] = cfg_logits[:, 0]+ cfg_logits = cfg_logits[:, 1:]+ sampling_logits[:, audio_start_id:audio_end_id] = cfg_logits+ next_token = torch.multinomial(torch.softmax(sampling_logits, dim=-1), num_samples=1, generator=generator).squeeze(1)+ else:+ next_token = torch.argmax(cfg_logits, dim=-1)+ if use_eos_score:+ next_token = torch.where(next_token == 0, eos_token_id, next_token + audio_start_id - 1)+ else:+ next_token += audio_start_id+ return next_token++def sample_manual_loop_no_classes(model,ids=None,@@ -64,78 +120,39 @@ def sample_manual_loop_no_classes(past_key_values = model.transformer.model.init_kv_cache(embeds_batch, embeds.shape[1] + max_new_tokens, device, execution_dtype)fixed_kv = isinstance(past_key_values[0], comfy.text_encoders.llama.FixedKV)- progress_bar = comfy.utils.ProgressBar(max_new_tokens)- sampling_logits = None-- for step in comfy.utils.model_trange(max_new_tokens, desc="LM sampling"):- outputs = model.transformer(None, attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=None, dtype=execution_dtype, embeds_info=embeds_info, past_key_values=past_key_values)- past_key_values = outputs[2]-- use_eos_score = eos_token_id is not None and eos_token_id < audio_start_id and min_tokens < step- audio_logits, eos_logits = _audio_logits(model.transformer.model, outputs[0], audio_start_id, audio_end_id, eos_token_id if use_eos_score else None)- if cfg_scale != 1.0:- cfg_logits = audio_logits[1:2] + cfg_scale * (audio_logits[0:1] - audio_logits[1:2])- if use_eos_score:- cond_eos = eos_logits[0:1, 0]- uncond_eos = eos_logits[1:2, 0]- eos_score = uncond_eos + cfg_scale * (cond_eos - uncond_eos)- else:- cfg_logits = audio_logits[0:1]- if use_eos_score:- eos_score = eos_logits[0:1, 0]+ decode_tokens = torch.empty((embeds_batch, 1), device=device, dtype=torch.long)+ decode_buffers = None+ if model.transformer.model.graph_dynamic_vbar_blocks and comfy.model_prefetch.malloc_graph_enabled(device) and not comfy.model_management.args.disable_cuda_graphs:+ decode_buffers = model.transformer.model.init_decode_buffers(embeds_batch, device, execution_dtype)- remove_logit_value = torch.finfo(cfg_logits.dtype).min- if use_eos_score:- cfg_logits = torch.cat((eos_score.unsqueeze(1), cfg_logits), dim=1)-- if top_k is not None and top_k > 0:- top_k_values = torch.topk(cfg_logits, min(top_k, cfg_logits.shape[-1])).values- cfg_logits[cfg_logits < top_k_values[..., -1, None]] = remove_logit_value-- if min_p is not None and min_p > 0:- probs = torch.softmax(cfg_logits, dim=-1)- p_max = probs.max(dim=-1, keepdim=True).values- indices_to_remove = probs < (min_p * p_max)- cfg_logits[indices_to_remove] = remove_logit_value-- if top_p is not None and top_p < 1.0:- sorted_logits, sorted_indices = torch.sort(cfg_logits, descending=True)- cumulative_probs = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)- sorted_indices_to_remove = cumulative_probs > top_p- sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()- sorted_indices_to_remove[..., 0] = 0- indices_to_remove = torch.zeros_like(cfg_logits, dtype=torch.bool)- indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove)- cfg_logits[indices_to_remove] = remove_logit_value-- if temperature > 0:- cfg_logits = cfg_logits / temperature- if sampling_logits is None:- sampling_logits = cfg_logits.new_empty((cfg_logits.shape[0], model.transformer.model.vocab_size))- sampling_logits.fill_(remove_logit_value)- if use_eos_score:- sampling_logits[:, eos_token_id] = cfg_logits[:, 0]- cfg_logits = cfg_logits[:, 1:]- sampling_logits[:, audio_start_id:audio_end_id] = cfg_logits- next_token = torch.multinomial(torch.softmax(sampling_logits, dim=-1), num_samples=1, generator=generator).squeeze(1)- else:- next_token = torch.argmax(cfg_logits, dim=-1)- if use_eos_score:- next_token = torch.where(next_token == 0, eos_token_id, next_token + audio_start_id - 1)- else:- next_token += audio_start_id-- if eos_token_id is not None and next_token.item() == eos_token_id:- break-- input_ids = next_token.repeat(embeds_batch).unsqueeze(1)- embeds = model.transformer.get_input_embeddings()(input_ids, out_dtype=execution_dtype)- if not fixed_kv:- attention_mask = torch.cat([attention_mask, torch.ones((embeds_batch, 1), device=device, dtype=attention_mask.dtype)], dim=1)-- output_audio_codes[generated_tokens] = next_token[0] - audio_start_id- generated_tokens += 1- progress_bar.update_absolute(step)+ progress_bar = comfy.utils.ProgressBar(max_new_tokens)+ try:+ for step in comfy.utils.model_trange(max_new_tokens, desc="LM sampling"):+ comfy.model_management.throw_exception_if_processing_interrupted()+ if step > 0:+ comfy.model_prefetch.malloc_graph_begin(device)+ embeds = model.transformer.get_input_embeddings()(decode_tokens, out_dtype=execution_dtype)++ outputs = model.transformer(None, attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=None, dtype=execution_dtype, embeds_info=embeds_info, past_key_values=past_key_values, decode_buffers=decode_buffers)+ past_key_values = outputs[2]+ use_eos_score = eos_token_id is not None and eos_token_id < audio_start_id and min_tokens < step+ next_token = _sample_audio_token(model.transformer.model, outputs[0], cfg_scale, temperature, top_p, top_k, min_p, generator, audio_start_id, audio_end_id, eos_token_id if use_eos_score else None)+ decode_tokens.copy_(next_token.unsqueeze(1))+ output_audio_codes[step] = next_token[0] - audio_start_id+ del outputs, next_token, embeds+ if step > 0:+ comfy.model_prefetch.malloc_graph_end()++ if eos_token_id is not None and decode_tokens[0, 0].item() == eos_token_id:+ break++ if not fixed_kv:+ attention_mask = torch.cat([attention_mask, torch.ones((embeds_batch, 1), device=device, dtype=attention_mask.dtype)], dim=1)++ generated_tokens += 1+ progress_bar.update_absolute(step)+ finally:+ comfy.model_prefetch.cleanup_prefetch_queues()return output_audio_codes[:generated_tokens].tolist()