The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Small speedup for text model sampling. (#14773)
comfyanonymous000c6b784e6c
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex e9f38a9a2..7403a60b8 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -937,22 +937,41 @@ class BaseGenerate:return torch.argmax(logits, dim=-1, keepdim=True)# Sampling mode- if repetition_penalty != 1.0:- for i in range(logits.shape[0]):- for token_id in set(token_history):- logits[i, token_id] *= repetition_penalty if logits[i, token_id] < 0 else 1/repetition_penalty-- if presence_penalty is not None and presence_penalty != 0.0:- for i in range(logits.shape[0]):- for token_id in set(token_history):- logits[i, token_id] -= presence_penalty+ if len(token_history) > 0 and (repetition_penalty != 1.0 or (presence_penalty is not None and presence_penalty != 0.0)):+ token_ids = torch.tensor(list(set(token_history)), device=logits.device)+ token_logits = logits[:, token_ids]+ if repetition_penalty != 1.0:+ token_logits = torch.where(token_logits < 0, token_logits * repetition_penalty, token_logits / repetition_penalty)+ if presence_penalty is not None and presence_penalty != 0.0:+ token_logits = token_logits - presence_penalty+ logits[:, token_ids] = token_logitsif temperature != 1.0:logits = logits / temperatureif top_k > 0:- indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]- logits[indices_to_remove] = torch.finfo(logits.dtype).min+ top_k = min(top_k, logits.shape[-1])+ logits, top_indices = torch.topk(logits, top_k)++ if min_p > 0.0:+ probs_before_filter = torch.nn.functional.softmax(logits, dim=-1)+ top_probs, _ = probs_before_filter.max(dim=-1, keepdim=True)+ min_threshold = min_p * top_probs+ indices_to_remove = probs_before_filter < min_threshold+ logits[indices_to_remove] = torch.finfo(logits.dtype).min++ if top_p < 1.0:+ sorted_logits, sorted_indices = torch.sort(logits, descending=True)+ cumulative_probs = torch.cumsum(torch.nn.functional.softmax(sorted_logits, dim=-1), dim=-1)+ sorted_indices_to_remove = cumulative_probs > top_p+ sorted_indices_to_remove[..., 0] = False+ indices_to_remove = torch.zeros_like(logits, dtype=torch.bool)+ indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove)+ logits[indices_to_remove] = torch.finfo(logits.dtype).min++ probs = torch.nn.functional.softmax(logits, dim=-1)+ next_token = torch.multinomial(probs, num_samples=1, generator=generator)+ return top_indices.gather(1, next_token)if min_p > 0.0:probs_before_filter = torch.nn.functional.softmax(logits, dim=-1)