FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(sampling): apply presence and frequency penalties (#527)
cherry77-cloudc8186396f583
diff --git a/python/freetoken/core.py b/python/freetoken/core.pyindex 45a763c0..812f9a43 100644--- a/python/freetoken/core.py+++ b/python/freetoken/core.py@@ -24,6 +24,8 @@ class SamplingParams:# Stop strings (OpenAI `stop` / Anthropic `stop_sequences`). Generation finishes when one# appears in the decoded output; the matched substring (and anything after) is trimmed.stop_strs: list[str] = field(default_factory=list)+ presence_penalty: float = 0.0+ frequency_penalty: float = 0.0@propertydef is_greedy(self) -> bool:@@ -66,6 +68,7 @@ class Req:# handler must not free resources under an in-flight forward; it sets this flag and# _process_last_data frees the request when the batch drains (after copy_done.synchronize).aborted: bool = False+ output_token_counts: torch.Tensor | None = field(default=None, init=False, repr=False)def __post_init__(self) -> None:assert self.input_ids.is_cpudiff --git a/python/freetoken/engine/sample.py b/python/freetoken/engine/sample.pyindex 3bdae769..c6fb2f40 100644--- a/python/freetoken/engine/sample.py+++ b/python/freetoken/engine/sample.py@@ -1,6 +1,6 @@from __future__ import annotations-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom typing import TYPE_CHECKING, Listimport torch@@ -16,6 +16,7 @@ class BatchSamplingArgs:top_k: torch.Tensor | None = Nonetop_p: torch.Tensor | None = Nonegreedy_mask: torch.Tensor | None = None+ penalties: list[tuple[int, torch.Tensor, float, float]] = field(default_factory=list)def make_device_tensor(data: List, dtype: torch.dtype, device: torch.device) -> torch.Tensor:@@ -59,8 +60,20 @@ class Sampler:def prepare(self, batch: Batch) -> BatchSamplingArgs:params = [r.sampling_params for r in batch.reqs]is_greedy = [p.is_greedy for p in params]+ penalties = []+ for row, req in enumerate(batch.reqs):+ p = req.sampling_params+ if not (p.presence_penalty or p.frequency_penalty) or not req.can_decode:+ continue+ if req.output_token_counts is None:+ req.output_token_counts = torch.zeros(+ self.vocab_size, dtype=torch.int32, device=self.device+ )+ penalties.append(+ (row, req.output_token_counts, p.presence_penalty, p.frequency_penalty)+ )if all(is_greedy):- return BatchSamplingArgs(temperatures=None)+ return BatchSamplingArgs(temperatures=None, penalties=penalties)MIN_P = MIN_T = 1e-6# Greedy outputs are selected explicitly in sample(); use neutral sampling@@ -83,17 +96,30 @@ class Sampler:greedy_mask = (make_device_tensor(is_greedy, torch.bool, self.device) if any(is_greedy) else None)- return BatchSamplingArgs(temperatures, top_k=top_k, top_p=top_p, greedy_mask=greedy_mask)+ return BatchSamplingArgs(+ temperatures, top_k=top_k, top_p=top_p, greedy_mask=greedy_mask, penalties=penalties+ )@nvtx_annotate("Sampler")def sample(self, logits: torch.Tensor, args: BatchSamplingArgs) -> torch.Tensor:with torch.cuda.nvtx.range("Sampler"):+ if args.penalties:+ logits = logits.float().clone()+ for row, counts, presence, frequency in args.penalties:+ logits[row] -= frequency * counts + presence * (counts > 0)if args.temperatures is None: # greedy sampling- return torch.argmax(logits, dim=-1)- tokens = sample_impl(logits.float(), args.temperatures, args.top_k, args.top_p)- if args.greedy_mask is not None:- # Mixed batches still run probability sampling for all rows, but- # greedy rows must follow argmax's deterministic tie-breaking.- greedy_tokens = torch.argmax(logits, dim=-1).to(tokens.dtype)- tokens = torch.where(args.greedy_mask, greedy_tokens, tokens)+ tokens = torch.argmax(logits, dim=-1)+ else:+ tokens = sample_impl(logits.float(), args.temperatures, args.top_k, args.top_p)+ if args.greedy_mask is not None:+ # Mixed batches still run probability sampling for all rows, but+ # greedy rows must follow argmax's deterministic tie-breaking.+ greedy_tokens = torch.argmax(logits, dim=-1).to(tokens.dtype)+ tokens = torch.where(args.greedy_mask, greedy_tokens, tokens)+ # Update on the sampling stream: overlapped scheduling can prepare the next+ # batch before the previous token reaches Req.input_ids on the CPU.+ for row, counts, _, _ in args.penalties:+ counts.scatter_add_(+ 0, tokens[row : row + 1].long(), counts.new_ones(1)+ )return tokensdiff --git a/python/freetoken/server/api_models.py b/python/freetoken/server/api_models.pyindex ffd71728..6c3ade09 100644--- a/python/freetoken/server/api_models.py+++ b/python/freetoken/server/api_models.py@@ -74,8 +74,8 @@ class ChatCompletionRequest(BaseModel):stream: bool = Falsestream_options: StreamOptions | None = Nonestop: str | list[str] | None = None- presence_penalty: float = 0.0- frequency_penalty: float = 0.0+ presence_penalty: float = Field(default=0.0, ge=-2.0, le=2.0, allow_inf_nan=False)+ frequency_penalty: float = Field(default=0.0, ge=-2.0, le=2.0, allow_inf_nan=False)chat_template_kwargs: dict[str, Any] = Field(default_factory=dict)reasoning_effort: str | None = None# DeepSeek-wire thinking toggle ({"type": "enabled"|"disabled"}). Any so a@@ -112,8 +112,8 @@ class CompletionRequest(BaseModel):stream: bool = Falsestream_options: StreamOptions | None = Nonestop: str | list[str] | None = None- presence_penalty: float = 0.0- frequency_penalty: float = 0.0+ presence_penalty: float = Field(default=0.0, ge=-2.0, le=2.0, allow_inf_nan=False)+ frequency_penalty: float = Field(default=0.0, ge=-2.0, le=2.0, allow_inf_nan=False)ignore_eos: bool = Falselogprobs: int | None = Noneecho: bool = Falsediff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.pyindex ac988edd..2e342ff8 100644--- a/python/freetoken/server/generation.py+++ b/python/freetoken/server/generation.py@@ -169,6 +169,8 @@ def resolve_sampling(model_sampling: dict[str, Any],stop: str | list[str] | None = None,default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,+ presence_penalty: float = 0.0,+ frequency_penalty: float = 0.0,) -> SamplingParams:"""Map a protocol's sampling fields onto the engine's neutral SamplingParams,filling unspecified fields from the checkpoint's recommended defaults."""@@ -190,6 +192,8 @@ def resolve_sampling(top_k=pick(top_k, "top_k", -1),top_p=pick(top_p, "top_p", 1.0),stop_strs=[s for s in stop_list if s], # drop empty strings (would match everything)+ presence_penalty=presence_penalty,+ frequency_penalty=frequency_penalty,)diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.pyindex dc2f73a9..99a42835 100644--- a/python/freetoken/server/openai_api.py+++ b/python/freetoken/server/openai_api.py@@ -79,6 +79,8 @@ def chat_request_to_genspec(model_sampling=model_sampling,stop=req.stop,default_max_tokens=default_max_tokens,+ presence_penalty=req.presence_penalty,+ frequency_penalty=req.frequency_penalty,),chat_template_kwargs=ctk,template_tools=_tools_for_template(req),@@ -550,6 +552,8 @@ def _resolve_sampling(model_sampling=model_sampling,stop=req.stop,default_max_tokens=default_max_tokens,+ presence_penalty=req.presence_penalty,+ frequency_penalty=req.frequency_penalty,)