FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
import importlib.util
import json
import os
import threading
from copy import deepcopy
from types import ModuleType
from typing import TYPE_CHECKING, Any, List
import torch
from freetoken.message import TokenizeMsg, UserMsg
from freetoken.utils import init_logger
from transformers import PreTrainedTokenizerBase
if TYPE_CHECKING:
from freetoken.mm.processor import MMProcessor
from .effort import (
EffortProfile,
ThinkingProfile,
probe_effort_profile,
probe_thinking_profile,
quantize_effort,
)
from .inline_system import has_inline_system, normalize_inline_system, probe_inline_system
logger = init_logger(__name__)
def resolve_thinking_mode(chat_template_kwargs: dict[str, Any] | None, tools: Any | None) -> str:
"""Resolve the thinking mode (``"thinking"`` or ``"chat"``) for a chat request.
The single source of truth for this decision: the encode side
(``_apply_dsv4_chat_encoder`` below) uses it to pick the prompt the model
sees, and the frontend parse side (``server/openai_api.py``) imports it to
decide whether the model's output begins inside a reasoning block. Keeping
one implementation prevents the two sides from disagreeing. Thinking is on
when tools are offered (dsv4 only emits well-formed tool calls in thinking
mode) or when the caller requests it via ``chat_template_kwargs``.
"""
ctk = chat_template_kwargs or {}
mode = str(ctk.get("thinking_mode") or "chat")
if tools or ctk.get("enable_thinking") or ctk.get("thinking"):
mode = "thinking"
if mode not in ("chat", "thinking"):
mode = "chat"
return mode
_EFFORT_PROBE_MESSAGES = [{"role": "user", "content": "ping"}]
class TokenizeManager:
def __init__(self, tokenizer: PreTrainedTokenizerBase, mm_processor: MMProcessor | None = None) -> None:
self.tokenizer = tokenizer
self.mm_processor = mm_processor # None: the model takes no images
self._dsv4_encoder = _load_dsv4_encoder_if_needed(tokenizer)
self._effort_profile: EffortProfile | None = None
self._thinking_profile: ThinkingProfile | None = None
self._effort_lock = threading.Lock()
self._logged_effort_maps: set[tuple[Any, str | int | None]] = set()
self._inline_system_lock = threading.Lock()
self._inline_system_modes: dict[str, str] = {}
def tokenize(self, msgs: List[TokenizeMsg]) -> List[UserMsg]:
results: List[UserMsg] = []
# TODO: batch tokenization
for msg in msgs:
prompt = self.render_prompt(msg)
# A jinja chat template owns every special token (HF's apply_chat_template
# tokenizes with add_special_tokens=False for the same reason): tokenizers
# that auto-add bos (muse-glimmer's, llama's) would otherwise double it --
# the template already rendered one. Raw-string prompts and the dsv4
# encoder path keep the default.
templated = isinstance(msg.text, list) and self._dsv4_encoder is None
input_ids: torch.Tensor = ( # type: ignore
self.tokenizer.encode(
prompt, return_tensors="pt", add_special_tokens=not templated
)
)
input_ids = input_ids.view(-1).to(torch.int32)
if msg.images:
if self.mm_processor is None:
raise ValueError("image input is not supported for this model")
mm = self.mm_processor.apply(input_ids, msg.images)
results.append(
UserMsg(
uid=msg.uid,
input_ids=mm.input_ids,
sampling_params=msg.sampling_params,
mm_items=mm.mm_items,
mrope_positions=mm.mrope_positions,
mrope_delta=mm.mrope_delta,
)
)
else:
results.append(
UserMsg(uid=msg.uid, input_ids=input_ids, sampling_params=msg.sampling_params)
)
return results
def render_prompt(self, msg: TokenizeMsg) -> str:
"""The template/encoder half of ``tokenize``, exposed so the frontend can
validate a request before committing an SSE stream. Sanitizes
``reasoning_effort`` first: every render path (worker, frontend
validation, count_tokens) must quantize identically."""
if not isinstance(msg.text, list):
return msg.text
kwargs = self._sanitize_effort(msg.chat_template_kwargs or {})
messages = msg.text
if msg.inline_system_policy is not None:
if msg.inline_system_policy not in ("auto", "preserve", "fold"):
raise ValueError(f"invalid inline system policy: {msg.inline_system_policy}")
if has_inline_system(messages):
mode = msg.inline_system_policy
if mode == "auto":
mode = self.inline_system_mode(msg.tools, kwargs)
messages = normalize_inline_system(messages, mode)
return self._render(messages, msg.tools, kwargs, images=msg.images)
def inline_system_mode(self, tools: list[dict] | None, kwargs: dict[str, Any]) -> str:
key = json.dumps([tools, kwargs], sort_keys=True)
with self._inline_system_lock:
if key not in self._inline_system_modes:
mode, reason = probe_inline_system(
lambda messages: self._render(messages, deepcopy(tools), deepcopy(kwargs)), self.tokenizer
)
if len(self._inline_system_modes) >= 128:
self._inline_system_modes.pop(next(iter(self._inline_system_modes)))
self._inline_system_modes[key] = mode
logger.info("Anthropic inline system policy: %s (%s)", mode, reason)
return self._inline_system_modes[key]
def _render(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]] | None,
chat_template_kwargs: dict[str, Any],
*, images: list[bytes] | None = None,
) -> str:
"""Raw render, no effort sanitation — the probe needs unsupported values
to actually reach the template so rejection is observable."""
if self._dsv4_encoder is not None:
return _apply_dsv4_chat_encoder(
self._dsv4_encoder, messages, tools, chat_template_kwargs, images=images
)
# Broadcast the effort in every spelling the ecosystem's templates read
# (muse-glimmer grades ``reasoning_strength``; Jinja ignores undeclared
# variables) -- the same rule the thinking toggles use. An explicit
# caller-provided spelling wins over the broadcast.
if "reasoning_effort" in chat_template_kwargs:
chat_template_kwargs = dict(chat_template_kwargs)
chat_template_kwargs.setdefault(
"reasoning_strength", chat_template_kwargs["reasoning_effort"]
)
if tools is not None:
chat_template_kwargs = {**chat_template_kwargs, "tools": tools}
prompt = self.tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
**chat_template_kwargs,
)
assert isinstance(prompt, str)
return prompt
def effort_profile(self) -> EffortProfile:
"""The checkpoint's effort vocabulary, probed on first use and cached
for the process lifetime."""
with self._effort_lock:
if self._effort_profile is None:
self._effort_profile = probe_effort_profile(self._probe_render)
logger.info(
"reasoning-effort profile: supported=%s default=%s",
sorted(self._effort_profile.supported) or "(none)",
self._effort_profile.default,
)
return self._effort_profile
def thinking_profile(self) -> ThinkingProfile:
"""The checkpoint's thinking controls (toggle behavior + effort
vocabulary), probed on first use and cached for the process lifetime.
Feeds the /v1/cache/status gear derivation."""
efforts = self.effort_profile()
with self._effort_lock:
if self._thinking_profile is None:
self._thinking_profile = probe_thinking_profile(self._probe_render, efforts)
return self._thinking_profile
def _probe_render(
self, kwargs: dict[str, Any], tools: list[dict[str, Any]] | None
) -> str:
return self._render(_EFFORT_PROBE_MESSAGES, tools, kwargs)
def _sanitize_effort(self, chat_template_kwargs: dict[str, Any]) -> dict[str, Any]:
if "reasoning_effort" not in chat_template_kwargs:
return chat_template_kwargs
raw = chat_template_kwargs.get("reasoning_effort")
mapped = quantize_effort(raw, self.effort_profile())
if mapped == raw:
return chat_template_kwargs
# raw is client-controlled and may be unhashable (a JSON list/dict).
key = (raw if isinstance(raw, str) else repr(raw), mapped)
if key not in self._logged_effort_maps:
self._logged_effort_maps.add(key)
logger.info(
"reasoning_effort %r is not supported by this checkpoint; using %s",
raw,
mapped if mapped is not None else "the template default",
)
sanitized = dict(chat_template_kwargs)
if mapped is None:
del sanitized["reasoning_effort"]
else:
sanitized["reasoning_effort"] = mapped
return sanitized
def _load_dsv4_encoder_if_needed(tokenizer: PreTrainedTokenizerBase) -> ModuleType | None:
if getattr(tokenizer, "chat_template", None):
return None
model_path = getattr(tokenizer, "name_or_path", None) or getattr(tokenizer, "_name_or_path", "")
if not model_path:
return None
# DeepSeek-V4 ships encoding/encoding_dsv4.py, V4.1 encoding/encoding.py; same encode_messages contract
encoder_path = next(
(p for p in (os.path.join(str(model_path), "encoding", name) for name in ("encoding_dsv4.py", "encoding.py")) if os.path.isfile(p)),
None,
)
if encoder_path is None:
return None
spec = importlib.util.spec_from_file_location("encoding_dsv4", encoder_path)
if spec is None or spec.loader is None:
return None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if not hasattr(module, "encode_messages"):
return None
return module
def _apply_dsv4_chat_encoder(
encoder: ModuleType,
messages: list[dict],
tools: list[dict] | None,
chat_template_kwargs: dict,
*, images: list[bytes] | None = None,
) -> str:
rendered_messages = [dict(message) for message in messages]
image_data = iter(images) if images is not None else None
for message in rendered_messages:
if isinstance(message.get("content"), list):
content = []
for part in message["content"]:
if isinstance(part, dict) and part.get("type") == "image":
# Prerender still has refs; the tokenizer worker receives fetched bytes.
# The checkpoint encoder validates sources even when rendering only text.
ref = part.get("freetoken_ref")
if image_data is not None:
try:
part = {"type": "image", "data": next(image_data)}
except StopIteration:
raise ValueError("image blocks outnumber fetched images") from None
elif ref is not None:
key = "data" if ref["kind"] == "b64" else "url"
part = {"type": "image", key: ref["data"]}
content.append(part)
message["content"] = content
if message.get("tool_calls"):
message["tool_calls"] = _dsv4_tool_calls(message["tool_calls"])
if tools:
_attach_tools_to_dsv4_messages(rendered_messages, tools)
# No effort filtering here: the caller sanitized already, and the probe
# needs raw values to reach the encoder's own validation.
return encoder.encode_messages(
rendered_messages,
thinking_mode=resolve_thinking_mode(chat_template_kwargs, tools),
reasoning_effort=chat_template_kwargs.get("reasoning_effort"),
)
def _dsv4_tool_calls(tool_calls: list[dict]) -> list[dict]:
"""The dsv4 encoder's contract is ``function.arguments`` = JSON-object STRING
(it json.loads then iterates .items()); a dict (what ``render_messages``
produces for Jinja templates) trips its bare-except fallback, which wraps the
whole payload in a bogus parameter literally named ``arguments``. Re-serialize
here. Copies each tool-call dict: the outer message copy is shallow, so these
are shared with the caller."""
rendered = []
for tc in tool_calls:
tc = dict(tc)
fn = dict(tc.get("function") or {})
fn["arguments"] = _dsv4_arguments_str(fn.get("arguments"))
tc["function"] = fn
rendered.append(tc)
return rendered
def _dsv4_arguments_str(arguments: Any) -> str:
"""Missing/empty means no arguments (vLLM parity); anything else that is not
a JSON object is rejected -- ValueError becomes a per-request "could not
encode request" error, never a worker crash -- matching sglang's
validate-then-400. A non-object would otherwise raise uncaught in the
encoder's .items() or be wrapped as garbage."""
if arguments is None or (isinstance(arguments, str) and not arguments.strip()):
return "{}"
if isinstance(arguments, dict):
return json.dumps(arguments, ensure_ascii=False)
shown = f"{arguments!r:.200}"
if isinstance(arguments, str):
try:
parsed = json.loads(arguments)
except json.JSONDecodeError as err:
raise ValueError(
f"tool call function.arguments must be valid JSON, got {shown}"
) from err
if isinstance(parsed, dict):
return arguments
raise ValueError(f"tool call function.arguments must be a JSON object, got {shown}")
def _attach_tools_to_dsv4_messages(messages: list[dict], tools: list[dict]) -> None:
for message in messages:
if message.get("role") == "system":
message["tools"] = tools
return
messages.insert(0, {"role": "system", "content": "", "tools": tools})