FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(server): keep Anthropic inline system messages in place for prefix reuse (#608)
William Xinze Zheng93b0a59164bb
diff --git a/docs/cli.md b/docs/cli.mdindex deeda184..b3cec998 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -99,6 +99,7 @@ See [models.md](models.md#moe-strategies) for what each strategy does.| `--tool-call-parser` | auto | Tool-call format; auto-inferred from the model family || `--reasoning-parser` | auto | Splits chain-of-thought into `reasoning_content`; auto-inferred; `off` disables || `--enable-cache-report` | off | Report prefix-cache hits in each response's usage block |+| `--anthropic-inline-system` | auto | Placement of late Anthropic system instructions: `auto`, `preserve`, or `fold` |### Image input@@ -203,4 +204,3 @@ profile that `ft serve --moe-strategy auto` and `--moe-hybrid-max-fetch -1` then- What to measure: `--dtype`, `--model`, `--formats`, `--isa`.- `--threshold` (default 2.0) sets the call: recommend hybrid when CPU bandwidth beats PCIeby that factor.-diff --git a/python/freetoken/message/tokenizer.py b/python/freetoken/message/tokenizer.pyindex 9442ee05..d52d658e 100644--- a/python/freetoken/message/tokenizer.py+++ b/python/freetoken/message/tokenizer.py@@ -73,6 +73,7 @@ class TokenizeMsg(BaseTokenizerMsg):chat_template_kwargs: Dict[str, Any] | None = Nonetools: List[Dict[str, Any]] | None = Noneimages: List[bytes] | None = None+ inline_system_policy: str | None = None@dataclassdiff --git a/python/freetoken/server/anthropic_api.py b/python/freetoken/server/anthropic_api.pyindex 0d60940c..d9a6972f 100644--- a/python/freetoken/server/anthropic_api.py+++ b/python/freetoken/server/anthropic_api.py@@ -120,6 +120,7 @@ async def handle_anthropic_messages(default_max_tokens=(getattr(state.config, "max_output_tokens", None) or DEFAULT_MAX_OUTPUT_TOKENS),+ inline_system_policy=getattr(state.config, "anthropic_inline_system", "auto"),)uid = await submit_generation(spec, state)except ValueError as exc:@@ -164,7 +165,10 @@ async def handle_anthropic_count_tokens(req: AnthropicCountTokensRequest, state:400, "invalid_request_error", "messages: no tokenizable content")try:- n_tokens = await count_prompt_tokens(messages, template_tools, ctk, state)+ n_tokens = await count_prompt_tokens(+ messages, template_tools, ctk, state,+ inline_system_policy=getattr(state.config, "anthropic_inline_system", "auto"),+ )except GenerationError as exc:# The chat template could not render this conversation (bad role ordering, an unmatched# tool_result, ...) — a client error, exactly as /v1/messages classifies the same failure.@@ -186,9 +190,8 @@ def convert_anthropic_prompt("""(messages, template_tools, parser_tools, chat_template_kwargs) — the promptside of the conversion, shared by /v1/messages and /v1/messages/count_tokens soa counted prompt is exactly the prompt a generation would tokenize."""- # Collect all system content (top-level `system` + any system-role messages- # Claude Code interleaves in the array) and emit ONE system message at the- # front: strict chat templates (e.g. Qwen3.5) require system at the beginning.+ # Hoisting turn-specific reminders rewrites the cached prompt prefix.+ # Preserve their position; the tokenizer adapts them to the model's roles.system_texts: list[str] = []if req.system:if isinstance(req.system, str):@@ -201,7 +204,11 @@ def convert_anthropic_prompt(other: list[dict[str, Any]] = []for msg in req.messages:if msg.role == "system":- system_texts.append(_content_text(msg.content))+ text = _content_text(msg.content)+ if not other:+ system_texts.append(text)+ elif text:+ other.append({"role": "system", "content": text})continueif isinstance(msg.content, str):@@ -308,6 +315,7 @@ def convert_anthropic_to_genspec(model_sampling: dict[str, Any],reasoning_parser: str | None = None,default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,+ inline_system_policy: str = "auto",) -> GenSpec:messages, template_tools, parser_tools, ctk = convert_anthropic_prompt(req, reasoning_parser=reasoning_parser@@ -327,6 +335,7 @@ def convert_anthropic_to_genspec(chat_template_kwargs=ctk,template_tools=template_tools,parser_tools=parser_tools,+ inline_system_policy=inline_system_policy,)diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex 4dd57b1a..2194b82e 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -59,6 +59,7 @@ class ServerArgs(SchedulerConfig):# prompt_tokens_details.cached_tokens, Anthropic cache_read_input_tokens, Responses# input_tokens_details.cached_tokens). Mirrors sglang's --enable-cache-report.enable_cache_report: bool = False+ anthropic_inline_system: str = "auto"# Comma-separated hostname allowlist for client-supplied image URLs; empty admits any domain.allowed_media_domains: str = ""# Directory file:// image refs may be read from; empty rejects local files.@@ -535,6 +536,14 @@ def parse_args(),)+ parser.add_argument(+ "--anthropic-inline-system",+ choices=("auto", "preserve", "fold"),+ default=ServerArgs.anthropic_inline_system,+ help="Preserve inline system instructions when supported by the renderer, "+ "or fold them into nearby user/tool content without hoisting the prompt prefix.",+ )+parser.add_argument("--sampling-defaults",type=str,diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.pyindex 91056e8d..ac988edd 100644--- a/python/freetoken/server/generation.py+++ b/python/freetoken/server/generation.py@@ -25,6 +25,7 @@ from freetoken.core import SamplingParamsfrom freetoken.message import TokenizeMsgfrom freetoken.mm.media import collect_image_refs, fetch_image_bytes, image_reject_reasonfrom freetoken.tokenizer.tokenize import resolve_thinking_mode+from freetoken.tokenizer.inline_system import InlineSystemErrortry:# Chat templates render through jinja2 (a transformers dependency): a TemplateError means@@ -142,6 +143,7 @@ class GenSpec:chat_template_kwargs: dict[str, Any] = field(default_factory=dict)template_tools: list[dict[str, Any]] | None = None # tools the model sees (TokenizeMsg.tools)parser_tools: list[dict[str, Any]] | None = None # tools for FunctionCallParser; None disables parsing+ inline_system_policy: str | None = None@propertydef parse_tools(self) -> bool:@@ -302,6 +304,7 @@ async def submit_generation(spec: GenSpec, state: Any) -> int:chat_template_kwargs=spec.chat_template_kwargs,tools=spec.template_tools,images=images,+ inline_system_policy=spec.inline_system_policy,))return uid@@ -312,6 +315,8 @@ async def count_prompt_tokens(tools: list[dict[str, Any]] | None,chat_template_kwargs: dict[str, Any],state: Any,+ *,+ inline_system_policy: str | None = None,) -> int:"""Token count of an already-converted (messages, tools, chat_template_kwargs) prompt,using the frontend's own tokenizer (``state.frontend_tokenizer()``) so the count equals the@@ -334,10 +339,13 @@ async def count_prompt_tokens(chat_template_kwargs=chat_template_kwargs,tools=tools,images=images,+ inline_system_policy=inline_system_policy,)manager = await asyncio.to_thread(state.frontend_tokenizer) # init failure -> server faulttry:(user_msg,) = await asyncio.to_thread(manager.tokenize, [msg])+ except InlineSystemError as exc:+ raise GenerationError(str(exc)) from excexcept _TemplateError as exc:raise GenerationError(str(exc)) from excreturn int(user_msg.input_ids.numel())@@ -361,6 +369,7 @@ async def prerender_error(spec: GenSpec, state: Any) -> GenerationError | None:sampling_params=SamplingParams(),chat_template_kwargs=spec.chat_template_kwargs,tools=spec.template_tools,+ inline_system_policy=spec.inline_system_policy,)try:manager = await asyncio.to_thread(build)diff --git a/python/freetoken/tokenizer/inline_system.py b/python/freetoken/tokenizer/inline_system.pynew file mode 100644index 00000000..7e6f2c06--- /dev/null+++ b/python/freetoken/tokenizer/inline_system.py@@ -0,0 +1,173 @@+# SPDX-License-Identifier: Apache-2.0+# Renderer-probe approach adapted from vLLM PR #58772.+"""Preserve or fold late system instructions without hoisting the prompt head."""+from __future__ import annotations++from copy import deepcopy+from typing import Any, Callable+++# Allow small role-framing differences, not a repeated environment header.+_INLINE_ROLE_SLACK_TOKENS = 8+++class InlineSystemError(ValueError):+ """An input instruction cannot be placed without corrupting tool ordering."""+++def has_inline_system(messages: list[dict]) -> bool:+ started = False+ for message in messages:+ if message.get("role") == "system":+ if started:+ return True+ else:+ started = True+ return False+++def _add_text(message: dict, text: str, *, prepend: bool = False) -> dict:+ result = dict(message)+ content = result.get("content")+ def join(a, b):+ return "\n\n".join(part for part in (a, b) if part)+ if content is None or isinstance(content, str):+ result["content"] = join(text, content) if prepend else join(content, text)+ elif isinstance(content, list):+ parts = deepcopy(content)+ index = 0 if prepend else -1+ if parts and parts[index].get("type") == "text":+ previous = parts[index].get("text") or ""+ parts[index]["text"] = join(text, previous) if prepend else join(previous, text)+ else:+ parts.insert(0 if prepend else len(parts), {"type": "text", "text": text})+ result["content"] = parts+ else:+ raise InlineSystemError("inline system folding requires text or content blocks")+ return result+++def _after_tool_results(messages: list[dict]) -> list[dict]:+ out, held = [], []+ pending_ids: set[str] = set()+ for message in messages:+ role = message.get("role")+ if role == "system" and pending_ids:+ held.append(message)+ continue+ if role == "tool":+ out.append(message)+ pending_ids.discard(message.get("tool_call_id", ""))+ if not pending_ids:+ out.extend(held)+ held.clear()+ continue+ if pending_ids and held:+ raise InlineSystemError("inline system instruction interrupts incomplete tool results")+ out.append(message)+ pending_ids = {call.get("id", "") for call in message.get("tool_calls", [])} if role == "assistant" else set()+ if held:+ raise InlineSystemError("inline system instruction is waiting for missing tool results")+ return out+++def normalize_inline_system(messages: list[dict], mode: str) -> list[dict]:+ if mode not in ("preserve", "fold"):+ raise InlineSystemError(f"invalid inline system mode: {mode}")+ if not has_inline_system(messages):+ return messages+ ordered = _after_tool_results(deepcopy(messages))+ # Without a stable leading system, some renderers attach tools to the first+ # late system message. Folding avoids moving tools when that message changes.+ if mode == "preserve" and ordered[0].get("role") == "system":+ return ordered+ out, pending = [], []+ started = False+ for message in ordered:+ role = message.get("role")+ if role == "system" and started:+ text = message.get("content") or ""+ if not isinstance(text, str):+ raise InlineSystemError("inline system instructions must be text")+ if text:+ if out and out[-1].get("role") in ("user", "tool") and not pending:+ out[-1] = _add_text(out[-1], text)+ else:+ pending.append(text)+ continue+ if pending:+ text = "\n\n".join(pending)+ pending.clear()+ if role == "user":+ message = _add_text(message, text, prepend=True)+ else:+ out.append({"role": "user", "content": text})+ out.append(message)+ started = started or role != "system"+ if pending:+ out.append({"role": "user", "content": "\n\n".join(pending)})+ return out+++def _insertion(base: list[int], new: list[int]) -> tuple[int, list[int]] | None:+ start = 0+ while start < min(len(base), len(new)) and base[start] == new[start]:+ start += 1+ suffix = 0+ while suffix < min(len(base), len(new)) - start and base[-suffix-1] == new[-suffix-1]:+ suffix += 1+ if start + suffix != len(base):+ return None+ return start, new[start:len(new)-suffix]+++def probe_inline_system(render: Callable[[list[dict]], str], tokenizer: Any) -> tuple[str, str]:+ """Conservatively require a distinct, prefix-stable system segment."""+ note, reasoning = "Echo inline instruction", "Foxtrot previous reasoning"+ system = {"role": "system", "content": "Alpha stable instructions"}+ user = {"role": "user", "content": "Bravo first question"}+ cases = [+ ("middle", [system, user, {"role": "assistant", "content": "Charlie first answer"},+ {"role": "user", "content": "Delta second question"}], 3),+ ("trailing", [system, user], 2),+ ("tool result", [system, user,+ {"role": "assistant", "content": "", "reasoning_content": reasoning, "thinking": reasoning,+ "reasoning": reasoning, "tool_calls": [{"id": "probe_call", "type": "function",+ "function": {"name": "read", "arguments": {}}}]},+ {"role": "tool", "tool_call_id": "probe_call", "content": "Golf tool output"}], 4),+ ]+ max_system_tokens = None+ try:+ special_ids = set(tokenizer.all_special_ids) | set(tokenizer.get_added_vocab().values())+ if not special_ids:+ return "fold", "renderer has no identifiable special tokens"+ for name, messages, index in cases:+ base_text = render(deepcopy(messages))+ base = tokenizer.encode(base_text, add_special_tokens=False)+ with_note = messages[:index] + [{"role": "system", "content": note}] + messages[index:]+ new_text = render(deepcopy(with_note))+ new = tokenizer.encode(new_text, add_special_tokens=False)+ found = _insertion(base, new)+ if found is None:+ return "fold", f"{name}: other prompt tokens change"+ position, segment = found+ if index == len(messages) and position >= len(base):+ return "fold", f"{name}: instruction follows the generation opening"+ if tokenizer.decode(segment).count(note) != 1 or not special_ids.intersection(segment):+ return "fold", f"{name}: no distinct role-marked instruction"+ if (reasoning in base_text) != (reasoning in new_text):+ return "fold", f"{name}: reasoning retention changes"+ if name == "middle":+ as_user = messages[:index] + [{"role": "user", "content": note}] + messages[index:]+ user_tokens = tokenizer.encode(render(deepcopy(as_user)), add_special_tokens=False)+ user_segment = _insertion(base, user_tokens)+ if user_segment is None:+ return "fold", "middle: cannot isolate a user instruction"+ if user_segment[1] == segment:+ return "fold", "system instructions render as ordinary user turns"+ max_system_tokens = len(user_segment[1]) + _INLINE_ROLE_SLACK_TOKENS+ if max_system_tokens is not None and len(segment) > max_system_tokens:+ return "fold", f"{name}: system insertion overhead ({len(segment)} > {max_system_tokens} tokens)"+ except Exception as exc:+ return "fold", f"probe could not render: {type(exc).__name__}"+ return "preserve", "renderer preserves distinct system turns and existing prompt tokens"diff --git a/python/freetoken/tokenizer/tokenize.py b/python/freetoken/tokenizer/tokenize.pyindex 758ffd4d..f4736652 100644--- a/python/freetoken/tokenizer/tokenize.py+++ b/python/freetoken/tokenizer/tokenize.py@@ -4,6 +4,7 @@ import importlib.utilimport jsonimport osimport threading+from copy import deepcopyfrom types import ModuleTypefrom typing import TYPE_CHECKING, Any, List@@ -22,6 +23,7 @@ from .effort import (probe_thinking_profile,quantize_effort,)+from .inline_system import has_inline_system, normalize_inline_system, probe_inline_systemlogger = init_logger(__name__)@@ -58,6 +60,8 @@ class TokenizeManager:self._thinking_profile: ThinkingProfile | None = Noneself._effort_lock = threading.Lock()self._logged_effort_maps: set[tuple[Any, str | None]] = set()+ self._inline_system_lock = threading.Lock()+ self._inline_system_modes: dict[str, str] = {}def tokenize(self, msgs: List[TokenizeMsg]) -> List[UserMsg]:results: List[UserMsg] = []@@ -103,9 +107,30 @@ class TokenizeManager:validation, count_tokens) must quantize identically."""if not isinstance(msg.text, list):return msg.text- return self._render(- msg.text, msg.tools, self._sanitize_effort(msg.chat_template_kwargs or {})- )+ kwargs = self._sanitize_effort(msg.chat_template_kwargs or {})+ messages = msg.text+ if msg.inline_system_policy is not None:+ if msg.inline_system_policy not in ("auto", "preserve", "fold"):+ raise ValueError(f"invalid inline system policy: {msg.inline_system_policy}")+ if has_inline_system(messages):+ mode = msg.inline_system_policy+ if mode == "auto":+ mode = self.inline_system_mode(msg.tools, kwargs)+ messages = normalize_inline_system(messages, mode)+ return self._render(messages, msg.tools, kwargs)++ def inline_system_mode(self, tools: list[dict] | None, kwargs: dict[str, Any]) -> str:+ key = json.dumps([tools, kwargs], sort_keys=True)+ with self._inline_system_lock:+ if key not in self._inline_system_modes:+ mode, reason = probe_inline_system(+ lambda messages: self._render(messages, deepcopy(tools), deepcopy(kwargs)), self.tokenizer+ )+ if len(self._inline_system_modes) >= 128:+ self._inline_system_modes.pop(next(iter(self._inline_system_modes)))+ self._inline_system_modes[key] = mode+ logger.info("Anthropic inline system policy: %s (%s)", mode, reason)+ return self._inline_system_modes[key]def _render(self,diff --git a/tests/README.md b/tests/README.mdindex 26dba633..be22c1fb 100644--- a/tests/README.md+++ b/tests/README.md@@ -53,7 +53,10 @@ checkpoint is set:| `FREETOKEN_GEMMA4_MODEL` | `models/test_gemma4_vision.py`, `tokenizer/test_mm_tokenize.py` — local Gemma-4 `gemma4` checkpoint (tower parity, boi/eoi expansion; the tokenizer test also takes the 12B `gemma4_unified` checkpoint) || `FREETOKEN_GEMMA4_UNIFIED_MODEL` | `models/test_gemma4_vision.py` — local gemma-4-12B `gemma4_unified` checkpoint (embedder parity) || `FREETOKEN_GLM53_MODEL` | `tokenizer/test_mm_tokenize.py` — local GLM-5.3-Flash checkpoint (image-token expansion) |-| `FREETOKEN_MUSE_MODEL` | `models/test_muse_glimmer_vision.py`, `tokenizer/test_mm_tokenize.py` — local Muse-Glimmer-30B checkpoint (tower parity, image_start/image_end expansion) |+| `FREETOKEN_DSV4_MODEL` | `tokenizer/test_tokenize.py` - local DSV4 checkpoint with its native `encoding/encoding_dsv4.py`; prefix stability and reminder retention, tokenizer only (no GPU or model weights loaded) |+| `FREETOKEN_INLINE_GLM_MODEL` | `tokenizer/test_tokenize.py` - local GLM-5.3 tokenizer/template; inline-system preserve policy, prefix stability and reasoning retention (no weights loaded) |+| `FREETOKEN_INLINE_QWEN_MODEL` | `tokenizer/test_tokenize.py` - local Qwen3.6 tokenizer/template; inline-system fold policy, prefix stability and reasoning retention (no weights loaded) |+| `FREETOKEN_MUSE_MODEL` | `models/test_muse_glimmer_vision.py`, `tokenizer/test_mm_tokenize.py`, `tokenizer/test_tokenize.py` - local Muse-Glimmer-30B checkpoint (vision tests; tokenizer-only inline-system regressions) || `FREETOKEN_MINIMAX_M3_MODEL` | `tokenizer/test_mm_tokenize.py` — local MiniMax-M3 checkpoint (start/end expansion, token budget) || `FREETOKEN_TEST_MOE_CACHE_SIZE` | `e2e/test_aime.py` — >0 switches to the offload MoE backend with this cache size || `FREETOKEN_TEST_MEM_RATIO` | `e2e/test_aime.py` — offload-mode memory_ratio (default `0.9`) |diff --git a/tests/server/test_anthropic_api.py b/tests/server/test_anthropic_api.pyindex 327bfcd8..f382093f 100644--- a/tests/server/test_anthropic_api.py+++ b/tests/server/test_anthropic_api.py@@ -231,10 +231,7 @@ def test_convert_thinking_replay_in_tool_loop():assert spec.messages[2]["role"] == "tool"-def test_convert_hoists_and_merges_system_messages():- # Claude Code interleaves system messages mid-array; strict chat templates- # (e.g. Qwen3.5: "System message must be at the beginning") require ONE system- # message at the front. Merge top-level system + in-array system, hoist to front.+def test_convert_preserves_mid_conversation_system_reminders():req = AnthropicMessagesRequest.model_validate({"model": "claude-x",@@ -247,11 +244,69 @@ def test_convert_hoists_and_merges_system_messages():],})+ original = req.model_dump()spec = A.convert_anthropic_to_genspec(req, {})- assert [m["role"] for m in spec.messages] == ["system", "user", "assistant"]- assert sum(1 for m in spec.messages if m["role"] == "system") == 1- assert "top-level sys" in spec.messages[0]["content"]- assert "mid-stream sys" in spec.messages[0]["content"]+ assert spec.messages == [+ {"role": "system", "content": "top-level sys"},+ {"role": "user", "content": "hello"},+ {"role": "system", "content": "mid-stream sys"},+ {"role": "assistant", "content": "hi"},+ ]+ assert req.model_dump() == original+++def test_convert_merges_only_leading_system_messages():+ req = AnthropicMessagesRequest.model_validate({+ "model": "claude-x", "max_tokens": 64,+ "system": [{"type": "text", "text": "top-level sys"}],+ "messages": [+ {"role": "system", "content": "leading sys"},+ {"role": "system", "content": [{"type": "text", "text": "second sys"}]},+ {"role": "user", "content": "hello"},+ {"role": "system", "content": [{"type": "text", "text": "turn reminder"}]},+ {"role": "assistant", "content": "hi"},+ {"role": "system", "content": ""},+ {"role": "system", "content": "next reminder"},+ ],+ })+ original = req.model_dump()+ spec = A.convert_anthropic_to_genspec(req, {})+ assert spec.messages == [+ {"role": "system", "content": "top-level sys\n\nleading sys\n\nsecond sys"},+ {"role": "user", "content": "hello"},+ {"role": "system", "content": "turn reminder"},+ {"role": "assistant", "content": "hi"},+ {"role": "system", "content": "next reminder"},+ ]+ assert req.model_dump() == original++[email protected]("system", [None, "stable instructions"])[email protected]("count_tokens", [False, True])+def test_appending_system_reminder_preserves_converted_prefix(system, count_tokens):+ from freetoken.server.anthropic_models import AnthropicCountTokensRequest++ data = {+ "model": "claude-x",+ "messages": [+ {"role": "user", "content": "inspect the source"},+ {"role": "assistant", "content": "I found the function"},+ ],+ "tools": [{"name": "read", "input_schema": {"type": "object", "properties": {}}}],+ }+ if system is not None:+ data["system"] = system+ if not count_tokens:+ data["max_tokens"] = 64+ request_type = AnthropicCountTokensRequest if count_tokens else AnthropicMessagesRequest+ first, first_tools, _, _ = A.convert_anthropic_prompt(request_type.model_validate(data))+ data["messages"].append({"role": "system", "content": "<total_tokens>1000 tokens left</total_tokens>"})+ second, second_tools, _, _ = A.convert_anthropic_prompt(request_type.model_validate(data))+ assert second[:-1] == first+ assert second[-1] == {+ "role": "system", "content": "<total_tokens>1000 tokens left</total_tokens>"+ }+ assert second_tools == first_tools# --------------------------------------------------------------------------- #@@ -664,6 +719,7 @@ def test_count_tokens_route():assert msg.text == spec.messagesassert msg.tools == spec.template_toolsassert msg.chat_template_kwargs == spec.chat_template_kwargs+ assert msg.inline_system_policy == spec.inline_system_policy == "auto"def test_count_tokens_empty_messages_400():@@ -673,6 +729,28 @@ def test_count_tokens_empty_messages_400():assert r.json()["error"]["type"] == "invalid_request_error"[email protected]("policy", ["auto", "preserve", "fold"])+def test_inline_system_override_reaches_counting_and_generation(policy):+ manager = _FakeTokenizeManager()+ fake = FakeState([("ok", True, 3, 1)])+ fake._count_manager = manager+ fake.config.anthropic_inline_system = policy+ submitted = []++ async def record(msg):+ submitted.append(msg)++ fake.send_one = record+ client = _client(fake)+ body = {**_COUNT_BODY, "messages": [+ {"role": "user", "content": "question"}, {"role": "system", "content": "instruction"}]}+ assert client.post("/v1/messages/count_tokens", json=body).status_code == 200+ assert client.post("/v1/messages", json={**body, "max_tokens": 32}).status_code == 200+ assert manager.msgs[0].inline_system_policy == submitted[0].inline_system_policy == policy+ assert manager.msgs[0].text == submitted[0].text+ assert submitted[0].text[0]["content"] == body["system"]++def test_count_tokens_works_while_not_serving():# Counting never touches the engine, so it must not 503 during load/rebuild.client, _ = _count_client(_FakeTokenizeManager(), maintenance_state="rebuilding")@@ -719,6 +797,15 @@ def test_count_tokens_template_render_error_400():assert r.json()["error"]["type"] == "invalid_request_error"+def test_count_tokens_inline_system_placement_error_is_400():+ from freetoken.tokenizer.inline_system import InlineSystemError++ client, _ = _count_client(_RaisingManager(InlineSystemError("missing tool results")))+ r = client.post("/v1/messages/count_tokens", json=_COUNT_BODY)+ assert r.status_code == 400+ assert r.json()["error"]["type"] == "invalid_request_error"++def test_count_tokens_excluded_from_request_ring():# count_tokens never enters generation accounting; its latency (incl. first-touch tokenizer# load) must not land in the /v1/requests ring or the /v1/stats p95.diff --git a/tests/server/test_message_wire.py b/tests/server/test_message_wire.pyindex e2286671..fdc09592 100644--- a/tests/server/test_message_wire.py+++ b/tests/server/test_message_wire.py@@ -28,6 +28,18 @@ from freetoken.message import (from freetoken.core import SamplingParams+def test_inline_system_policy_tokenizer_roundtrip():+ for policy in (None, "auto", "preserve", "fold"):+ msg = TokenizeMsg(uid=1, text=[{"role": "system", "content": "instruction"}],+ sampling_params=SamplingParams(), inline_system_policy=policy)+ decoded = BaseTokenizerMsg.decoder(BaseTokenizerMsg.encoder(msg))+ assert decoded.inline_system_policy == policy+ assert decoded.text == msg.text+ legacy = BaseTokenizerMsg.encoder(msg)+ legacy.pop("inline_system_policy")+ assert BaseTokenizerMsg.decoder(legacy).inline_system_policy is None++def test_cache_rebuild_msg_roundtrip():msg = CacheRebuildMsg(request_id="abc", moe_cache_size=8, num_pages=1024, mode="if_idle")out = BaseTokenizerMsg.decoder(BaseTokenizerMsg.encoder(msg))diff --git a/tests/tokenizer/test_tokenize.py b/tests/tokenizer/test_tokenize.pyindex 369576a1..16ef6d74 100644--- a/tests/tokenizer/test_tokenize.py+++ b/tests/tokenizer/test_tokenize.py@@ -1,6 +1,8 @@from __future__ import annotationsimport json+import os+from copy import deepcopyimport pytestimport torch@@ -8,14 +10,303 @@ import torchfrom freetoken.core import SamplingParamsfrom freetoken.message import TokenizeMsgfrom freetoken.tokenizer.tokenize import TokenizeManager, _dsv4_arguments_str+from freetoken.tokenizer.inline_system import InlineSystemError, normalize_inline_system, probe_inline_system+++def test_fold_keeps_leading_system_and_appends_text_without_mutation():+ messages = [+ {"role": "system", "content": "stable"},+ {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "image"}},+ {"type": "text", "text": "question"}]},+ {"role": "system", "content": "instruction 1"},+ {"role": "system", "content": "instruction 2"},+ ]+ original = deepcopy(messages)+ result = normalize_inline_system(messages, "fold")+ assert result == [messages[0], {"role": "user", "content": [+ messages[1]["content"][0], {"type": "text", "text": "question\n\ninstruction 1\n\ninstruction 2"}]}]+ assert messages == original++[email protected]("mode", ["preserve", "fold"])+def test_inline_system_waits_for_all_parallel_tool_results(mode):+ messages = [+ {"role": "system", "content": "stable"}, {"role": "user", "content": "question"},+ {"role": "assistant", "reasoning_content": "reason", "tool_calls": [{"id": "a"}, {"id": "b"}]},+ {"role": "system", "content": "before results"},+ {"role": "tool", "tool_call_id": "a", "content": "result a"},+ {"role": "system", "content": "between results"},+ {"role": "tool", "tool_call_id": "b", "content": "result b"},+ {"role": "system", "content": "after results"},+ ]+ original = deepcopy(messages)+ result = normalize_inline_system(messages, mode)+ assert [m["tool_call_id"] for m in result if m["role"] == "tool"] == ["a", "b"]+ assert result[2] == messages[2]+ assert [m["role"] for m in result[3:5]] == ["tool", "tool"]+ if mode == "fold":+ assert len(result) == 5+ assert result[-1]["content"] == "result b\n\nbefore results\n\nbetween results\n\nafter results"+ else:+ assert [m["content"] for m in result[5:]] == ["before results", "between results", "after results"]+ assert messages == original++[email protected]("mode", ["preserve", "fold"])+def test_inline_system_does_not_hide_missing_tool_results(mode):+ messages = [{"role": "assistant", "tool_calls": [{"id": "missing"}]},+ {"role": "system", "content": "instruction"}]+ with pytest.raises(InlineSystemError, match="missing tool results"):+ normalize_inline_system(messages, mode)+++def test_fold_after_assistant_prepends_next_user_or_supplies_a_user_turn():+ messages = [{"role": "user", "content": "first"}, {"role": "assistant", "content": "answer"},+ {"role": "system", "content": "instruction"}]+ assert normalize_inline_system(messages, "fold")[-1] == {"role": "user", "content": "instruction"}+ result = normalize_inline_system(messages + [{"role": "user", "content": "next"}], "fold")+ assert result == messages[:2] + [{"role": "user", "content": "instruction\n\nnext"}]+++def test_preserve_keeps_instruction_role_and_only_folds_without_a_stable_system():+ messages = [{"role": "system", "content": "stable"}, {"role": "user", "content": "question"},+ {"role": "system", "content": "late"}]+ assert normalize_inline_system(messages, "preserve") == messages+ assert normalize_inline_system(messages[1:], "preserve") == [{"role": "user", "content": "question\n\nlate"}]+++class ProbeTokenizer:+ vocabulary = {text: 10000+i for i, text in enumerate(+ ["<system>", "</system>", "<user>", "</user>", "<assistant>", "</assistant>",+ "<tool>", "</tool>", "<think>", "</think>"])}+ all_special_ids = list(vocabulary.values())++ def get_added_vocab(self):+ return self.vocabulary++ def encode(self, text, **kwargs):+ import re+ parts = re.split("(" + "|".join(re.escape(x) for x in self.vocabulary) + ")", text)+ return [token for part in parts for token in ([self.vocabulary[part]] if part in self.vocabulary else map(ord, part))]++ def decode(self, ids):+ inverse = {v: k for k, v in self.vocabulary.items()}+ return "".join(inverse[x] if x in inverse else chr(x) for x in ids)++[email protected]("behavior,expected", [("preserve", "preserve"), ("reject", "fold"),+ ("drop", "fold"), ("bare", "fold"), ("user", "fold"), ("rewrite", "fold"),+ ("after_generation", "fold"), ("drop_reasoning", "fold")])+def test_renderer_probe_checks_more_than_template_acceptance(behavior, expected):+ def render(messages):+ inline = any(m["role"] == "system" for m in messages[1:])+ output, late = "", ""+ for i, message in enumerate(messages):+ role, content = message["role"], message.get("content", "")+ if role == "system" and i:+ if behavior == "reject":+ raise ValueError("system first only")+ if behavior == "drop":+ continue+ if behavior == "bare":+ output += content+ continue+ if behavior == "user":+ role = "user"+ if behavior == "after_generation":+ late += content+ continue+ if behavior == "rewrite" and inline and i == 0:+ content = "changed head"+ reason = message.get("reasoning_content")+ if reason and not (behavior == "drop_reasoning" and inline):+ content = "<think>" + reason + "</think>" + content+ output += f"<{role}>{content}</{role}>"+ return output + "<assistant>" + late+ assert probe_inline_system(render, ProbeTokenizer())[0] == expected++[email protected]("position", ["middle", "trailing", "tool result"])[email protected]("metadata", ["Reasoning strength: high. Valid recipients: self, user.", "tool schema " * 1000],+ ids=["metadata", "tools"])+def test_inline_probe_rejects_repeated_system_metadata(position, metadata):+ def render(messages):+ output = ""+ for i, message in enumerate(messages):+ role = message["role"]+ content = message.get("content", "")+ if role == "system" and i:+ if i < len(messages)-1:+ location = "middle"+ else:+ location = "tool result" if messages[i-1]["role"] == "tool" else "trailing"+ if location == position:+ content += metadata+ content = message.get("reasoning_content", "") + content+ output += f"<{role}>{content}</{role}>"+ return output + "<assistant>"++ mode, reason = probe_inline_system(render, ProbeTokenizer())+ assert mode == "fold"+ assert "overhead" in reason+++def test_inline_probe_allows_small_role_framing_overhead():+ def render(messages):+ return "".join(f'<{m["role"]}>' + ("role: " if m["role"] == "system" else "")+ + m.get("reasoning_content", "") + m.get("content", "")+ + f'</{m["role"]}>' for m in messages) + "<assistant>"++ assert probe_inline_system(render, ProbeTokenizer())[0] == "preserve"+++def test_inline_probe_folds_when_user_insertion_cannot_be_measured():+ def render(messages):+ rewrite = any(m["role"] == "user" and m.get("content") == "Echo inline instruction" for m in messages)+ return ("changed head" if rewrite else "stable head") + "".join(+ f'<{m["role"]}>' + m.get("reasoning_content", "") + m.get("content", "")+ + f'</{m["role"]}>' for m in messages) + "<assistant>"++ mode, reason = probe_inline_system(render, ProbeTokenizer())+ assert mode == "fold"+ assert "cannot isolate" in reason+++def test_inline_probe_cache_is_scoped_to_tools_and_render_kwargs(monkeypatch):+ import freetoken.tokenizer.tokenize as module+ calls = []+ monkeypatch.setattr(module, "probe_inline_system", lambda *_: (calls.append(1) or ("fold", "test")))+ manager = TokenizeManager(FakeTokenizer())+ for _ in range(3):+ assert manager.inline_system_mode(None, {}) == "fold"+ manager.inline_system_mode([], {})+ manager.inline_system_mode(None, {"enable_thinking": True})+ assert len(calls) == 3+++def test_inline_policy_is_opt_in_and_not_forwarded_to_templates():+ tokenizer = FakeTokenizer()+ manager = TokenizeManager(tokenizer)+ messages = [{"role": "user", "content": "question"}, {"role": "system", "content": "instruction"}]+ msg = TokenizeMsg(uid=1, text=messages, sampling_params=SamplingParams())+ manager.render_prompt(msg)+ assert tokenizer.chat_template_messages == messages+ msg.inline_system_policy = "fold"+ manager.render_prompt(msg)+ assert tokenizer.chat_template_messages == [{"role": "user", "content": "question\n\ninstruction"}]+ assert "inline_system_policy" not in tokenizer.chat_template_kwargs++[email protected]_weights[email protected]("variable,expected_mode", [+ ("FREETOKEN_DSV4_MODEL", "fold"),+ ("FREETOKEN_INLINE_GLM_MODEL", "preserve"),+ ("FREETOKEN_INLINE_QWEN_MODEL", "fold"),+])+def test_native_model_inline_system_tool_history(variable, expected_mode):+ from transformers import AutoTokenizer+ from freetoken.server.anthropic_api import convert_anthropic_to_genspec+ from freetoken.server.anthropic_models import AnthropicMessagesRequest++ path = os.environ.get(variable)+ if not path:+ pytest.skip(f"set {variable} to a local checkpoint")+ manager = TokenizeManager(AutoTokenizer.from_pretrained(path, local_files_only=True))+ data = {+ "model": "local", "max_tokens": 64, "system": "Stable project instructions.",+ "thinking": {"type": "enabled"},+ "tools": [{"name": "read", "input_schema": {"type": "object", "properties": {}}}],+ "messages": [+ {"role": "user", "content": "Inspect source:\n" + "value = 1\n" * 512},+ {"role": "system", "content": "Keep all existing tests."},+ ],+ }+ ids, prompts = [], []+ for turn in (1, 2):+ data["messages"].extend([+ {"role": "assistant", "content": [+ {"type": "thinking", "thinking": f"Retained thought {turn}.", "signature": ""},+ {"type": "tool_use", "id": f"read_{turn}", "name": "read", "input": {}},+ ]},+ {"role": "system", "content": f"During tool call {turn}."},+ {"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": f"read_{turn}", "content": f"Source result {turn}."},+ ]},+ {"role": "system", "content": f"Remaining budget: {1000-turn}."},+ ])+ req = AnthropicMessagesRequest.model_validate(data)+ spec = convert_anthropic_to_genspec(req, {})+ assert manager.inline_system_mode(spec.template_tools, spec.chat_template_kwargs) == expected_mode+ original = deepcopy(spec.messages)+ msg = TokenizeMsg(uid=turn, text=spec.messages, tools=spec.template_tools,+ sampling_params=spec.sampling_params, chat_template_kwargs=spec.chat_template_kwargs,+ inline_system_policy=spec.inline_system_policy)+ prompts.append(manager.render_prompt(msg))+ ids.append(manager.tokenize([msg])[0].input_ids.tolist())+ assert spec.messages == original+ assert prompts[-1].count("Keep all existing tests.") == 1+ assert "Retained thought 1." in prompts[-1]+ for previous in range(1, turn+1):+ assert prompts[-1].count(f"During tool call {previous}.") == 1+ assert prompts[-1].count(f"Remaining budget: {1000-previous}.") == 1+ common = next((i for i, pair in enumerate(zip(*ids)) if pair[0] != pair[1]), min(map(len, ids)))+ assert common >= len(ids[0]) - 64+ assert common > 2048++[email protected]_weights[email protected]("tool_count", [0, 20])+def test_native_muse_inline_system_does_not_repeat_environment(tool_count):+ from transformers import AutoTokenizer++ path = os.environ.get("FREETOKEN_MUSE_MODEL")+ if not path:+ pytest.skip("set FREETOKEN_MUSE_MODEL to a local checkpoint")+ manager = TokenizeManager(AutoTokenizer.from_pretrained(path, local_files_only=True))+ tools = [{"type": "function", "function": {"name": f"read_{i}",+ "description": "Inspect the repository source. " * 100,+ "parameters": {"type": "object", "properties": {}}}} for i in range(tool_count)] or None+ messages = [{"role": "system", "content": "Stable instructions."},+ {"role": "user", "content": "Inspect this file.\n" + "value = 1\n" * 256}]+ assert manager.inline_system_mode(tools, {}) == "fold"+ for turn in range(3):+ messages.extend([{"role": "assistant", "content": f"Answer {turn}."},+ {"role": "user", "content": f"Question {turn}."},+ {"role": "system", "content": f"Budget reminder {turn}."}])+ original = deepcopy(messages)+ msg = TokenizeMsg(uid=turn, text=messages, tools=tools, sampling_params=SamplingParams(),+ inline_system_policy="auto")+ prompt = manager.render_prompt(msg)+ assert prompt.count("Reasoning strength:") == 1+ assert prompt.count("# Valid recipients:") == 1+ assert prompt.count("// Function schemas") == bool(tools)+ for previous in range(turn+1):+ assert prompt.count(f"Budget reminder {previous}.") == 1+ ids = manager.tokenize([msg])[0].input_ids.tolist()+ msg.inline_system_policy = "fold"+ assert ids == manager.tokenize([msg])[0].input_ids.tolist()+ assert messages == original++ latest = [m for i, m in enumerate(messages) if m["role"] != "system" or i in (0, len(messages)-1)]+ updated = deepcopy(latest)+ updated[-1]["content"] = "Budget reminder 900."+ tokens = []+ for conversation in (latest, updated):+ msg = TokenizeMsg(uid=0, text=conversation, tools=tools, sampling_params=SamplingParams(),+ inline_system_policy="auto")+ tokens.append(manager.tokenize([msg])[0].input_ids.tolist())+ common = next((i for i, pair in enumerate(zip(*tokens)) if pair[0] != pair[1]), min(map(len, tokens)))+ assert common >= min(map(len, tokens)) - 16class FakeTokenizer:def __init__(self) -> None:self.chat_template_kwargs = None+ self.chat_template_messages = Nonedef apply_chat_template(self, messages, **kwargs):self.chat_template_kwargs = kwargs+ self.chat_template_messages = deepcopy(messages)return "rendered prompt"def encode(self, prompt, return_tensors=None, add_special_tokens=True):@@ -74,6 +365,31 @@ def test_tokenize_manager_passes_tools_to_chat_template():assert input_ids.tolist() == [1, 2, 3][email protected]("initial_system", [[], [{"role": "system", "content": "stable instructions"}]])+def test_generic_template_folds_system_messages_without_mutating_input(initial_system):+ tokenizer = FakeTokenizer()+ manager = TokenizeManager(tokenizer)+ messages = initial_system + [+ {"role": "user", "content": "inspect files"},+ {"role": "system", "content": "first reminder"},+ {"role": "assistant", "content": "found the bug"},+ {"role": "system", "content": "second reminder"},+ ]+ original = deepcopy(messages)+ tools = [{"type": "function", "function": {"name": "read", "parameters": {"type": "object"}}}]+ original_tools = deepcopy(tools)+ manager.tokenize([TokenizeMsg(uid=1, text=messages, tools=tools, sampling_params=SamplingParams(),+ inline_system_policy="auto")])+ assert tokenizer.chat_template_messages == initial_system + [+ {"role": "user", "content": "inspect files\n\nfirst reminder"},+ {"role": "assistant", "content": "found the bug"},+ {"role": "user", "content": "second reminder"},+ ]+ assert tokenizer.chat_template_kwargs["tools"] == original_tools+ assert messages == original+ assert tools == original_tools++class FakeDsv4Tokenizer:chat_template = None@@ -126,6 +442,166 @@ def encode_messages(messages, thinking_mode, reasoning_effort=None):assert input_ids.tolist() == [4, 5, 6][email protected]("initial_system", [[], [{"role": "system", "content": "stable instructions"}]])+def test_dsv4_fold_keeps_instructions_and_tools_on_stable_prefix(tmp_path, initial_system):+ encoding_dir = tmp_path / "encoding"+ encoding_dir.mkdir()+ (encoding_dir / "encoding_dsv4.py").write_text(+ "import json\n"+ "def encode_messages(messages, thinking_mode, reasoning_effort=None):\n"+ " return json.dumps(messages)\n"+ )+ tokenizer = FakeDsv4Tokenizer(tmp_path)+ manager = TokenizeManager(tokenizer)+ tools = [{"type": "function", "function": {"name": "read", "parameters": {"type": "object"}}}]+ messages = initial_system + [+ {"role": "user", "content": "inspect files"},+ {"role": "system", "content": "first reminder"},+ {"role": "assistant", "content": "found the bug"},+ ]+ original, original_tools = deepcopy(messages), deepcopy(tools)+ manager.tokenize([TokenizeMsg(uid=1, text=messages, tools=tools, sampling_params=SamplingParams(),+ inline_system_policy="fold")])+ first = json.loads(tokenizer.prompt)+ extended = messages + [{"role": "system", "content": "next reminder"}]+ original_extended = deepcopy(extended)+ manager.tokenize([TokenizeMsg(uid=2, text=extended, tools=tools, sampling_params=SamplingParams(),+ inline_system_policy="fold")])+ second = json.loads(tokenizer.prompt)+ assert [m["role"] for m in first] == ["system", "user", "assistant"]+ assert first[0]["content"] == (initial_system[0]["content"] if initial_system else "")+ assert first[0]["tools"] == original_tools+ assert first[1]["content"] == "inspect files\n\nfirst reminder"+ assert second[:-1] == first+ assert second[-1] == {"role": "user", "content": "next reminder"}+ assert all("tools" not in m for m in second[1:])+ assert messages == original and extended == original_extended and tools == original_tools++[email protected](scope="module")+def native_dsv4_manager():+ model_path = os.environ.get("FREETOKEN_DSV4_MODEL")+ if not model_path:+ pytest.skip("set FREETOKEN_DSV4_MODEL to a local DSV4 checkpoint")+ from transformers import AutoTokenizer++ manager = TokenizeManager(AutoTokenizer.from_pretrained(model_path, local_files_only=True))+ assert manager._dsv4_encoder is not None, "checkpoint must supply the native DSV4 encoder"+ return manager+++def test_dsv4_no_tools_retains_reminders_when_encoder_drops_developer_turns(tmp_path):+ encoding_dir = tmp_path / "encoding"+ encoding_dir.mkdir()+ (encoding_dir / "encoding_dsv4.py").write_text(+ "import json\n"+ "def encode_messages(messages, thinking_mode, reasoning_effort=None):\n"+ " if thinking_mode == 'thinking':\n"+ " messages = [m for m in messages if m['role'] != 'developer']\n"+ " return json.dumps(messages)\n"+ )+ manager = TokenizeManager(FakeDsv4Tokenizer(tmp_path))+ messages = [+ {"role": "user", "content": "First task."},+ {"role": "system", "content": "Keep this earlier instruction."},+ {"role": "assistant", "content": "First answer."},+ {"role": "user", "content": "Second task."},+ ]+ original = deepcopy(messages)+ msg = TokenizeMsg(uid=1, text=messages, sampling_params=SamplingParams(),+ chat_template_kwargs={"enable_thinking": True}, inline_system_policy="fold")+ rendered = json.loads(manager.render_prompt(msg))+ assert all(m["role"] in ("system", "user", "assistant") for m in rendered)+ assert sum(m["content"].count("Keep this earlier instruction.") for m in rendered) == 1+ assert messages == original++[email protected]_weights[email protected]("system", [None, "You are a coding assistant."])[email protected]("change", ["update", "append"])+def test_native_dsv4_reminder_update_preserves_token_prefix(native_dsv4_manager, system, change):+ from freetoken.server.anthropic_api import convert_anthropic_to_genspec+ from freetoken.server.anthropic_models import AnthropicMessagesRequest++ data = {+ "model": "dsv4", "max_tokens": 64,+ "tools": [{"name": "read", "input_schema": {"type": "object", "properties": {}}}],+ "messages": [+ {"role": "user", "content": "Inspect this file:\n" + "value = 1\n" * 512},+ {"role": "system", "content": "Keep the changes minimal."},+ {"role": "assistant", "content": [+ {"type": "thinking", "thinking": "I should read the source.", "signature": ""},+ {"type": "tool_use", "id": "call_read", "name": "read", "input": {}},+ ]},+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_read", "content": "value = 1"}]},+ {"role": "system", "content": "Remaining token budget: 1000"},+ ],+ }+ if system is not None:+ data["system"] = system+ tokens = []+ for budget in (1000, 900):+ if budget == 900 and change == "append":+ data["messages"].extend([+ {"role": "assistant", "content": [+ {"type": "thinking", "thinking": "The file is simple.", "signature": ""},+ {"type": "text", "text": "I inspected the file."},+ ]},+ {"role": "system", "content": "Remaining token budget: 900"},+ ])+ else:+ data["messages"][-1]["content"] = f"Remaining token budget: {budget}"+ req = AnthropicMessagesRequest.model_validate(data)+ original = req.model_dump()+ spec = convert_anthropic_to_genspec(req, {})+ msg = TokenizeMsg(uid=budget, text=spec.messages, tools=spec.template_tools,+ sampling_params=spec.sampling_params, chat_template_kwargs=spec.chat_template_kwargs,+ inline_system_policy=spec.inline_system_policy)+ prompt = native_dsv4_manager.render_prompt(msg)+ assert prompt.count("Keep the changes minimal.") == 1+ assert prompt.index("Inspect this file:") < prompt.index(f"Remaining token budget: {budget}")+ tokens.append(native_dsv4_manager.tokenize([msg])[0].input_ids.tolist())+ assert req.model_dump() == original+ common = next((i for i, pair in enumerate(zip(*tokens)) if pair[0] != pair[1]), min(map(len, tokens)))+ assert common > 2048+ assert common >= min(map(len, tokens)) - 64++[email protected]_weights[email protected]("disable_declared_tools", [False, True])[email protected]("thinking_enabled", [False, True])+def test_native_dsv4_no_tools_keeps_earlier_reminder(+ native_dsv4_manager, disable_declared_tools, thinking_enabled+):+ from freetoken.server.anthropic_api import convert_anthropic_to_genspec+ from freetoken.server.anthropic_models import AnthropicMessagesRequest++ data = {+ "model": "dsv4", "max_tokens": 64, "system": "Stable instructions.",+ "thinking": {"type": "enabled" if thinking_enabled else "disabled"},+ "messages": [+ {"role": "user", "content": "First task."},+ {"role": "system", "content": "Keep this earlier instruction."},+ {"role": "assistant", "content": "First answer."},+ {"role": "user", "content": "Second task."},+ ],+ }+ if disable_declared_tools:+ data["tools"] = [{"name": "read", "input_schema": {"type": "object", "properties": {}}}]+ data["tool_choice"] = {"type": "none"}+ req = AnthropicMessagesRequest.model_validate(data)+ original = req.model_dump()+ spec = convert_anthropic_to_genspec(req, {})+ assert not spec.template_tools+ msg = TokenizeMsg(uid=1, text=spec.messages, tools=spec.template_tools,+ sampling_params=spec.sampling_params, chat_template_kwargs=spec.chat_template_kwargs,+ inline_system_policy=spec.inline_system_policy)+ prompt = native_dsv4_manager.render_prompt(msg)+ assert prompt.count("Keep this earlier instruction.") == 1+ assert "Second task." in prompt+ assert req.model_dump() == original++def test_dsv4_encoder_gets_tool_call_arguments_as_json_string(tmp_path):"""Regression: render_messages hands the template dict arguments; the dsv4encoder contract is a JSON-object STRING -- a dict trips its fallback that