Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(server): keep Anthropic inline system messages in place for prefix reuse (#608)

William Xinze Zheng93b0a59164bb
diff --git a/docs/cli.md b/docs/cli.md
index deeda184..b3cec998 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -99,6 +99,7 @@ See [models.md](models.md#moe-strategies) for what each strategy does.
| `--tool-call-parser` | auto | Tool-call format; auto-inferred from the model family |
| `--reasoning-parser` | auto | Splits chain-of-thought into `reasoning_content`; auto-inferred; `off` disables |
| `--enable-cache-report` | off | Report prefix-cache hits in each response's usage block |
+| `--anthropic-inline-system` | auto | Placement of late Anthropic system instructions: `auto`, `preserve`, or `fold` |
### Image input
@@ -203,4 +204,3 @@ profile that `ft serve --moe-strategy auto` and `--moe-hybrid-max-fetch -1` then
- What to measure: `--dtype`, `--model`, `--formats`, `--isa`.
- `--threshold` (default 2.0) sets the call: recommend hybrid when CPU bandwidth beats PCIe
by that factor.
-
diff --git a/python/freetoken/message/tokenizer.py b/python/freetoken/message/tokenizer.py
index 9442ee05..d52d658e 100644
--- a/python/freetoken/message/tokenizer.py
+++ b/python/freetoken/message/tokenizer.py
@@ -73,6 +73,7 @@ class TokenizeMsg(BaseTokenizerMsg):
chat_template_kwargs: Dict[str, Any] | None = None
tools: List[Dict[str, Any]] | None = None
images: List[bytes] | None = None
+ inline_system_policy: str | None = None
@dataclass
diff --git a/python/freetoken/server/anthropic_api.py b/python/freetoken/server/anthropic_api.py
index 0d60940c..d9a6972f 100644
--- a/python/freetoken/server/anthropic_api.py
+++ b/python/freetoken/server/anthropic_api.py
@@ -120,6 +120,7 @@ async def handle_anthropic_messages(
default_max_tokens=(
getattr(state.config, "max_output_tokens", None) or DEFAULT_MAX_OUTPUT_TOKENS
),
+ inline_system_policy=getattr(state.config, "anthropic_inline_system", "auto"),
)
uid = await submit_generation(spec, state)
except ValueError as exc:
@@ -164,7 +165,10 @@ async def handle_anthropic_count_tokens(req: AnthropicCountTokensRequest, state:
400, "invalid_request_error", "messages: no tokenizable content"
)
try:
- n_tokens = await count_prompt_tokens(messages, template_tools, ctk, state)
+ n_tokens = await count_prompt_tokens(
+ messages, template_tools, ctk, state,
+ inline_system_policy=getattr(state.config, "anthropic_inline_system", "auto"),
+ )
except GenerationError as exc:
# The chat template could not render this conversation (bad role ordering, an unmatched
# tool_result, ...) — a client error, exactly as /v1/messages classifies the same failure.
@@ -186,9 +190,8 @@ def convert_anthropic_prompt(
"""(messages, template_tools, parser_tools, chat_template_kwargs) — the prompt
side of the conversion, shared by /v1/messages and /v1/messages/count_tokens so
a counted prompt is exactly the prompt a generation would tokenize."""
- # Collect all system content (top-level `system` + any system-role messages
- # Claude Code interleaves in the array) and emit ONE system message at the
- # front: strict chat templates (e.g. Qwen3.5) require system at the beginning.
+ # Hoisting turn-specific reminders rewrites the cached prompt prefix.
+ # Preserve their position; the tokenizer adapts them to the model's roles.
system_texts: list[str] = []
if req.system:
if isinstance(req.system, str):
@@ -201,7 +204,11 @@ def convert_anthropic_prompt(
other: list[dict[str, Any]] = []
for msg in req.messages:
if msg.role == "system":
- system_texts.append(_content_text(msg.content))
+ text = _content_text(msg.content)
+ if not other:
+ system_texts.append(text)
+ elif text:
+ other.append({"role": "system", "content": text})
continue
if isinstance(msg.content, str):
@@ -308,6 +315,7 @@ def convert_anthropic_to_genspec(
model_sampling: dict[str, Any],
reasoning_parser: str | None = None,
default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,
+ inline_system_policy: str = "auto",
) -> GenSpec:
messages, template_tools, parser_tools, ctk = convert_anthropic_prompt(
req, reasoning_parser=reasoning_parser
@@ -327,6 +335,7 @@ def convert_anthropic_to_genspec(
chat_template_kwargs=ctk,
template_tools=template_tools,
parser_tools=parser_tools,
+ inline_system_policy=inline_system_policy,
)
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index 4dd57b1a..2194b82e 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -59,6 +59,7 @@ class ServerArgs(SchedulerConfig):
# prompt_tokens_details.cached_tokens, Anthropic cache_read_input_tokens, Responses
# input_tokens_details.cached_tokens). Mirrors sglang's --enable-cache-report.
enable_cache_report: bool = False
+ anthropic_inline_system: str = "auto"
# Comma-separated hostname allowlist for client-supplied image URLs; empty admits any domain.
allowed_media_domains: str = ""
# Directory file:// image refs may be read from; empty rejects local files.
@@ -535,6 +536,14 @@ def parse_args(
),
)
+ parser.add_argument(
+ "--anthropic-inline-system",
+ choices=("auto", "preserve", "fold"),
+ default=ServerArgs.anthropic_inline_system,
+ help="Preserve inline system instructions when supported by the renderer, "
+ "or fold them into nearby user/tool content without hoisting the prompt prefix.",
+ )
+
parser.add_argument(
"--sampling-defaults",
type=str,
diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py
index 91056e8d..ac988edd 100644
--- a/python/freetoken/server/generation.py
+++ b/python/freetoken/server/generation.py
@@ -25,6 +25,7 @@ from freetoken.core import SamplingParams
from freetoken.message import TokenizeMsg
from freetoken.mm.media import collect_image_refs, fetch_image_bytes, image_reject_reason
from freetoken.tokenizer.tokenize import resolve_thinking_mode
+from freetoken.tokenizer.inline_system import InlineSystemError
try:
# Chat templates render through jinja2 (a transformers dependency): a TemplateError means
@@ -142,6 +143,7 @@ class GenSpec:
chat_template_kwargs: dict[str, Any] = field(default_factory=dict)
template_tools: list[dict[str, Any]] | None = None # tools the model sees (TokenizeMsg.tools)
parser_tools: list[dict[str, Any]] | None = None # tools for FunctionCallParser; None disables parsing
+ inline_system_policy: str | None = None
@property
def parse_tools(self) -> bool:
@@ -302,6 +304,7 @@ async def submit_generation(spec: GenSpec, state: Any) -> int:
chat_template_kwargs=spec.chat_template_kwargs,
tools=spec.template_tools,
images=images,
+ inline_system_policy=spec.inline_system_policy,
)
)
return uid
@@ -312,6 +315,8 @@ async def count_prompt_tokens(
tools: list[dict[str, Any]] | None,
chat_template_kwargs: dict[str, Any],
state: Any,
+ *,
+ inline_system_policy: str | None = None,
) -> int:
"""Token count of an already-converted (messages, tools, chat_template_kwargs) prompt,
using the frontend's own tokenizer (``state.frontend_tokenizer()``) so the count equals the
@@ -334,10 +339,13 @@ async def count_prompt_tokens(
chat_template_kwargs=chat_template_kwargs,
tools=tools,
images=images,
+ inline_system_policy=inline_system_policy,
)
manager = await asyncio.to_thread(state.frontend_tokenizer) # init failure -> server fault
try:
(user_msg,) = await asyncio.to_thread(manager.tokenize, [msg])
+ except InlineSystemError as exc:
+ raise GenerationError(str(exc)) from exc
except _TemplateError as exc:
raise GenerationError(str(exc)) from exc
return int(user_msg.input_ids.numel())
@@ -361,6 +369,7 @@ async def prerender_error(spec: GenSpec, state: Any) -> GenerationError | None:
sampling_params=SamplingParams(),
chat_template_kwargs=spec.chat_template_kwargs,
tools=spec.template_tools,
+ inline_system_policy=spec.inline_system_policy,
)
try:
manager = await asyncio.to_thread(build)
diff --git a/python/freetoken/tokenizer/inline_system.py b/python/freetoken/tokenizer/inline_system.py
new file mode 100644
index 00000000..7e6f2c06
--- /dev/null
+++ b/python/freetoken/tokenizer/inline_system.py
@@ -0,0 +1,173 @@
+# SPDX-License-Identifier: Apache-2.0
+# Renderer-probe approach adapted from vLLM PR #58772.
+"""Preserve or fold late system instructions without hoisting the prompt head."""
+from __future__ import annotations
+
+from copy import deepcopy
+from typing import Any, Callable
+
+
+# Allow small role-framing differences, not a repeated environment header.
+_INLINE_ROLE_SLACK_TOKENS = 8
+
+
+class InlineSystemError(ValueError):
+ """An input instruction cannot be placed without corrupting tool ordering."""
+
+
+def has_inline_system(messages: list[dict]) -> bool:
+ started = False
+ for message in messages:
+ if message.get("role") == "system":
+ if started:
+ return True
+ else:
+ started = True
+ return False
+
+
+def _add_text(message: dict, text: str, *, prepend: bool = False) -> dict:
+ result = dict(message)
+ content = result.get("content")
+ def join(a, b):
+ return "\n\n".join(part for part in (a, b) if part)
+ if content is None or isinstance(content, str):
+ result["content"] = join(text, content) if prepend else join(content, text)
+ elif isinstance(content, list):
+ parts = deepcopy(content)
+ index = 0 if prepend else -1
+ if parts and parts[index].get("type") == "text":
+ previous = parts[index].get("text") or ""
+ parts[index]["text"] = join(text, previous) if prepend else join(previous, text)
+ else:
+ parts.insert(0 if prepend else len(parts), {"type": "text", "text": text})
+ result["content"] = parts
+ else:
+ raise InlineSystemError("inline system folding requires text or content blocks")
+ return result
+
+
+def _after_tool_results(messages: list[dict]) -> list[dict]:
+ out, held = [], []
+ pending_ids: set[str] = set()
+ for message in messages:
+ role = message.get("role")
+ if role == "system" and pending_ids:
+ held.append(message)
+ continue
+ if role == "tool":
+ out.append(message)
+ pending_ids.discard(message.get("tool_call_id", ""))
+ if not pending_ids:
+ out.extend(held)
+ held.clear()
+ continue
+ if pending_ids and held:
+ raise InlineSystemError("inline system instruction interrupts incomplete tool results")
+ out.append(message)
+ pending_ids = {call.get("id", "") for call in message.get("tool_calls", [])} if role == "assistant" else set()
+ if held:
+ raise InlineSystemError("inline system instruction is waiting for missing tool results")
+ return out
+
+
+def normalize_inline_system(messages: list[dict], mode: str) -> list[dict]:
+ if mode not in ("preserve", "fold"):
+ raise InlineSystemError(f"invalid inline system mode: {mode}")
+ if not has_inline_system(messages):
+ return messages
+ ordered = _after_tool_results(deepcopy(messages))
+ # Without a stable leading system, some renderers attach tools to the first
+ # late system message. Folding avoids moving tools when that message changes.
+ if mode == "preserve" and ordered[0].get("role") == "system":
+ return ordered
+ out, pending = [], []
+ started = False
+ for message in ordered:
+ role = message.get("role")
+ if role == "system" and started:
+ text = message.get("content") or ""
+ if not isinstance(text, str):
+ raise InlineSystemError("inline system instructions must be text")
+ if text:
+ if out and out[-1].get("role") in ("user", "tool") and not pending:
+ out[-1] = _add_text(out[-1], text)
+ else:
+ pending.append(text)
+ continue
+ if pending:
+ text = "\n\n".join(pending)
+ pending.clear()
+ if role == "user":
+ message = _add_text(message, text, prepend=True)
+ else:
+ out.append({"role": "user", "content": text})
+ out.append(message)
+ started = started or role != "system"
+ if pending:
+ out.append({"role": "user", "content": "\n\n".join(pending)})
+ return out
+
+
+def _insertion(base: list[int], new: list[int]) -> tuple[int, list[int]] | None:
+ start = 0
+ while start < min(len(base), len(new)) and base[start] == new[start]:
+ start += 1
+ suffix = 0
+ while suffix < min(len(base), len(new)) - start and base[-suffix-1] == new[-suffix-1]:
+ suffix += 1
+ if start + suffix != len(base):
+ return None
+ return start, new[start:len(new)-suffix]
+
+
+def probe_inline_system(render: Callable[[list[dict]], str], tokenizer: Any) -> tuple[str, str]:
+ """Conservatively require a distinct, prefix-stable system segment."""
+ note, reasoning = "Echo inline instruction", "Foxtrot previous reasoning"
+ system = {"role": "system", "content": "Alpha stable instructions"}
+ user = {"role": "user", "content": "Bravo first question"}
+ cases = [
+ ("middle", [system, user, {"role": "assistant", "content": "Charlie first answer"},
+ {"role": "user", "content": "Delta second question"}], 3),
+ ("trailing", [system, user], 2),
+ ("tool result", [system, user,
+ {"role": "assistant", "content": "", "reasoning_content": reasoning, "thinking": reasoning,
+ "reasoning": reasoning, "tool_calls": [{"id": "probe_call", "type": "function",
+ "function": {"name": "read", "arguments": {}}}]},
+ {"role": "tool", "tool_call_id": "probe_call", "content": "Golf tool output"}], 4),
+ ]
+ max_system_tokens = None
+ try:
+ special_ids = set(tokenizer.all_special_ids) | set(tokenizer.get_added_vocab().values())
+ if not special_ids:
+ return "fold", "renderer has no identifiable special tokens"
+ for name, messages, index in cases:
+ base_text = render(deepcopy(messages))
+ base = tokenizer.encode(base_text, add_special_tokens=False)
+ with_note = messages[:index] + [{"role": "system", "content": note}] + messages[index:]
+ new_text = render(deepcopy(with_note))
+ new = tokenizer.encode(new_text, add_special_tokens=False)
+ found = _insertion(base, new)
+ if found is None:
+ return "fold", f"{name}: other prompt tokens change"
+ position, segment = found
+ if index == len(messages) and position >= len(base):
+ return "fold", f"{name}: instruction follows the generation opening"
+ if tokenizer.decode(segment).count(note) != 1 or not special_ids.intersection(segment):
+ return "fold", f"{name}: no distinct role-marked instruction"
+ if (reasoning in base_text) != (reasoning in new_text):
+ return "fold", f"{name}: reasoning retention changes"
+ if name == "middle":
+ as_user = messages[:index] + [{"role": "user", "content": note}] + messages[index:]
+ user_tokens = tokenizer.encode(render(deepcopy(as_user)), add_special_tokens=False)
+ user_segment = _insertion(base, user_tokens)
+ if user_segment is None:
+ return "fold", "middle: cannot isolate a user instruction"
+ if user_segment[1] == segment:
+ return "fold", "system instructions render as ordinary user turns"
+ max_system_tokens = len(user_segment[1]) + _INLINE_ROLE_SLACK_TOKENS
+ if max_system_tokens is not None and len(segment) > max_system_tokens:
+ return "fold", f"{name}: system insertion overhead ({len(segment)} > {max_system_tokens} tokens)"
+ except Exception as exc:
+ return "fold", f"probe could not render: {type(exc).__name__}"
+ return "preserve", "renderer preserves distinct system turns and existing prompt tokens"
diff --git a/python/freetoken/tokenizer/tokenize.py b/python/freetoken/tokenizer/tokenize.py
index 758ffd4d..f4736652 100644
--- a/python/freetoken/tokenizer/tokenize.py
+++ b/python/freetoken/tokenizer/tokenize.py
@@ -4,6 +4,7 @@ import importlib.util
import json
import os
import threading
+from copy import deepcopy
from types import ModuleType
from typing import TYPE_CHECKING, Any, List
@@ -22,6 +23,7 @@ from .effort import (
probe_thinking_profile,
quantize_effort,
)
+from .inline_system import has_inline_system, normalize_inline_system, probe_inline_system
logger = init_logger(__name__)
@@ -58,6 +60,8 @@ class TokenizeManager:
self._thinking_profile: ThinkingProfile | None = None
self._effort_lock = threading.Lock()
self._logged_effort_maps: set[tuple[Any, str | None]] = set()
+ self._inline_system_lock = threading.Lock()
+ self._inline_system_modes: dict[str, str] = {}
def tokenize(self, msgs: List[TokenizeMsg]) -> List[UserMsg]:
results: List[UserMsg] = []
@@ -103,9 +107,30 @@ class TokenizeManager:
validation, count_tokens) must quantize identically."""
if not isinstance(msg.text, list):
return msg.text
- return self._render(
- msg.text, msg.tools, self._sanitize_effort(msg.chat_template_kwargs or {})
- )
+ kwargs = self._sanitize_effort(msg.chat_template_kwargs or {})
+ messages = msg.text
+ if msg.inline_system_policy is not None:
+ if msg.inline_system_policy not in ("auto", "preserve", "fold"):
+ raise ValueError(f"invalid inline system policy: {msg.inline_system_policy}")
+ if has_inline_system(messages):
+ mode = msg.inline_system_policy
+ if mode == "auto":
+ mode = self.inline_system_mode(msg.tools, kwargs)
+ messages = normalize_inline_system(messages, mode)
+ return self._render(messages, msg.tools, kwargs)
+
+ def inline_system_mode(self, tools: list[dict] | None, kwargs: dict[str, Any]) -> str:
+ key = json.dumps([tools, kwargs], sort_keys=True)
+ with self._inline_system_lock:
+ if key not in self._inline_system_modes:
+ mode, reason = probe_inline_system(
+ lambda messages: self._render(messages, deepcopy(tools), deepcopy(kwargs)), self.tokenizer
+ )
+ if len(self._inline_system_modes) >= 128:
+ self._inline_system_modes.pop(next(iter(self._inline_system_modes)))
+ self._inline_system_modes[key] = mode
+ logger.info("Anthropic inline system policy: %s (%s)", mode, reason)
+ return self._inline_system_modes[key]
def _render(
self,
diff --git a/tests/README.md b/tests/README.md
index 26dba633..be22c1fb 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -53,7 +53,10 @@ checkpoint is set:
| `FREETOKEN_GEMMA4_MODEL` | `models/test_gemma4_vision.py`, `tokenizer/test_mm_tokenize.py` — local Gemma-4 `gemma4` checkpoint (tower parity, boi/eoi expansion; the tokenizer test also takes the 12B `gemma4_unified` checkpoint) |
| `FREETOKEN_GEMMA4_UNIFIED_MODEL` | `models/test_gemma4_vision.py` — local gemma-4-12B `gemma4_unified` checkpoint (embedder parity) |
| `FREETOKEN_GLM53_MODEL` | `tokenizer/test_mm_tokenize.py` — local GLM-5.3-Flash checkpoint (image-token expansion) |
-| `FREETOKEN_MUSE_MODEL` | `models/test_muse_glimmer_vision.py`, `tokenizer/test_mm_tokenize.py` — local Muse-Glimmer-30B checkpoint (tower parity, image_start/image_end expansion) |
+| `FREETOKEN_DSV4_MODEL` | `tokenizer/test_tokenize.py` - local DSV4 checkpoint with its native `encoding/encoding_dsv4.py`; prefix stability and reminder retention, tokenizer only (no GPU or model weights loaded) |
+| `FREETOKEN_INLINE_GLM_MODEL` | `tokenizer/test_tokenize.py` - local GLM-5.3 tokenizer/template; inline-system preserve policy, prefix stability and reasoning retention (no weights loaded) |
+| `FREETOKEN_INLINE_QWEN_MODEL` | `tokenizer/test_tokenize.py` - local Qwen3.6 tokenizer/template; inline-system fold policy, prefix stability and reasoning retention (no weights loaded) |
+| `FREETOKEN_MUSE_MODEL` | `models/test_muse_glimmer_vision.py`, `tokenizer/test_mm_tokenize.py`, `tokenizer/test_tokenize.py` - local Muse-Glimmer-30B checkpoint (vision tests; tokenizer-only inline-system regressions) |
| `FREETOKEN_MINIMAX_M3_MODEL` | `tokenizer/test_mm_tokenize.py` — local MiniMax-M3 checkpoint (start/end expansion, token budget) |
| `FREETOKEN_TEST_MOE_CACHE_SIZE` | `e2e/test_aime.py` — >0 switches to the offload MoE backend with this cache size |
| `FREETOKEN_TEST_MEM_RATIO` | `e2e/test_aime.py` — offload-mode memory_ratio (default `0.9`) |
diff --git a/tests/server/test_anthropic_api.py b/tests/server/test_anthropic_api.py
index 327bfcd8..f382093f 100644
--- a/tests/server/test_anthropic_api.py
+++ b/tests/server/test_anthropic_api.py
@@ -231,10 +231,7 @@ def test_convert_thinking_replay_in_tool_loop():
assert spec.messages[2]["role"] == "tool"
-def test_convert_hoists_and_merges_system_messages():
- # Claude Code interleaves system messages mid-array; strict chat templates
- # (e.g. Qwen3.5: "System message must be at the beginning") require ONE system
- # message at the front. Merge top-level system + in-array system, hoist to front.
+def test_convert_preserves_mid_conversation_system_reminders():
req = AnthropicMessagesRequest.model_validate(
{
"model": "claude-x",
@@ -247,11 +244,69 @@ def test_convert_hoists_and_merges_system_messages():
],
}
)
+ original = req.model_dump()
spec = A.convert_anthropic_to_genspec(req, {})
- assert [m["role"] for m in spec.messages] == ["system", "user", "assistant"]
- assert sum(1 for m in spec.messages if m["role"] == "system") == 1
- assert "top-level sys" in spec.messages[0]["content"]
- assert "mid-stream sys" in spec.messages[0]["content"]
+ assert spec.messages == [
+ {"role": "system", "content": "top-level sys"},
+ {"role": "user", "content": "hello"},
+ {"role": "system", "content": "mid-stream sys"},
+ {"role": "assistant", "content": "hi"},
+ ]
+ assert req.model_dump() == original
+
+
+def test_convert_merges_only_leading_system_messages():
+ req = AnthropicMessagesRequest.model_validate({
+ "model": "claude-x", "max_tokens": 64,
+ "system": [{"type": "text", "text": "top-level sys"}],
+ "messages": [
+ {"role": "system", "content": "leading sys"},
+ {"role": "system", "content": [{"type": "text", "text": "second sys"}]},
+ {"role": "user", "content": "hello"},
+ {"role": "system", "content": [{"type": "text", "text": "turn reminder"}]},
+ {"role": "assistant", "content": "hi"},
+ {"role": "system", "content": ""},
+ {"role": "system", "content": "next reminder"},
+ ],
+ })
+ original = req.model_dump()
+ spec = A.convert_anthropic_to_genspec(req, {})
+ assert spec.messages == [
+ {"role": "system", "content": "top-level sys\n\nleading sys\n\nsecond sys"},
+ {"role": "user", "content": "hello"},
+ {"role": "system", "content": "turn reminder"},
+ {"role": "assistant", "content": "hi"},
+ {"role": "system", "content": "next reminder"},
+ ]
+ assert req.model_dump() == original
+
+
[email protected]("system", [None, "stable instructions"])
[email protected]("count_tokens", [False, True])
+def test_appending_system_reminder_preserves_converted_prefix(system, count_tokens):
+ from freetoken.server.anthropic_models import AnthropicCountTokensRequest
+
+ data = {
+ "model": "claude-x",
+ "messages": [
+ {"role": "user", "content": "inspect the source"},
+ {"role": "assistant", "content": "I found the function"},
+ ],
+ "tools": [{"name": "read", "input_schema": {"type": "object", "properties": {}}}],
+ }
+ if system is not None:
+ data["system"] = system
+ if not count_tokens:
+ data["max_tokens"] = 64
+ request_type = AnthropicCountTokensRequest if count_tokens else AnthropicMessagesRequest
+ first, first_tools, _, _ = A.convert_anthropic_prompt(request_type.model_validate(data))
+ data["messages"].append({"role": "system", "content": "<total_tokens>1000 tokens left</total_tokens>"})
+ second, second_tools, _, _ = A.convert_anthropic_prompt(request_type.model_validate(data))
+ assert second[:-1] == first
+ assert second[-1] == {
+ "role": "system", "content": "<total_tokens>1000 tokens left</total_tokens>"
+ }
+ assert second_tools == first_tools
# --------------------------------------------------------------------------- #
@@ -664,6 +719,7 @@ def test_count_tokens_route():
assert msg.text == spec.messages
assert msg.tools == spec.template_tools
assert msg.chat_template_kwargs == spec.chat_template_kwargs
+ assert msg.inline_system_policy == spec.inline_system_policy == "auto"
def test_count_tokens_empty_messages_400():
@@ -673,6 +729,28 @@ def test_count_tokens_empty_messages_400():
assert r.json()["error"]["type"] == "invalid_request_error"
[email protected]("policy", ["auto", "preserve", "fold"])
+def test_inline_system_override_reaches_counting_and_generation(policy):
+ manager = _FakeTokenizeManager()
+ fake = FakeState([("ok", True, 3, 1)])
+ fake._count_manager = manager
+ fake.config.anthropic_inline_system = policy
+ submitted = []
+
+ async def record(msg):
+ submitted.append(msg)
+
+ fake.send_one = record
+ client = _client(fake)
+ body = {**_COUNT_BODY, "messages": [
+ {"role": "user", "content": "question"}, {"role": "system", "content": "instruction"}]}
+ assert client.post("/v1/messages/count_tokens", json=body).status_code == 200
+ assert client.post("/v1/messages", json={**body, "max_tokens": 32}).status_code == 200
+ assert manager.msgs[0].inline_system_policy == submitted[0].inline_system_policy == policy
+ assert manager.msgs[0].text == submitted[0].text
+ assert submitted[0].text[0]["content"] == body["system"]
+
+
def test_count_tokens_works_while_not_serving():
# Counting never touches the engine, so it must not 503 during load/rebuild.
client, _ = _count_client(_FakeTokenizeManager(), maintenance_state="rebuilding")
@@ -719,6 +797,15 @@ def test_count_tokens_template_render_error_400():
assert r.json()["error"]["type"] == "invalid_request_error"
+def test_count_tokens_inline_system_placement_error_is_400():
+ from freetoken.tokenizer.inline_system import InlineSystemError
+
+ client, _ = _count_client(_RaisingManager(InlineSystemError("missing tool results")))
+ r = client.post("/v1/messages/count_tokens", json=_COUNT_BODY)
+ assert r.status_code == 400
+ assert r.json()["error"]["type"] == "invalid_request_error"
+
+
def test_count_tokens_excluded_from_request_ring():
# count_tokens never enters generation accounting; its latency (incl. first-touch tokenizer
# load) must not land in the /v1/requests ring or the /v1/stats p95.
diff --git a/tests/server/test_message_wire.py b/tests/server/test_message_wire.py
index e2286671..fdc09592 100644
--- a/tests/server/test_message_wire.py
+++ b/tests/server/test_message_wire.py
@@ -28,6 +28,18 @@ from freetoken.message import (
from freetoken.core import SamplingParams
+def test_inline_system_policy_tokenizer_roundtrip():
+ for policy in (None, "auto", "preserve", "fold"):
+ msg = TokenizeMsg(uid=1, text=[{"role": "system", "content": "instruction"}],
+ sampling_params=SamplingParams(), inline_system_policy=policy)
+ decoded = BaseTokenizerMsg.decoder(BaseTokenizerMsg.encoder(msg))
+ assert decoded.inline_system_policy == policy
+ assert decoded.text == msg.text
+ legacy = BaseTokenizerMsg.encoder(msg)
+ legacy.pop("inline_system_policy")
+ assert BaseTokenizerMsg.decoder(legacy).inline_system_policy is None
+
+
def test_cache_rebuild_msg_roundtrip():
msg = CacheRebuildMsg(request_id="abc", moe_cache_size=8, num_pages=1024, mode="if_idle")
out = BaseTokenizerMsg.decoder(BaseTokenizerMsg.encoder(msg))
diff --git a/tests/tokenizer/test_tokenize.py b/tests/tokenizer/test_tokenize.py
index 369576a1..16ef6d74 100644
--- a/tests/tokenizer/test_tokenize.py
+++ b/tests/tokenizer/test_tokenize.py
@@ -1,6 +1,8 @@
from __future__ import annotations
import json
+import os
+from copy import deepcopy
import pytest
import torch
@@ -8,14 +10,303 @@ import torch
from freetoken.core import SamplingParams
from freetoken.message import TokenizeMsg
from freetoken.tokenizer.tokenize import TokenizeManager, _dsv4_arguments_str
+from freetoken.tokenizer.inline_system import InlineSystemError, normalize_inline_system, probe_inline_system
+
+
+def test_fold_keeps_leading_system_and_appends_text_without_mutation():
+ messages = [
+ {"role": "system", "content": "stable"},
+ {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "image"}},
+ {"type": "text", "text": "question"}]},
+ {"role": "system", "content": "instruction 1"},
+ {"role": "system", "content": "instruction 2"},
+ ]
+ original = deepcopy(messages)
+ result = normalize_inline_system(messages, "fold")
+ assert result == [messages[0], {"role": "user", "content": [
+ messages[1]["content"][0], {"type": "text", "text": "question\n\ninstruction 1\n\ninstruction 2"}]}]
+ assert messages == original
+
+
[email protected]("mode", ["preserve", "fold"])
+def test_inline_system_waits_for_all_parallel_tool_results(mode):
+ messages = [
+ {"role": "system", "content": "stable"}, {"role": "user", "content": "question"},
+ {"role": "assistant", "reasoning_content": "reason", "tool_calls": [{"id": "a"}, {"id": "b"}]},
+ {"role": "system", "content": "before results"},
+ {"role": "tool", "tool_call_id": "a", "content": "result a"},
+ {"role": "system", "content": "between results"},
+ {"role": "tool", "tool_call_id": "b", "content": "result b"},
+ {"role": "system", "content": "after results"},
+ ]
+ original = deepcopy(messages)
+ result = normalize_inline_system(messages, mode)
+ assert [m["tool_call_id"] for m in result if m["role"] == "tool"] == ["a", "b"]
+ assert result[2] == messages[2]
+ assert [m["role"] for m in result[3:5]] == ["tool", "tool"]
+ if mode == "fold":
+ assert len(result) == 5
+ assert result[-1]["content"] == "result b\n\nbefore results\n\nbetween results\n\nafter results"
+ else:
+ assert [m["content"] for m in result[5:]] == ["before results", "between results", "after results"]
+ assert messages == original
+
+
[email protected]("mode", ["preserve", "fold"])
+def test_inline_system_does_not_hide_missing_tool_results(mode):
+ messages = [{"role": "assistant", "tool_calls": [{"id": "missing"}]},
+ {"role": "system", "content": "instruction"}]
+ with pytest.raises(InlineSystemError, match="missing tool results"):
+ normalize_inline_system(messages, mode)
+
+
+def test_fold_after_assistant_prepends_next_user_or_supplies_a_user_turn():
+ messages = [{"role": "user", "content": "first"}, {"role": "assistant", "content": "answer"},
+ {"role": "system", "content": "instruction"}]
+ assert normalize_inline_system(messages, "fold")[-1] == {"role": "user", "content": "instruction"}
+ result = normalize_inline_system(messages + [{"role": "user", "content": "next"}], "fold")
+ assert result == messages[:2] + [{"role": "user", "content": "instruction\n\nnext"}]
+
+
+def test_preserve_keeps_instruction_role_and_only_folds_without_a_stable_system():
+ messages = [{"role": "system", "content": "stable"}, {"role": "user", "content": "question"},
+ {"role": "system", "content": "late"}]
+ assert normalize_inline_system(messages, "preserve") == messages
+ assert normalize_inline_system(messages[1:], "preserve") == [{"role": "user", "content": "question\n\nlate"}]
+
+
+class ProbeTokenizer:
+ vocabulary = {text: 10000+i for i, text in enumerate(
+ ["<system>", "</system>", "<user>", "</user>", "<assistant>", "</assistant>",
+ "<tool>", "</tool>", "<think>", "</think>"])}
+ all_special_ids = list(vocabulary.values())
+
+ def get_added_vocab(self):
+ return self.vocabulary
+
+ def encode(self, text, **kwargs):
+ import re
+ parts = re.split("(" + "|".join(re.escape(x) for x in self.vocabulary) + ")", text)
+ return [token for part in parts for token in ([self.vocabulary[part]] if part in self.vocabulary else map(ord, part))]
+
+ def decode(self, ids):
+ inverse = {v: k for k, v in self.vocabulary.items()}
+ return "".join(inverse[x] if x in inverse else chr(x) for x in ids)
+
+
[email protected]("behavior,expected", [("preserve", "preserve"), ("reject", "fold"),
+ ("drop", "fold"), ("bare", "fold"), ("user", "fold"), ("rewrite", "fold"),
+ ("after_generation", "fold"), ("drop_reasoning", "fold")])
+def test_renderer_probe_checks_more_than_template_acceptance(behavior, expected):
+ def render(messages):
+ inline = any(m["role"] == "system" for m in messages[1:])
+ output, late = "", ""
+ for i, message in enumerate(messages):
+ role, content = message["role"], message.get("content", "")
+ if role == "system" and i:
+ if behavior == "reject":
+ raise ValueError("system first only")
+ if behavior == "drop":
+ continue
+ if behavior == "bare":
+ output += content
+ continue
+ if behavior == "user":
+ role = "user"
+ if behavior == "after_generation":
+ late += content
+ continue
+ if behavior == "rewrite" and inline and i == 0:
+ content = "changed head"
+ reason = message.get("reasoning_content")
+ if reason and not (behavior == "drop_reasoning" and inline):
+ content = "<think>" + reason + "</think>" + content
+ output += f"<{role}>{content}</{role}>"
+ return output + "<assistant>" + late
+ assert probe_inline_system(render, ProbeTokenizer())[0] == expected
+
+
[email protected]("position", ["middle", "trailing", "tool result"])
[email protected]("metadata", ["Reasoning strength: high. Valid recipients: self, user.", "tool schema " * 1000],
+ ids=["metadata", "tools"])
+def test_inline_probe_rejects_repeated_system_metadata(position, metadata):
+ def render(messages):
+ output = ""
+ for i, message in enumerate(messages):
+ role = message["role"]
+ content = message.get("content", "")
+ if role == "system" and i:
+ if i < len(messages)-1:
+ location = "middle"
+ else:
+ location = "tool result" if messages[i-1]["role"] == "tool" else "trailing"
+ if location == position:
+ content += metadata
+ content = message.get("reasoning_content", "") + content
+ output += f"<{role}>{content}</{role}>"
+ return output + "<assistant>"
+
+ mode, reason = probe_inline_system(render, ProbeTokenizer())
+ assert mode == "fold"
+ assert "overhead" in reason
+
+
+def test_inline_probe_allows_small_role_framing_overhead():
+ def render(messages):
+ return "".join(f'<{m["role"]}>' + ("role: " if m["role"] == "system" else "")
+ + m.get("reasoning_content", "") + m.get("content", "")
+ + f'</{m["role"]}>' for m in messages) + "<assistant>"
+
+ assert probe_inline_system(render, ProbeTokenizer())[0] == "preserve"
+
+
+def test_inline_probe_folds_when_user_insertion_cannot_be_measured():
+ def render(messages):
+ rewrite = any(m["role"] == "user" and m.get("content") == "Echo inline instruction" for m in messages)
+ return ("changed head" if rewrite else "stable head") + "".join(
+ f'<{m["role"]}>' + m.get("reasoning_content", "") + m.get("content", "")
+ + f'</{m["role"]}>' for m in messages) + "<assistant>"
+
+ mode, reason = probe_inline_system(render, ProbeTokenizer())
+ assert mode == "fold"
+ assert "cannot isolate" in reason
+
+
+def test_inline_probe_cache_is_scoped_to_tools_and_render_kwargs(monkeypatch):
+ import freetoken.tokenizer.tokenize as module
+ calls = []
+ monkeypatch.setattr(module, "probe_inline_system", lambda *_: (calls.append(1) or ("fold", "test")))
+ manager = TokenizeManager(FakeTokenizer())
+ for _ in range(3):
+ assert manager.inline_system_mode(None, {}) == "fold"
+ manager.inline_system_mode([], {})
+ manager.inline_system_mode(None, {"enable_thinking": True})
+ assert len(calls) == 3
+
+
+def test_inline_policy_is_opt_in_and_not_forwarded_to_templates():
+ tokenizer = FakeTokenizer()
+ manager = TokenizeManager(tokenizer)
+ messages = [{"role": "user", "content": "question"}, {"role": "system", "content": "instruction"}]
+ msg = TokenizeMsg(uid=1, text=messages, sampling_params=SamplingParams())
+ manager.render_prompt(msg)
+ assert tokenizer.chat_template_messages == messages
+ msg.inline_system_policy = "fold"
+ manager.render_prompt(msg)
+ assert tokenizer.chat_template_messages == [{"role": "user", "content": "question\n\ninstruction"}]
+ assert "inline_system_policy" not in tokenizer.chat_template_kwargs
+
+
[email protected]("variable,expected_mode", [
+ ("FREETOKEN_DSV4_MODEL", "fold"),
+ ("FREETOKEN_INLINE_GLM_MODEL", "preserve"),
+ ("FREETOKEN_INLINE_QWEN_MODEL", "fold"),
+])
+def test_native_model_inline_system_tool_history(variable, expected_mode):
+ from transformers import AutoTokenizer
+ from freetoken.server.anthropic_api import convert_anthropic_to_genspec
+ from freetoken.server.anthropic_models import AnthropicMessagesRequest
+
+ path = os.environ.get(variable)
+ if not path:
+ pytest.skip(f"set {variable} to a local checkpoint")
+ manager = TokenizeManager(AutoTokenizer.from_pretrained(path, local_files_only=True))
+ data = {
+ "model": "local", "max_tokens": 64, "system": "Stable project instructions.",
+ "thinking": {"type": "enabled"},
+ "tools": [{"name": "read", "input_schema": {"type": "object", "properties": {}}}],
+ "messages": [
+ {"role": "user", "content": "Inspect source:\n" + "value = 1\n" * 512},
+ {"role": "system", "content": "Keep all existing tests."},
+ ],
+ }
+ ids, prompts = [], []
+ for turn in (1, 2):
+ data["messages"].extend([
+ {"role": "assistant", "content": [
+ {"type": "thinking", "thinking": f"Retained thought {turn}.", "signature": ""},
+ {"type": "tool_use", "id": f"read_{turn}", "name": "read", "input": {}},
+ ]},
+ {"role": "system", "content": f"During tool call {turn}."},
+ {"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": f"read_{turn}", "content": f"Source result {turn}."},
+ ]},
+ {"role": "system", "content": f"Remaining budget: {1000-turn}."},
+ ])
+ req = AnthropicMessagesRequest.model_validate(data)
+ spec = convert_anthropic_to_genspec(req, {})
+ assert manager.inline_system_mode(spec.template_tools, spec.chat_template_kwargs) == expected_mode
+ original = deepcopy(spec.messages)
+ msg = TokenizeMsg(uid=turn, text=spec.messages, tools=spec.template_tools,
+ sampling_params=spec.sampling_params, chat_template_kwargs=spec.chat_template_kwargs,
+ inline_system_policy=spec.inline_system_policy)
+ prompts.append(manager.render_prompt(msg))
+ ids.append(manager.tokenize([msg])[0].input_ids.tolist())
+ assert spec.messages == original
+ assert prompts[-1].count("Keep all existing tests.") == 1
+ assert "Retained thought 1." in prompts[-1]
+ for previous in range(1, turn+1):
+ assert prompts[-1].count(f"During tool call {previous}.") == 1
+ assert prompts[-1].count(f"Remaining budget: {1000-previous}.") == 1
+ common = next((i for i, pair in enumerate(zip(*ids)) if pair[0] != pair[1]), min(map(len, ids)))
+ assert common >= len(ids[0]) - 64
+ assert common > 2048
+
+
[email protected]("tool_count", [0, 20])
+def test_native_muse_inline_system_does_not_repeat_environment(tool_count):
+ from transformers import AutoTokenizer
+
+ path = os.environ.get("FREETOKEN_MUSE_MODEL")
+ if not path:
+ pytest.skip("set FREETOKEN_MUSE_MODEL to a local checkpoint")
+ manager = TokenizeManager(AutoTokenizer.from_pretrained(path, local_files_only=True))
+ tools = [{"type": "function", "function": {"name": f"read_{i}",
+ "description": "Inspect the repository source. " * 100,
+ "parameters": {"type": "object", "properties": {}}}} for i in range(tool_count)] or None
+ messages = [{"role": "system", "content": "Stable instructions."},
+ {"role": "user", "content": "Inspect this file.\n" + "value = 1\n" * 256}]
+ assert manager.inline_system_mode(tools, {}) == "fold"
+ for turn in range(3):
+ messages.extend([{"role": "assistant", "content": f"Answer {turn}."},
+ {"role": "user", "content": f"Question {turn}."},
+ {"role": "system", "content": f"Budget reminder {turn}."}])
+ original = deepcopy(messages)
+ msg = TokenizeMsg(uid=turn, text=messages, tools=tools, sampling_params=SamplingParams(),
+ inline_system_policy="auto")
+ prompt = manager.render_prompt(msg)
+ assert prompt.count("Reasoning strength:") == 1
+ assert prompt.count("# Valid recipients:") == 1
+ assert prompt.count("// Function schemas") == bool(tools)
+ for previous in range(turn+1):
+ assert prompt.count(f"Budget reminder {previous}.") == 1
+ ids = manager.tokenize([msg])[0].input_ids.tolist()
+ msg.inline_system_policy = "fold"
+ assert ids == manager.tokenize([msg])[0].input_ids.tolist()
+ assert messages == original
+
+ latest = [m for i, m in enumerate(messages) if m["role"] != "system" or i in (0, len(messages)-1)]
+ updated = deepcopy(latest)
+ updated[-1]["content"] = "Budget reminder 900."
+ tokens = []
+ for conversation in (latest, updated):
+ msg = TokenizeMsg(uid=0, text=conversation, tools=tools, sampling_params=SamplingParams(),
+ inline_system_policy="auto")
+ tokens.append(manager.tokenize([msg])[0].input_ids.tolist())
+ common = next((i for i, pair in enumerate(zip(*tokens)) if pair[0] != pair[1]), min(map(len, tokens)))
+ assert common >= min(map(len, tokens)) - 16
class FakeTokenizer:
def __init__(self) -> None:
self.chat_template_kwargs = None
+ self.chat_template_messages = None
def apply_chat_template(self, messages, **kwargs):
self.chat_template_kwargs = kwargs
+ self.chat_template_messages = deepcopy(messages)
return "rendered prompt"
def encode(self, prompt, return_tensors=None, add_special_tokens=True):
@@ -74,6 +365,31 @@ def test_tokenize_manager_passes_tools_to_chat_template():
assert input_ids.tolist() == [1, 2, 3]
[email protected]("initial_system", [[], [{"role": "system", "content": "stable instructions"}]])
+def test_generic_template_folds_system_messages_without_mutating_input(initial_system):
+ tokenizer = FakeTokenizer()
+ manager = TokenizeManager(tokenizer)
+ messages = initial_system + [
+ {"role": "user", "content": "inspect files"},
+ {"role": "system", "content": "first reminder"},
+ {"role": "assistant", "content": "found the bug"},
+ {"role": "system", "content": "second reminder"},
+ ]
+ original = deepcopy(messages)
+ tools = [{"type": "function", "function": {"name": "read", "parameters": {"type": "object"}}}]
+ original_tools = deepcopy(tools)
+ manager.tokenize([TokenizeMsg(uid=1, text=messages, tools=tools, sampling_params=SamplingParams(),
+ inline_system_policy="auto")])
+ assert tokenizer.chat_template_messages == initial_system + [
+ {"role": "user", "content": "inspect files\n\nfirst reminder"},
+ {"role": "assistant", "content": "found the bug"},
+ {"role": "user", "content": "second reminder"},
+ ]
+ assert tokenizer.chat_template_kwargs["tools"] == original_tools
+ assert messages == original
+ assert tools == original_tools
+
+
class FakeDsv4Tokenizer:
chat_template = None
@@ -126,6 +442,166 @@ def encode_messages(messages, thinking_mode, reasoning_effort=None):
assert input_ids.tolist() == [4, 5, 6]
[email protected]("initial_system", [[], [{"role": "system", "content": "stable instructions"}]])
+def test_dsv4_fold_keeps_instructions_and_tools_on_stable_prefix(tmp_path, initial_system):
+ encoding_dir = tmp_path / "encoding"
+ encoding_dir.mkdir()
+ (encoding_dir / "encoding_dsv4.py").write_text(
+ "import json\n"
+ "def encode_messages(messages, thinking_mode, reasoning_effort=None):\n"
+ " return json.dumps(messages)\n"
+ )
+ tokenizer = FakeDsv4Tokenizer(tmp_path)
+ manager = TokenizeManager(tokenizer)
+ tools = [{"type": "function", "function": {"name": "read", "parameters": {"type": "object"}}}]
+ messages = initial_system + [
+ {"role": "user", "content": "inspect files"},
+ {"role": "system", "content": "first reminder"},
+ {"role": "assistant", "content": "found the bug"},
+ ]
+ original, original_tools = deepcopy(messages), deepcopy(tools)
+ manager.tokenize([TokenizeMsg(uid=1, text=messages, tools=tools, sampling_params=SamplingParams(),
+ inline_system_policy="fold")])
+ first = json.loads(tokenizer.prompt)
+ extended = messages + [{"role": "system", "content": "next reminder"}]
+ original_extended = deepcopy(extended)
+ manager.tokenize([TokenizeMsg(uid=2, text=extended, tools=tools, sampling_params=SamplingParams(),
+ inline_system_policy="fold")])
+ second = json.loads(tokenizer.prompt)
+ assert [m["role"] for m in first] == ["system", "user", "assistant"]
+ assert first[0]["content"] == (initial_system[0]["content"] if initial_system else "")
+ assert first[0]["tools"] == original_tools
+ assert first[1]["content"] == "inspect files\n\nfirst reminder"
+ assert second[:-1] == first
+ assert second[-1] == {"role": "user", "content": "next reminder"}
+ assert all("tools" not in m for m in second[1:])
+ assert messages == original and extended == original_extended and tools == original_tools
+
+
[email protected](scope="module")
+def native_dsv4_manager():
+ model_path = os.environ.get("FREETOKEN_DSV4_MODEL")
+ if not model_path:
+ pytest.skip("set FREETOKEN_DSV4_MODEL to a local DSV4 checkpoint")
+ from transformers import AutoTokenizer
+
+ manager = TokenizeManager(AutoTokenizer.from_pretrained(model_path, local_files_only=True))
+ assert manager._dsv4_encoder is not None, "checkpoint must supply the native DSV4 encoder"
+ return manager
+
+
+def test_dsv4_no_tools_retains_reminders_when_encoder_drops_developer_turns(tmp_path):
+ encoding_dir = tmp_path / "encoding"
+ encoding_dir.mkdir()
+ (encoding_dir / "encoding_dsv4.py").write_text(
+ "import json\n"
+ "def encode_messages(messages, thinking_mode, reasoning_effort=None):\n"
+ " if thinking_mode == 'thinking':\n"
+ " messages = [m for m in messages if m['role'] != 'developer']\n"
+ " return json.dumps(messages)\n"
+ )
+ manager = TokenizeManager(FakeDsv4Tokenizer(tmp_path))
+ messages = [
+ {"role": "user", "content": "First task."},
+ {"role": "system", "content": "Keep this earlier instruction."},
+ {"role": "assistant", "content": "First answer."},
+ {"role": "user", "content": "Second task."},
+ ]
+ original = deepcopy(messages)
+ msg = TokenizeMsg(uid=1, text=messages, sampling_params=SamplingParams(),
+ chat_template_kwargs={"enable_thinking": True}, inline_system_policy="fold")
+ rendered = json.loads(manager.render_prompt(msg))
+ assert all(m["role"] in ("system", "user", "assistant") for m in rendered)
+ assert sum(m["content"].count("Keep this earlier instruction.") for m in rendered) == 1
+ assert messages == original
+
+
[email protected]("system", [None, "You are a coding assistant."])
[email protected]("change", ["update", "append"])
+def test_native_dsv4_reminder_update_preserves_token_prefix(native_dsv4_manager, system, change):
+ from freetoken.server.anthropic_api import convert_anthropic_to_genspec
+ from freetoken.server.anthropic_models import AnthropicMessagesRequest
+
+ data = {
+ "model": "dsv4", "max_tokens": 64,
+ "tools": [{"name": "read", "input_schema": {"type": "object", "properties": {}}}],
+ "messages": [
+ {"role": "user", "content": "Inspect this file:\n" + "value = 1\n" * 512},
+ {"role": "system", "content": "Keep the changes minimal."},
+ {"role": "assistant", "content": [
+ {"type": "thinking", "thinking": "I should read the source.", "signature": ""},
+ {"type": "tool_use", "id": "call_read", "name": "read", "input": {}},
+ ]},
+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_read", "content": "value = 1"}]},
+ {"role": "system", "content": "Remaining token budget: 1000"},
+ ],
+ }
+ if system is not None:
+ data["system"] = system
+ tokens = []
+ for budget in (1000, 900):
+ if budget == 900 and change == "append":
+ data["messages"].extend([
+ {"role": "assistant", "content": [
+ {"type": "thinking", "thinking": "The file is simple.", "signature": ""},
+ {"type": "text", "text": "I inspected the file."},
+ ]},
+ {"role": "system", "content": "Remaining token budget: 900"},
+ ])
+ else:
+ data["messages"][-1]["content"] = f"Remaining token budget: {budget}"
+ req = AnthropicMessagesRequest.model_validate(data)
+ original = req.model_dump()
+ spec = convert_anthropic_to_genspec(req, {})
+ msg = TokenizeMsg(uid=budget, text=spec.messages, tools=spec.template_tools,
+ sampling_params=spec.sampling_params, chat_template_kwargs=spec.chat_template_kwargs,
+ inline_system_policy=spec.inline_system_policy)
+ prompt = native_dsv4_manager.render_prompt(msg)
+ assert prompt.count("Keep the changes minimal.") == 1
+ assert prompt.index("Inspect this file:") < prompt.index(f"Remaining token budget: {budget}")
+ tokens.append(native_dsv4_manager.tokenize([msg])[0].input_ids.tolist())
+ assert req.model_dump() == original
+ common = next((i for i, pair in enumerate(zip(*tokens)) if pair[0] != pair[1]), min(map(len, tokens)))
+ assert common > 2048
+ assert common >= min(map(len, tokens)) - 64
+
+
[email protected]("disable_declared_tools", [False, True])
[email protected]("thinking_enabled", [False, True])
+def test_native_dsv4_no_tools_keeps_earlier_reminder(
+ native_dsv4_manager, disable_declared_tools, thinking_enabled
+):
+ from freetoken.server.anthropic_api import convert_anthropic_to_genspec
+ from freetoken.server.anthropic_models import AnthropicMessagesRequest
+
+ data = {
+ "model": "dsv4", "max_tokens": 64, "system": "Stable instructions.",
+ "thinking": {"type": "enabled" if thinking_enabled else "disabled"},
+ "messages": [
+ {"role": "user", "content": "First task."},
+ {"role": "system", "content": "Keep this earlier instruction."},
+ {"role": "assistant", "content": "First answer."},
+ {"role": "user", "content": "Second task."},
+ ],
+ }
+ if disable_declared_tools:
+ data["tools"] = [{"name": "read", "input_schema": {"type": "object", "properties": {}}}]
+ data["tool_choice"] = {"type": "none"}
+ req = AnthropicMessagesRequest.model_validate(data)
+ original = req.model_dump()
+ spec = convert_anthropic_to_genspec(req, {})
+ assert not spec.template_tools
+ msg = TokenizeMsg(uid=1, text=spec.messages, tools=spec.template_tools,
+ sampling_params=spec.sampling_params, chat_template_kwargs=spec.chat_template_kwargs,
+ inline_system_policy=spec.inline_system_policy)
+ prompt = native_dsv4_manager.render_prompt(msg)
+ assert prompt.count("Keep this earlier instruction.") == 1
+ assert "Second task." in prompt
+ assert req.model_dump() == original
+
+
def test_dsv4_encoder_gets_tool_call_arguments_as_json_string(tmp_path):
"""Regression: render_messages hands the template dict arguments; the dsv4
encoder contract is a JSON-object STRING -- a dict trips its fallback that