Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

7.7 KB
Raw Blame
# SPDX-License-Identifier: Apache-2.0
# Renderer-probe approach adapted from vLLM PR #58772.
"""Preserve or fold late system instructions without hoisting the prompt head."""
from __future__ import annotations

from copy import deepcopy
from typing import Any, Callable


# Allow small role-framing differences, not a repeated environment header.
_INLINE_ROLE_SLACK_TOKENS = 8


class InlineSystemError(ValueError):
    """An input instruction cannot be placed without corrupting tool ordering."""


def has_inline_system(messages: list[dict]) -> bool:
    started = False
    for message in messages:
        if message.get("role") == "system":
            if started:
                return True
        else:
            started = True
    return False


def _add_text(message: dict, text: str, *, prepend: bool = False) -> dict:
    result = dict(message)
    content = result.get("content")
    def join(a, b):
        return "\n\n".join(part for part in (a, b) if part)
    if content is None or isinstance(content, str):
        result["content"] = join(text, content) if prepend else join(content, text)
    elif isinstance(content, list):
        parts = deepcopy(content)
        index = 0 if prepend else -1
        if parts and parts[index].get("type") == "text":
            previous = parts[index].get("text") or ""
            parts[index]["text"] = join(text, previous) if prepend else join(previous, text)
        else:
            parts.insert(0 if prepend else len(parts), {"type": "text", "text": text})
        result["content"] = parts
    else:
        raise InlineSystemError("inline system folding requires text or content blocks")
    return result


def _after_tool_results(messages: list[dict]) -> list[dict]:
    out, held = [], []
    pending_ids: set[str] = set()
    for message in messages:
        role = message.get("role")
        if role == "system" and pending_ids:
            held.append(message)
            continue
        if role == "tool":
            out.append(message)
            pending_ids.discard(message.get("tool_call_id", ""))
            if not pending_ids:
                out.extend(held)
                held.clear()
            continue
        if pending_ids and held:
            raise InlineSystemError("inline system instruction interrupts incomplete tool results")
        out.append(message)
        pending_ids = {call.get("id", "") for call in message.get("tool_calls", [])} if role == "assistant" else set()
    if held:
        raise InlineSystemError("inline system instruction is waiting for missing tool results")
    return out


def normalize_inline_system(messages: list[dict], mode: str) -> list[dict]:
    if mode not in ("preserve", "fold"):
        raise InlineSystemError(f"invalid inline system mode: {mode}")
    if not has_inline_system(messages):
        return messages
    ordered = _after_tool_results(deepcopy(messages))
    # Without a stable leading system, some renderers attach tools to the first
    # late system message. Folding avoids moving tools when that message changes.
    if mode == "preserve" and ordered[0].get("role") == "system":
        return ordered
    out, pending = [], []
    started = False
    for message in ordered:
        role = message.get("role")
        if role == "system" and started:
            text = message.get("content") or ""
            if not isinstance(text, str):
                raise InlineSystemError("inline system instructions must be text")
            if text:
                if out and out[-1].get("role") in ("user", "tool") and not pending:
                    out[-1] = _add_text(out[-1], text)
                else:
                    pending.append(text)
            continue
        if pending:
            text = "\n\n".join(pending)
            pending.clear()
            if role == "user":
                message = _add_text(message, text, prepend=True)
            else:
                out.append({"role": "user", "content": text})
        out.append(message)
        started = started or role != "system"
    if pending:
        out.append({"role": "user", "content": "\n\n".join(pending)})
    return out


def _insertion(base: list[int], new: list[int]) -> tuple[int, list[int]] | None:
    start = 0
    while start < min(len(base), len(new)) and base[start] == new[start]:
        start += 1
    suffix = 0
    while suffix < min(len(base), len(new)) - start and base[-suffix-1] == new[-suffix-1]:
        suffix += 1
    if start + suffix != len(base):
        return None
    return start, new[start:len(new)-suffix]


def probe_inline_system(render: Callable[[list[dict]], str], tokenizer: Any) -> tuple[str, str]:
    """Conservatively require a distinct, prefix-stable system segment."""
    note, reasoning = "Echo inline instruction", "Foxtrot previous reasoning"
    system = {"role": "system", "content": "Alpha stable instructions"}
    user = {"role": "user", "content": "Bravo first question"}
    cases = [
        ("middle", [system, user, {"role": "assistant", "content": "Charlie first answer"},
                    {"role": "user", "content": "Delta second question"}], 3),
        ("trailing", [system, user], 2),
        ("tool result", [system, user,
            {"role": "assistant", "content": "", "reasoning_content": reasoning, "thinking": reasoning,
             "reasoning": reasoning, "tool_calls": [{"id": "probe_call", "type": "function",
                "function": {"name": "read", "arguments": {}}}]},
            {"role": "tool", "tool_call_id": "probe_call", "content": "Golf tool output"}], 4),
    ]
    max_system_tokens = None
    try:
        special_ids = set(tokenizer.all_special_ids) | set(tokenizer.get_added_vocab().values())
        if not special_ids:
            return "fold", "renderer has no identifiable special tokens"
        for name, messages, index in cases:
            base_text = render(deepcopy(messages))
            base = tokenizer.encode(base_text, add_special_tokens=False)
            with_note = messages[:index] + [{"role": "system", "content": note}] + messages[index:]
            new_text = render(deepcopy(with_note))
            new = tokenizer.encode(new_text, add_special_tokens=False)
            found = _insertion(base, new)
            if found is None:
                return "fold", f"{name}: other prompt tokens change"
            position, segment = found
            if index == len(messages) and position >= len(base):
                return "fold", f"{name}: instruction follows the generation opening"
            if tokenizer.decode(segment).count(note) != 1 or not special_ids.intersection(segment):
                return "fold", f"{name}: no distinct role-marked instruction"
            if (reasoning in base_text) != (reasoning in new_text):
                return "fold", f"{name}: reasoning retention changes"
            if name == "middle":
                as_user = messages[:index] + [{"role": "user", "content": note}] + messages[index:]
                user_tokens = tokenizer.encode(render(deepcopy(as_user)), add_special_tokens=False)
                user_segment = _insertion(base, user_tokens)
                if user_segment is None:
                    return "fold", "middle: cannot isolate a user instruction"
                if user_segment[1] == segment:
                    return "fold", "system instructions render as ordinary user turns"
                max_system_tokens = len(user_segment[1]) + _INLINE_ROLE_SLACK_TOKENS
            if max_system_tokens is not None and len(segment) > max_system_tokens:
                return "fold", f"{name}: system insertion overhead ({len(segment)} > {max_system_tokens} tokens)"
    except Exception as exc:
        return "fold", f"probe could not render: {type(exc).__name__}"
    return "preserve", "renderer preserves distinct system turns and existing prompt tokens"