Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
8.8 KB
Raw Blame
"""serve/test_prompt_encoder.py - #567's incremental prompt encoding with #537's plain spans: every prompt's ids
are those of a full encode, and a turn reuses the previous one's.

    python -m unittest serve.test_prompt_encoder -v
"""
from __future__ import annotations

import random
import re
import sys
import unittest
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "tools"))
from serve.frontend import ChatTemplate  # noqa: E402
from serve.server import ByteTokenizer, MockEngine, Service  # noqa: E402
from strata_tokenizer import PromptEncoder, plain_agree_until  # noqa: E402


class ThinkTokenizer(ByteTokenizer):
    """ByteTokenizer whose <think> / </think> are specials matched even without parse_special (type 4), as in the
    Qwen vocabulary: the case #537's plain spans exist for."""
    SPECIALS = ByteTokenizer.SPECIALS + ["<think>", "</think>"]
    ALWAYS = ("<think>", "</think>")


PIECES = ["<|im_start|>user\n", "<|im_end|>\n", "<think>", "</think>", "hello ", "é你 ", "<|im_sta", "x" * 7,
          "<|im_start|>assistant\n", "</thi", "\n\n", "<|endoftext|>", "<|vision_start|>", "<|image_pad|>"]


def spans_of(text, rng):
    """Random plain spans over the specials' texts (<think>, </think>, <|im_start|>, ...), as unmark_think_literals
    marks the ones quoted inside a message."""
    matches = re.finditer("|".join(map(re.escape, ThinkTokenizer.SPECIALS)), text)
    return tuple((m.start(), m.end()) for m in matches if rng.random() < 0.5)   # finditer: left to right, no overlap


class Units(unittest.TestCase):
    def test_plain_agree_until(self):
        self.assertEqual(plain_agree_until((), (), 50), 50)
        self.assertEqual(plain_agree_until(((5, 13),), ((5, 13),), 50), 50)
        self.assertEqual(plain_agree_until(((5, 13),), (), 50), 5)
        self.assertEqual(plain_agree_until(((5, 13),), ((5, 9),), 50), 9)
        self.assertEqual(plain_agree_until(((60, 70),), (), 50), 50)          # beyond the shared text: no matter
        self.assertEqual(plain_agree_until(((5, 13), (20, 28)), ((5, 13),), 50), 20)

    def test_same_ids_as_a_full_encode(self):
        tok = ThinkTokenizer()
        rng = random.Random(7)
        for conv in range(300):
            enc = PromptEncoder(tok)
            text, plain = "", ()
            for turn in range(8):
                text += "".join(rng.choice(PIECES) for _ in range(rng.randint(1, 12)))
                if rng.random() < 0.6:
                    plain = spans_of(text, rng)                     # the spans may change in the shared part too
                with self.subTest(conv=conv, turn=turn):
                    self.assertEqual(enc.encode(text, plain), tok.encode(text, True, plain))
                if rng.random() < 0.2 and len(text) > 10:              # an edit: a client re-rendering its history
                    cut = rng.randrange(len(text))
                    text = text[:cut]
                    plain = tuple((a, b) for a, b in plain if b <= cut)

    def test_a_turn_reuses_the_previous_one(self):
        tok = ThinkTokenizer()
        enc = PromptEncoder(tok)
        # as a chat turn: the earlier prompt goes on past its last special (the answer being written), and the
        # next one shares it up to there - the resume point needs max_special_len characters after it
        base = "<|im_start|>user\n" + "quoted </think> here " * 200 + "<|im_end|>\n<|im_start|>assistant\nan answer"
        plain = tuple((i, i + 8) for i in range(len(base)) if base.startswith("</think>", i))
        enc.encode(base, plain)
        more = base + " that goes on<|im_end|>\n<|im_start|>user\nnext<|im_end|>\n"
        self.assertEqual(enc.encode(more, plain), tok.encode(more, True, plain))
        self.assertGreater(enc.last_reused, len(base) // 2)
        # the same text with the quoted </think> no longer plain: nothing before it may be reused past its start
        self.assertEqual(enc.encode(more, ()), tok.encode(more, True, ()))
        self.assertLessEqual(enc.last_reused, plain[0][0])


class Served(unittest.TestCase):
    def test_quoted_think_across_turns(self):
        tok = ThinkTokenizer()
        template = ChatTemplate(ROOT / "serve/chat_template.jinja")
        svc = Service(MockEngine(tok, "</think>\n\nok", max_context=1 << 20), tok, template)
        full = Service(MockEngine(tok, "</think>\n\nok", max_context=1 << 20), tok, template)
        full.prompts = None                                        # the reference: a full encode every time
        self.assertIsNotNone(svc.prompts)
        msgs = [{"role": "system", "content": "Explain tags."}]
        for turn in range(5):
            msgs.append({"role": "user", "content": f"turn {turn}: what does </think> mean? <think> too " * 3})
            got, want = svc.encode_prompt(msgs, None, {}), full.encode_prompt(msgs, None, {})
            self.assertEqual(got, want)
            if turn:
                self.assertGreater(svc.prompts.last_reused, 0)
            msgs.append({"role": "assistant", "content": "It closes the reasoning: </think> is a marker."})

    def service(self, max_context=1 << 22):
        tok = ByteTokenizer()
        return Service(MockEngine(tok, "ok", max_context=max_context), tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))

    def test_another_conversation_does_not_evict_the_first(self):
        """A subagent beside the main conversation: every rendered prompt starts with the template's header, which
        must not make the second client's prompt a continuation of the first's entry."""
        svc = self.service()
        turn2 = [{"role": "user", "content": "a long question " * 500}, {"role": "assistant", "content": "answer"},
                 {"role": "user", "content": "next"}]
        svc.encode_prompt(turn2, None, {})
        svc.encode_prompt([{"role": "user", "content": "one unrelated prompt " * 300}], None, {})
        self.assertEqual(len(svc.prompts.entries), 2)
        turn3 = turn2 + [{"role": "assistant", "content": "x"}, {"role": "user", "content": "more"}]
        svc.encode_prompt(turn3, None, {})
        self.assertGreater(svc.prompts.last_reused, len(turn2[0]["content"]))
        self.assertEqual(len(svc.prompts.entries), 2)           # turn 3 replaced turn 2, the other one stayed

    def test_a_refused_prompt_is_not_kept(self):
        svc = self.service(max_context=4096)
        svc.fit_max_tokens = False   # the strict mode refuses an over-long max_tokens (the default now shortens it)
        msgs = [{"role": "user", "content": "kept " * 20}]
        svc.prepare(msgs, None, {}, 16)
        refused = msgs + [{"role": "assistant", "content": "a"}, {"role": "user", "content": "more " * 20}]
        full = self.service(max_context=4096)
        full.prompts = None
        refused_ids = full.encode_prompt(refused, None, {})
        with self.assertRaisesRegex(ValueError, "exceeds the context"):
            svc.prepare(refused, None, {}, 1 << 20)
        self.assertNotIn(refused_ids, [e[2] for e in svc.prompts.entries])
        self.assertNotIn(full.render_prompt(refused, None, {}), [e[0] for e in svc.prompts.entries])

    def test_random_messages_equal_a_full_encode(self):
        """Messages quoting control-token texts and <think> tags at random, growing, edited and cut back between
        requests: the incremental encoder's ids are the full encode's, 2400 prompts."""
        tok = ThinkTokenizer()
        template = ChatTemplate(ROOT / "serve/chat_template.jinja")
        svc = Service(MockEngine(tok, "ok", max_context=1 << 20), tok, template)
        full = Service(MockEngine(tok, "ok", max_context=1 << 20), tok, template)
        full.prompts = None
        rng = random.Random(567)
        quoted = ThinkTokenizer.SPECIALS + ["<|im_sta", "</thi"]
        words = ["hello ", "é你 ", "x" * 9, "\n", "tool "]

        def text():
            return "".join(rng.choice(quoted if rng.random() < 0.4 else words) for _ in range(rng.randint(1, 10)))

        reused = 0
        for conv in range(300):
            msgs = [{"role": "system", "content": text()}]
            for turn in range(8):
                msgs.append({"role": "user", "content": text()})
                if rng.random() < 0.15:                             # an earlier turn edited by the client
                    msgs[rng.randrange(len(msgs))]["content"] = text()
                if rng.random() < 0.1 and len(msgs) > 2:            # a shorter conversation
                    msgs = msgs[:rng.randrange(1, len(msgs))] + msgs[-1:]
                with self.subTest(conv=conv, turn=turn):
                    self.assertEqual(svc.encode_prompt(msgs, None, {}), full.encode_prompt(msgs, None, {}))
                reused += svc.prompts.last_reused > 0
                msgs.append({"role": "assistant", "content": text()})
        self.assertGreater(reused, 1000)


if __name__ == "__main__":
    unittest.main()