Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
"""Tests for tools/strata_tokenizer.py: llama.cpp's reference vectors, the piece cache (0.1.41: the ids are the ones the
merge loop gives, cache cold, warm or switched off), and the incremental prompt encoder, which must give exactly the
ids of a plain encode.
python -m unittest tools.test_strata_tokenizer -v
The real vocabulary comes from $STRATA_TOKENIZER (a pack's tokenizer/ directory) when set, else from llama.cpp's
`models/ggml-vocab-qwen35.gguf` next to the gguf-py the tools use ($STRATA_GGUF_PY, third_party/llama.cpp); the
tests that need it are skipped without one. The chat template's USER_DEFINED tokens (`<think>`, `<tool_call>`,
...) are added to the llama.cpp vocabulary, which does not have them, as the pack's has.
"""
from __future__ import annotations
import json
import os
import random
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tools"))
sys.path.insert(0, str(ROOT))
import strata_tokenizer as ST # noqa: E402
ADDED = ["<think>", "</think>", "<tool_call>", "</tool_call>", "<tool_response>", "</tool_response>",
"<|vision_start|>", "<|image_pad|>", "<|vision_end|>"]
def vocab_gguf() -> Path | None:
cands = [os.environ.get("STRATA_GGUF_PY"), ROOT / "third_party" / "llama.cpp" / "gguf-py", "/opt/llama.cpp/gguf-py"]
for c in cands:
if c and (Path(c).parent / "models" / "ggml-vocab-qwen35.gguf").is_file():
return Path(c).parent / "models" / "ggml-vocab-qwen35.gguf"
return None
def load_tokenizer():
pack = os.environ.get("STRATA_TOKENIZER")
if pack and (Path(pack) / "vocab.json").is_file():
t = Path(pack)
vocab = json.loads((t / "vocab.json").read_text(encoding="utf-8"))
tokens = [None] * len(vocab)
for s, i in vocab.items():
tokens[i] = s
return ST.Tokenizer(tokens, (t / "merges.txt").read_text(encoding="utf-8").split("\n"),
json.loads((t / "token_type.json").read_text()))
path = vocab_gguf()
return None if path is None else gguf_tokenizer(path)
def gguf_tokenizer(path):
"""llama.cpp's qwen35 vocabulary with the template's added tokens: the one its test vectors were made with."""
from gguf_reader import GGUFFile
md = GGUFFile(path).metadata
tokens, types = list(md["tokenizer.ggml.tokens"]), list(md["tokenizer.ggml.token_type"])
for s in ADDED:
if s not in tokens:
tokens.append(s)
types.append(4)
return ST.Tokenizer(tokens, list(md["tokenizer.ggml.merges"]), types)
def byte_tokenizer(specials: list[str]) -> ST.Tokenizer:
"""A vocabulary of the 256 byte tokens and `specials` (CONTROL), no merges: small enough to build per test."""
tokens = [ST.BYTE_TO_UNICODE[b] for b in range(256)] + specials
return ST.Tokenizer(tokens, [], [1] * 256 + [3] * len(specials))
WORDS = ["the", "tokenizer", " ", " ", "\n", "\n\n", "\t", "def", "f(x):", "return", "x", "==", "->", "café",
"你好", "\U0001f600", "é", "1234", "don't", "I'm", "<|im_", "<think", "</", "tool", "_call>",
" ", "\r\n", "Hello,", "world!", "<", ">", "|", " ", "x" * 70]
def random_text(rnd: random.Random, n: int, specials=()) -> str:
parts = []
for _ in range(n):
parts.append(rnd.choice(WORDS) if not specials or rnd.random() > 0.05 else rnd.choice(specials))
if rnd.random() < 0.5:
parts.append(" ")
return "".join(parts)
def conversation_prompts(template, rnd: random.Random) -> list[tuple[str, str]]:
"""Rendered prompts, in the order a server would see them, from clients that grow a conversation turn after turn,
use tools, send images, change the start of the prompt, edit an earlier turn, send a shorter conversation, and
interleave with each other: (what happens, prompt)."""
out: list[tuple[str, str]] = []
specials = ["<|im_start|>", "<|im_end|>", "<think>", "</think>", "<tool_call>", "</tool_response>"]
weather = [{"type": "function", "function": {"name": "get_weather", "description": "Get the weather",
"parameters": {"type": "object", "properties": {
"city": {"type": "string"}}, "required": ["city"]}}}]
def add(what, msgs, tools=None, **kw):
out.append((what, template.render(msgs, tools=tools, **kw)))
# several turns, with unicode, a literal special token inside the text, and reasoning kept or not
msgs = [{"role": "system", "content": "Be brief. Réponds en français si on te parle en français."}]
for turn in range(6):
msgs.append({"role": "user", "content": random_text(rnd, 60, specials) + " é你\U0001f600 " * turn})
add(f"turn {turn}", msgs)
answer = {"role": "assistant", "content": random_text(rnd, 30)}
if turn % 2:
answer["reasoning_content"] = random_text(rnd, 30)
msgs.append(answer)
grown = list(msgs)
# tools: definitions, a call, its response, then a following turn
tmsgs = [{"role": "user", "content": "Weather in Oslo?"}]
add("tools 1", tmsgs, weather)
tmsgs += [{"role": "assistant", "content": "", "tool_calls": [{"type": "function", "function": {
"name": "get_weather", "arguments": {"city": "Oslo"}}}]},
{"role": "tool", "content": "{\"temp\": 3}", "name": "get_weather"}]
add("tools 2", tmsgs, weather)
tmsgs += [{"role": "assistant", "content": "3 degrees."}, {"role": "user", "content": "And in Bergen? " * 20}]
add("tools 3", tmsgs, weather)
# images: one per user turn, the placeholder tokens are specials
imsgs = []
for turn in range(3):
imsgs.append({"role": "user", "content": [{"type": "image", "source": f"img{turn}"},
{"type": "text", "text": f"What is on picture {turn}?"}]})
add(f"images {turn}", imsgs)
imsgs.append({"role": "assistant", "content": "A cat."})
# the start of the prompt changes (a thinking level, a system message), then the old conversation grows again
add("effort changed", grown, reasoning_effort="low")
add("system changed", [dict(grown[0], content=grown[0]["content"] + " Be kind.")] + grown[1:])
grown.append({"role": "user", "content": "one more question"})
add("grown again", grown)
# an earlier turn edited, then a conversation that got shorter, then the full one again
edited = list(grown)
edited[3] = dict(edited[3], content=edited[3]["content"] + " (edited)")
add("edited", edited)
add("shorter", grown[:5])
add("empty user", [{"role": "user", "content": ""}])
add("full again", grown)
return out
class Oracle(unittest.TestCase):
"""llama.cpp's own tokenizer test vectors for qwen35."""
def test_llama_cpp_vectors(self):
path = vocab_gguf()
if path is None or not path.with_suffix(".gguf.inp").is_file():
self.skipTest("llama.cpp's ggml-vocab-qwen35.gguf test vectors are not here")
tk = gguf_tokenizer(path) # not $STRATA_TOKENIZER: the vectors belong to this vocabulary
inp = path.with_suffix(".gguf.inp").read_text(encoding="utf-8").split("\n__ggml_vocab_test__\n")
out = path.with_suffix(".gguf.out").read_text(encoding="utf-8").split("\n")
n = 0
for text, want in zip(inp, out):
if text.endswith("\n__ggml_vocab_test__"):
text = text[:-len("\n__ggml_vocab_test__")]
self.assertEqual(tk.encode(text), [int(x) for x in want.split()], repr(text[:40]))
n += 1
self.assertGreater(n, 40)
class Marks(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tk = load_tokenizer()
if cls.tk is None:
raise unittest.SkipTest("no tokenizer (set STRATA_TOKENIZER or STRATA_GGUF_PY)")
def test_marks(self):
text = "a<|im_start|>user\nhi<|im_end|>\n<think>\n"
ids, marks = self.tk.encode_marked(text, parse_special=True)
self.assertEqual(ids, self.tk.encode(text, parse_special=True))
self.assertEqual([m[0] for m in marks], [text.index("user"), text.index("<|im_end|>") + len("<|im_end|>"),
len(text) - 1])
for end, n in marks: # each mark: the ids of the text up to it
self.assertEqual(ids[:n], self.tk.encode(text[:end], parse_special=True))
class Prompts(unittest.TestCase):
"""PromptEncoder: the ids of a plain encode, whatever the earlier prompts were."""
@classmethod
def setUpClass(cls):
cls.tk = load_tokenizer()
if cls.tk is None:
raise unittest.SkipTest("no tokenizer (set STRATA_TOKENIZER or STRATA_GGUF_PY)")
from serve.frontend import ChatTemplate
cls.template = ChatTemplate(ROOT / "serve" / "chat_template.jinja")
def check(self, enc, text):
self.assertEqual(enc.encode(text), self.tk.encode(text, parse_special=True))
def test_chat_golden(self):
cases = json.loads((ROOT / "serve" / "chat_golden.json").read_text(encoding="utf-8"))
enc = ST.PromptEncoder(self.tk)
for c in cases + cases: # the second round finds its own earlier prompts
if c.get("rendered"):
with self.subTest(case=c["name"]):
self.check(enc, c["rendered"])
def test_conversations(self):
for seed in range(3):
prompts = conversation_prompts(self.template, random.Random(seed))
wants = [self.tk.encode(prompt, parse_special=True) for _, prompt in prompts]
for keep in (1, 2, 4):
enc = ST.PromptEncoder(self.tk, keep=keep)
for (what, prompt), want in zip(prompts, wants):
with self.subTest(seed=seed, keep=keep, what=what):
self.assertEqual(enc.encode(prompt), want)
def test_growing_turn_reuses_the_previous_prompt(self):
enc = ST.PromptEncoder(self.tk)
msgs = [{"role": "user", "content": "first question " * 50}]
prev = ""
for turn in range(4):
prompt = self.template.render(msgs)
self.check(enc, prompt)
if turn:
self.assertGreater(enc.last_reused, len(prev) - 100)
prev = prompt
msgs += [{"role": "assistant", "content": f"answer {turn}"}, {"role": "user", "content": f"question {turn}"}]
def test_prefix_of_an_earlier_prompt_and_unrelated_prompts(self):
enc = ST.PromptEncoder(self.tk, keep=2)
a = self.template.render([{"role": "user", "content": "first question " * 50}])
b = self.template.render([{"role": "user", "content": "another conversation"}])
c = self.template.render([{"role": "user", "content": "a third one"}])
for text in (a, a[:len(a) // 2], b, c, a, a + "tail", "", "<|im_end|>", "<|im_end|><|im_end|>x"):
with self.subTest(text=text[:30]):
self.check(enc, text)
def test_a_read_only_encode_gives_the_ids_and_stores_nothing(self):
enc = ST.PromptEncoder(self.tk)
a = self.template.render([{"role": "user", "content": "first question " * 50}])
enc.encode(a)
kept = list(enc.entries)
for text in (a + "<|im_start|>user\nand a follow-up<|im_end|>\n", a[:len(a) // 2], "<|im_end|>x",
self.template.render([{"role": "user", "content": "another conversation"}])):
with self.subTest(text=text[:30]):
self.assertEqual(enc.encode(text, store=False), self.tk.encode(text, parse_special=True))
self.assertTrue(enc.entries == kept, "a read-only encode wrote an entry")
self.assertEqual(enc.encode(a + "tail", store=False), self.tk.encode(a + "tail", parse_special=True))
self.assertGreater(enc.last_reused, len(a) // 2)
def test_the_returned_ids_are_the_callers(self):
enc = ST.PromptEncoder(self.tk)
text = "<|im_start|>user\nhi<|im_end|>\n" + "and a tail longer than the look-ahead margin. " * 3
want = self.tk.encode(text, parse_special=True)
got = enc.encode(text)
got[0] = -1 # inside the part the next call takes from this one
got.append(-1)
self.assertEqual(enc.encode(text), want)
self.assertGreater(enc.last_reused, 0)
class Margin(unittest.TestCase):
"""A special literal that is a prefix of a longer one: cutting at the shorter one's end without the look-ahead
margin would give different ids."""
def test_longer_literal_past_the_common_prefix(self):
tk = byte_tokenizer(["<|a|>", "<|a|>zz"])
enc = ST.PromptEncoder(tk)
first = "x<|a|>y" + "<|a|>" * 3 + "q"
self.assertEqual(enc.encode(first), tk.encode(first, parse_special=True))
second = "x<|a|>y" + "<|a|>" * 3 + "zz" # shares everything up to the last <|a|>
got = enc.encode(second)
self.assertEqual(got, tk.encode(second, parse_special=True))
self.assertEqual(got[-1], tk.special_tokens["<|a|>zz"])
self.assertGreater(enc.last_reused, 0) # the earlier boundaries were still reused
def test_literal_ending_inside_another(self):
tk = byte_tokenizer(["<|a|>", "b<|a|>c"])
enc = ST.PromptEncoder(tk)
for text in ("zzb<|a|>d", "zzb<|a|>c", "zzb<|a|>cz<|a|>", "zzb<|a|>cz<|a|>b<|a|>c"):
with self.subTest(text=text):
self.assertEqual(enc.encode(text), tk.encode(text, parse_special=True))
def test_random_specials(self):
"""Literals that extend or end inside one another, texts that change right after one of them."""
rnd = random.Random(3)
def word(lo, hi):
return "".join(rnd.choice("<|ab>") for _ in range(rnd.randint(lo, hi)))
for trial in range(60):
a, b = word(2, 4), word(2, 4)
lits = sorted({a, a + word(1, 3), b, word(1, 2) + b + word(0, 2)})
tk = byte_tokenizer(lits)
enc = ST.PromptEncoder(tk, keep=3)
pieces = lits + [x[:rnd.randint(1, len(x))] for x in lits] + ["x", "y", "a", "|"]
base = "".join(rnd.choice(pieces) for _ in range(60))
for _ in range(30):
cut = rnd.randint(0, len(base))
text = base[:cut] + "".join(rnd.choice(pieces) for _ in range(rnd.randint(0, 6)))
self.assertEqual(enc.encode(text), tk.encode(text, parse_special=True), (lits, text))
if rnd.random() < 0.3:
base = text + base[cut:]
class CommonPrefix(unittest.TestCase):
def test_against_a_loop(self):
rnd = random.Random(5)
for _ in range(500):
a = "".join(rnd.choice("ab") for _ in range(rnd.randint(0, 9000)))
k = rnd.randint(0, len(a))
b = a[:k] + rnd.choice(["", "c", "cd"]) + a[k:k + rnd.randint(0, 50)]
want = 0
while want < min(len(a), len(b)) and a[want] == b[want]:
want += 1
self.assertEqual(ST.common_prefix_len(a, b), want)
self.assertEqual(ST.common_prefix_len(b, a), want)
def toy() -> ST.Tokenizer:
base = [ST.BYTE_TO_UNICODE[b] for b in range(256)]
merges = ["t h", "th e", "i n", "a n", "an d", "Ġ t", "Ġt h", "Ġth e", "e r", "o n"]
merges = [m.replace("Ġ", ST.BYTE_TO_UNICODE[32]) for m in merges]
tokens = list(base)
for m in merges:
a, b = m.split(" ")
tokens.append(a + b)
return ST.Tokenizer(tokens, merges)
class PieceCache(unittest.TestCase):
def test_same_ids_cold_warm_and_off(self):
rng = random.Random(7)
words = ["the", "and", "in", "on", "there", "other", "x" * 70, "你好", "def", "return", "\n\n", " "]
text = "".join(rng.choice(words) + rng.choice([" ", "", "\n", "\t"]) for _ in range(3000))
a = toy()
a.PIECE_CACHE_MAX = 0
want = a.encode(text)
b = toy()
cold = b.encode(text)
warm = b.encode(text)
self.assertEqual(cold, want)
self.assertEqual(warm, want)
self.assertGreater(len(b._piece_ids), 0)
self.assertEqual(b.decode(warm), text)
def test_the_cache_is_bounded(self):
t = toy()
t.PIECE_CACHE_MAX = 5
t.encode(" ".join("w%d" % i for i in range(100)))
self.assertLessEqual(len(t._piece_ids), 5)
class ByteCache(unittest.TestCase):
def test_decode_before_encode_cold_and_warm(self):
t = toy()
text = "café 日本語 🙂\r\n\x00"
ids = list(text.encode("utf-8")) # toy's first 256 token IDs are the raw bytes
self.assertEqual(t.decode(ids, errors="strict"), text)
self.assertEqual(t.decode(ids, errors="strict"), text)
self.assertEqual(t.decode([]), "")
for i in range(256):
self.assertEqual(t.token_bytes(i), bytes([i]))
def test_invalid_ids_before_and_after_cache_use(self):
t = toy()
for _ in range(2):
for i in (-1, len(t.tokens), len(t.tokens) + 100):
with self.assertRaises(IndexError):
t.token_bytes(i)
self.assertEqual(t.token_bytes(65), b"A")
def test_instances_with_different_vocabularies_stay_independent(self):
tokens = [ST.BYTE_TO_UNICODE[b] for b in range(256)]
a, b = ST.Tokenizer(tokens, []), ST.Tokenizer(tokens[::-1], [])
text = "the café 日本語 🙂\n"
for t in (a, b, a, b):
expected = [t.ids[ST.BYTE_TO_UNICODE[byte]] for byte in text.encode("utf-8")]
self.assertEqual(t.encode(text), expected)
self.assertEqual(t.decode(expected, errors="strict"), text)
self.assertEqual(a.token_bytes(0), b"\x00")
self.assertEqual(b.token_bytes(0), b"\xff")
def test_shared_tokenizer_concurrent_first_decode(self):
import threading
from concurrent.futures import ThreadPoolExecutor
t = toy()
texts = ["café", "日本語", "🙂\r\n", "\x00plain"]
barrier = threading.Barrier(len(texts))
def decode(text):
ids = list(text.encode("utf-8"))
barrier.wait(timeout=5)
for _ in range(100):
if t.decode(ids, errors="strict") != text:
raise AssertionError("concurrent decode changed the text")
return text
with ThreadPoolExecutor(len(texts)) as pool:
self.assertEqual(list(pool.map(decode, texts)), texts)
if __name__ == "__main__":
unittest.main()