Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Server: only a one-token end marker is a stop id; pieces of <|im_end|> no longer end a DeepSeek reply ("Nerazim" came out as "Neraz", #4576)
Niko122143f32b304628
diff --git a/serve/server.py b/serve/server.pyindex d332e97ab..7990c47ad 100644--- a/serve/server.py+++ b/serve/server.py@@ -3777,6 +3777,26 @@ class ByteTokenizer:return raw.decode("utf-8", errors=errors)+# ------------------------------------------------------------------------------------------------ end tokens+def end_token_ids(tokenizer) -> set[int]:+ """The ids that end a reply for a tokenizer whose family was not named (Service.set_family sets them for DeepSeek).++ Qwen's <|im_end|> and <|endoftext|> end a reply only where each is ONE token. A tokenizer without them (DeepSeek-V4.1's+ splits "<|im_end|>" into '<', '|', 'im', '_end', '|', '>') would make every piece of them a stop id: the reply ended in front+ of the first "|", ">", "end", "of", "text" or "im" it wrote, so "Nerazim" came out as "Neraz" (#4576, and a table's first+ "|" in #4137). Those ids are left out, and the end token the tokenizer names (special_ids "eos") ends the reply instead."""+ ids: set[int] = set()+ for marker in (IM_END, "<|endoftext|>"):+ got = tokenizer.encode(marker, parse_special=True)+ if len(got) == 1:+ ids.add(int(got[0]))+ if not ids:+ eos = (getattr(tokenizer, "special_ids", None) or {}).get("eos")+ if eos is not None:+ ids.add(int(eos))+ return ids++# ------------------------------------------------------------------------------------------------ coreclass Detokenizer:"""Incremental decode: each token's bytes go through an incremental UTF-8 decoder, which emits the complete@@ -4044,8 +4064,7 @@ class Service:self.tool_call_recovery = False # opt-in: tool calls in forms next to the template'sself.images = None # serve/images.py's ImageGen when the config has "images"self.restore_after_load = None # a session file an image job saved before it unloaded- self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +- tokenizer.encode("<|endoftext|>", parse_special=True))+ self.stop_ids = end_token_ids(tokenizer)self.family = None # the model family ("deepseek41"): set_family() belowself.parser_cls = OutputParser # reads the model's output (reasoning, content, tool calls)# #567: a prompt re-encodes only what follows the last special token it shares with a recent prompt (thediff --git a/serve/test_dsv41.py b/serve/test_dsv41.pyindex 122a3ab60..ad80fdd1e 100644--- a/serve/test_dsv41.py+++ b/serve/test_dsv41.py@@ -776,13 +776,13 @@ class ArchFromTokenizer(unittest.TestCase):self.assertIsNone(server.arch_from_tokenizer(None, "qwen35"))self.assertIsNone(server.arch_from_tokenizer(None, None))- def test_without_the_family_the_end_token_is_not_a_stop_id_and_pieces_of_im_end_are(self):- # what the reporter's config got: the DeepSeek tokenizer splits <|im_end|> into pieces, so "|" and ">" stopped a reply (a table's first- # cell), and the end token (id 1) did not stop it at all+ def test_without_the_family_the_pieces_of_im_end_are_not_stop_ids_and_the_end_token_is(self):+ # what the reporter's config got (#4137, #4576): the DeepSeek tokenizer splits <|im_end|> into pieces, so "|", ">" and "im" (Nerazim)+ # stopped a reply, and the end token (id 1) did not stop it at alltok = mini_tokenizer()wrong = Service(DsEngine(tok, "x"), tok, ChatTemplate(TEMPLATE))- self.assertNotIn(EOS, wrong.stop_ids)- self.assertTrue(wrong.stop_ids & set(tok.encode("|", parse_special=True)))+ self.assertIn(EOS, wrong.stop_ids)+ self.assertFalse(wrong.stop_ids & set(tok.encode("|", parse_special=True)))right = Service(DsEngine(tok, "x"), tok, ChatTemplate(TEMPLATE))right.set_family("deepseek41")self.addCleanup(frontend.use_family, None)diff --git a/serve/test_stop_holdback.py b/serve/test_stop_holdback.pynew file mode 100644index 000000000..7cadf31d2--- /dev/null+++ b/serve/test_stop_holdback.py@@ -0,0 +1,278 @@+"""The end of a reply and the text held back in front of a stop string (#4576: "Nerazim" came out as "Neraz").++DeepSeek-V4.1's tokenizer has no single token for Qwen's <|im_end|> / <|endoftext|>: it splits them into '<', '|', 'im', '_end',+'|', '>' and 'end', 'of', 'text'. A server that took the ids of those pieces as its stop ids (a config without "arch") ended the+reply in front of the first "im" it wrote: Nerazim = N, er, az, im. These tests keep three things true, for the OpenAI and the+Anthropic route, streamed and not, with and without the family named:++ * the reply that has no stop string comes out whole (every word that ends in a piece of a stop string or of an end marker);+ * a stop string cuts the reply exactly there, whatever the chunking;+ * the text held back for a stop string that does not come is given back at the end.++ python -m unittest serve.test_stop_holdback -v+"""+from __future__ import annotations++import json+import sys+import unittest+import urllib.request+from pathlib import Path++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT))+sys.path.insert(0, str(ROOT / "tools"))+import strata_tokenizer as ST # noqa: E402+from serve import frontend, server # noqa: E402+from serve.frontend import ChatTemplate # noqa: E402+from serve.server import ByteTokenizer, Detokenizer, Service, StopMatcher, end_token_ids, serve # noqa: E402+from serve.test_dsv41 import EOS, SPECIALS, TEMPLATE, DsEngine # noqa: E402++QWEN_STOPS = ["<|im_end|>", "<|im_start|>", "<|endoftext|>", "\nUser:"]+DEEPSEEK_STOPS = ["<|end▁of▁sentence|>", "<|User|>", "</s>", "\n\nHuman:"]+# words and endings that look like the start of a stop string or a piece of an end marker+WORDS = ["Nerazim", "Zim", "swim", "Neraz", "im", "im_end", "Nerazimville", "a|b|c|", "x > y and a < b", "the end of the text",+ "<", "<|", "<|im", "<|im_end", "|>", "friend", "legend", "\nUser", "\n\nHuman", "</", "</s", "<|User", "<|", "Nerazim<",+ "Nerazim|", "Nerazim <|im_start|"]+++def nerazim_tokenizer() -> ST.Tokenizer:+ """DeepSeek's specials, the 256 byte tokens and the pieces of the real vocabulary that matter here: N er az im ('im' is id+ 328 in the real one), and the pieces its tokenizer cuts Qwen's end markers into."""+ b2u = ST.bytes_to_unicode()+ extra = ["er", "az", "im", "_e", "_en", "_end", "en", "end", "of", "te", "tex", "text"]+ merges = ["e r", "a z", "i m", "_ e", "_e n", "_en d", "e n", "en d", "o f", "t e", "te x", "tex t"]+ tokens = [s for s, _ in SPECIALS] + [b2u[b] for b in range(256)] + extra+ types = [t for _, t in SPECIALS] + [1] * (256 + len(extra))+ return ST.Tokenizer(tokens, merges, types, "deepseek3", {"bos": 0, "eos": EOS})+++class Fixture(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.tok = nerazim_tokenizer()++ def test_the_fixture_has_the_real_tokenizer_shape(self):+ self.assertEqual([self.tok.tokens[i] for i in self.tok.encode("Nerazim")], ["N", "er", "az", "im"])+ pieces = [self.tok.tokens[i] for i in self.tok.encode("<|im_end|>", parse_special=True)]+ self.assertEqual(pieces, ["<", "|", "im", "_end", "|", ">"])+ pieces = [self.tok.tokens[i] for i in self.tok.encode("<|endoftext|>", parse_special=True)]+ self.assertEqual(pieces, ["<", "|", "end", "of", "text", "|", ">"])+++class EndTokenIds(Fixture):+ def test_pieces_of_the_qwen_markers_are_not_stop_ids(self):+ ids = end_token_ids(self.tok)+ self.assertEqual(ids, {EOS})+ pieces = set(self.tok.encode("<|im_end|>", parse_special=True)) | set(self.tok.encode("<|endoftext|>", parse_special=True))+ self.assertFalse(ids & pieces)++ def test_qwen_markers_that_are_one_token_each_are_the_stop_ids(self):+ tok = ByteTokenizer()+ self.assertEqual(end_token_ids(tok), set(tok.encode("<|im_end|>", parse_special=True))+ | set(tok.encode("<|endoftext|>", parse_special=True)))+ self.assertEqual(len(end_token_ids(tok)), 2)++ def test_no_marker_and_no_eos_means_no_stop_id(self):+ class Bare:+ def encode(self, text, parse_special=False):+ return list(text.encode())+ self.assertEqual(end_token_ids(Bare()), set())++ def test_a_service_without_a_family_does_not_stop_on_im(self):+ svc = Service(DsEngine(self.tok, "x"), self.tok, ChatTemplate(TEMPLATE))+ im = self.tok.encode("im")[0]+ for piece in ("im", "<", "|", ">", "end", "of", "text", "_end"):+ self.assertNotIn(self.tok.encode(piece)[0], svc.stop_ids, piece)+ self.assertIn(im, set(self.tok.encode("<|im_end|>", parse_special=True)))+ self.assertEqual(svc.stop_ids, {EOS})+++class StopMatcherSplits(unittest.TestCase):+ """StopMatcher alone: every way to cut the text in two, and one character at a time."""++ @staticmethod+ def run_matcher(stops, chunks):+ m = StopMatcher(stops)+ out = "".join(m.push(c) for c in chunks)+ if m.hit is None:+ out += m.flush()+ return out, m.hit++ def chunkings(self, text):+ yield [text]+ for i in range(1, len(text)):+ yield [text[:i], text[i:]]+ yield list(text)++ def test_text_without_a_stop_string_comes_out_whole_at_every_split(self):+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS, ["im_end"], ["im"], ["zim"], ["|", ">"], ["end"]):+ for word in WORDS:+ text = "My friend " + word+ if any(s in text for s in stops):+ continue+ for chunks in self.chunkings(text):+ with self.subTest(stops=stops, text=text, chunks=chunks):+ self.assertEqual(self.run_matcher(stops, chunks), (text, None))++ def test_a_stop_string_cuts_exactly_there_at_every_split(self):+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS):+ for stop in stops:+ for word in ("Nerazim", "swim"):+ text = f"My friend {word}{stop}never seen"+ for chunks in self.chunkings(text):+ with self.subTest(stops=stops, text=text, chunks=chunks):+ first = min((text.find(s), s) for s in stops if s in text)+ self.assertEqual(self.run_matcher(stops, chunks), (text[:first[0]], first[1]))++ def test_the_held_back_tail_is_given_back_at_the_end(self):+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS):+ for text in ("Nerazim <|im", "Nerazim <|User", "Nerazim\nUser", "Nerazim <", "ends with a pipe |"):+ with self.subTest(stops=stops, text=text):+ self.assertEqual(self.run_matcher(stops, list(text)), (text, None))++ def test_a_stop_string_that_starts_like_the_word_costs_nothing_but_the_wait(self):+ # "im_end" as a stop: "Nerazim" holds "im" until the next character says it is not the stop string+ m = StopMatcher(["im_end"])+ self.assertEqual(m.push("Neraz"), "Neraz")+ self.assertEqual(m.push("im"), "")+ self.assertEqual(m.held, "im")+ self.assertEqual(m.flush(), "im")+++class ByteWise(Fixture):+ """The text path as the server runs it: one token per byte through the Detokenizer (a UTF-8 character split across tokens is held+ until it is whole), then the StopMatcher."""++ def run_bytes(self, stops, text):+ tok = ByteTokenizer()+ detok, m, out = Detokenizer(tok), StopMatcher(stops), ""+ for b in text.encode("utf-8"):+ out += m.push(detok.push(b))+ if m.hit is not None:+ break+ if m.hit is None:+ out += m.flush()+ return out, m.hit++ def test_every_word_comes_out_whole_byte_by_byte(self):+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS):+ for word in WORDS + ["Zażółć", "日本語の名前"]:+ text = "Hello " + word+ if any(s in text for s in stops):+ continue+ with self.subTest(stops=stops, text=text):+ self.assertEqual(self.run_bytes(stops, text), (text, None))++ def test_a_multibyte_stop_string_is_cut_byte_by_byte(self):+ text = "Nerazim<|end▁of▁sentence|>not shown"+ self.assertEqual(self.run_bytes(DEEPSEEK_STOPS, text), ("Nerazim", "<|end▁of▁sentence|>"))+++class Routes(Fixture):+ """The four ways out of the server (OpenAI and Anthropic, streamed and whole), with the family named and without it."""++ def start(self, script, named):+ """A server for `named` (one per test and setting, reused); the engine replays `script` for the next request."""+ if named in self.servers:+ engine = self.servers[named]+ engine.script = self.tok.encode(script) + [EOS] # (a marker in the script is text the model wrote)+ self.base = self.bases[named]+ return engine+ engine = self.servers[named] = DsEngine(self.tok, script)+ engine.script = self.tok.encode(script) + [EOS]+ svc = Service(engine, self.tok, ChatTemplate(TEMPLATE), model_name="deepseek-v4.1-flash")+ if named:+ svc.model_arch = "deepseek41"+ svc.set_family("deepseek41")+ httpd = serve(svc, port=0)+ self.addCleanup(httpd.server_close)+ self.addCleanup(httpd.shutdown)+ self.addCleanup(frontend.use_family, None)+ self.base = self.bases[named] = f"http://127.0.0.1:{httpd.server_address[1]}"+ return engine++ def send(self, route, stream, stops):+ if route == "openai":+ path = "/v1/chat/completions"+ body = {"model": "m", "max_tokens": 200, "stream": stream, "chat_template_kwargs": {"enable_thinking": False},+ "messages": [{"role": "user", "content": "Use the name Nerazim in a sentence."}]}+ if stops:+ body["stop"] = stops[:4]+ else:+ path = "/v1/messages"+ body = {"model": "m", "max_tokens": 200, "stream": stream, "thinking": {"type": "disabled"},+ "messages": [{"role": "user", "content": "Use the name Nerazim in a sentence."}]}+ if stops:+ body["stop_sequences"] = stops+ req = urllib.request.Request(self.base + path, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"})+ with urllib.request.urlopen(req, timeout=30) as r:+ raw = r.read().decode("utf-8")+ text, finish, stop_seq = "", None, None+ if not stream:+ d = json.loads(raw)+ if route == "openai":+ text, finish = d["choices"][0]["message"].get("content") or "", d["choices"][0]["finish_reason"]+ else:+ text = "".join(b.get("text", "") for b in d["content"] if b["type"] == "text")+ finish, stop_seq = d["stop_reason"], d.get("stop_sequence")+ return text, finish, stop_seq+ for line in raw.splitlines():+ if not line.startswith("data:") or line.strip() == "data: [DONE]":+ continue+ d = json.loads(line[5:])+ if route == "openai":+ for ch in d.get("choices", []):+ text += (ch.get("delta") or {}).get("content") or ""+ finish = ch.get("finish_reason") or finish+ elif d.get("type") == "content_block_delta":+ text += d["delta"].get("text", "")+ elif d.get("type") == "message_delta":+ finish, stop_seq = d["delta"].get("stop_reason"), d["delta"].get("stop_sequence")+ return text, finish, stop_seq++ def setUp(self):+ self.servers = {}+ self.bases = {}++ def every_route(self):+ for named in (False, True):+ for route in ("openai", "anthropic"):+ for stream in (False, True):+ yield named, route, stream++ def test_nerazim_and_its_kin_come_out_whole_with_no_stop_string(self):+ for word in ("Nerazim", "Nerazimville and Nerazim", "swim", "Zim", "a|b|c|", "x > y", "the end of the text", "im_end", "<", "Nerazim<|im"):+ for named, route, stream in self.every_route():+ with self.subTest(word=word, named=named, route=route, stream=stream):+ self.start(word, named)+ text, finish, _ = self.send(route, stream, None)+ self.assertEqual(text.strip(), word)+ self.assertIn(finish, ("stop", "end_turn"))++ def test_the_held_tail_is_given_back_when_the_stop_string_never_comes(self):+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS, ["im_end", "|>"]):+ for named, route, stream in self.every_route():+ for word in ("My friend Nerazim", "Nerazim <|im", "Nerazim <|User", "ends with a pipe |"):+ with self.subTest(stops=stops, word=word, named=named, route=route, stream=stream):+ self.start(word, named)+ text, finish, stop_seq = self.send(route, stream, stops)+ self.assertEqual(text.strip(), word)+ self.assertIn(finish, ("stop", "end_turn"))+ self.assertIsNone(stop_seq)++ def test_a_stop_string_cuts_the_reply_there(self):+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS):+ for named, route, stream in self.every_route():+ for stop in stops:+ with self.subTest(stops=stops, stop=stop, named=named, route=route, stream=stream):+ self.start("My friend Nerazim" + stop + "and more", named)+ text, finish, stop_seq = self.send(route, stream, stops)+ self.assertEqual(text.strip(), "My friend Nerazim")+ self.assertIn(finish, ("stop", "end_turn", "stop_sequence"))+ if route == "anthropic":+ self.assertEqual(stop_seq, stop)+++if __name__ == "__main__":+ unittest.main()