Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Server: only a one-token end marker is a stop id; pieces of <|im_end|> no longer end a DeepSeek reply ("Nerazim" came out as "Neraz", #4576)

Niko122143f32b304628
diff --git a/serve/server.py b/serve/server.py
index d332e97ab..7990c47ad 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -3777,6 +3777,26 @@ class ByteTokenizer:
return raw.decode("utf-8", errors=errors)
+# ------------------------------------------------------------------------------------------------ end tokens
+def end_token_ids(tokenizer) -> set[int]:
+ """The ids that end a reply for a tokenizer whose family was not named (Service.set_family sets them for DeepSeek).
+
+ Qwen's <|im_end|> and <|endoftext|> end a reply only where each is ONE token. A tokenizer without them (DeepSeek-V4.1's
+ splits "<|im_end|>" into '<', '|', 'im', '_end', '|', '>') would make every piece of them a stop id: the reply ended in front
+ of the first "|", ">", "end", "of", "text" or "im" it wrote, so "Nerazim" came out as "Neraz" (#4576, and a table's first
+ "|" in #4137). Those ids are left out, and the end token the tokenizer names (special_ids "eos") ends the reply instead."""
+ ids: set[int] = set()
+ for marker in (IM_END, "<|endoftext|>"):
+ got = tokenizer.encode(marker, parse_special=True)
+ if len(got) == 1:
+ ids.add(int(got[0]))
+ if not ids:
+ eos = (getattr(tokenizer, "special_ids", None) or {}).get("eos")
+ if eos is not None:
+ ids.add(int(eos))
+ return ids
+
+
# ------------------------------------------------------------------------------------------------ core
class Detokenizer:
"""Incremental decode: each token's bytes go through an incremental UTF-8 decoder, which emits the complete
@@ -4044,8 +4064,7 @@ class Service:
self.tool_call_recovery = False # opt-in: tool calls in forms next to the template's
self.images = None # serve/images.py's ImageGen when the config has "images"
self.restore_after_load = None # a session file an image job saved before it unloaded
- self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
- tokenizer.encode("<|endoftext|>", parse_special=True))
+ self.stop_ids = end_token_ids(tokenizer)
self.family = None # the model family ("deepseek41"): set_family() below
self.parser_cls = OutputParser # reads the model's output (reasoning, content, tool calls)
# #567: a prompt re-encodes only what follows the last special token it shares with a recent prompt (the
diff --git a/serve/test_dsv41.py b/serve/test_dsv41.py
index 122a3ab60..ad80fdd1e 100644
--- a/serve/test_dsv41.py
+++ b/serve/test_dsv41.py
@@ -776,13 +776,13 @@ class ArchFromTokenizer(unittest.TestCase):
self.assertIsNone(server.arch_from_tokenizer(None, "qwen35"))
self.assertIsNone(server.arch_from_tokenizer(None, None))
- def test_without_the_family_the_end_token_is_not_a_stop_id_and_pieces_of_im_end_are(self):
- # what the reporter's config got: the DeepSeek tokenizer splits <|im_end|> into pieces, so "|" and ">" stopped a reply (a table's first
- # cell), and the end token (id 1) did not stop it at all
+ def test_without_the_family_the_pieces_of_im_end_are_not_stop_ids_and_the_end_token_is(self):
+ # what the reporter's config got (#4137, #4576): the DeepSeek tokenizer splits <|im_end|> into pieces, so "|", ">" and "im" (Nerazim)
+ # stopped a reply, and the end token (id 1) did not stop it at all
tok = mini_tokenizer()
wrong = Service(DsEngine(tok, "x"), tok, ChatTemplate(TEMPLATE))
- self.assertNotIn(EOS, wrong.stop_ids)
- self.assertTrue(wrong.stop_ids & set(tok.encode("|", parse_special=True)))
+ self.assertIn(EOS, wrong.stop_ids)
+ self.assertFalse(wrong.stop_ids & set(tok.encode("|", parse_special=True)))
right = Service(DsEngine(tok, "x"), tok, ChatTemplate(TEMPLATE))
right.set_family("deepseek41")
self.addCleanup(frontend.use_family, None)
diff --git a/serve/test_stop_holdback.py b/serve/test_stop_holdback.py
new file mode 100644
index 000000000..7cadf31d2
--- /dev/null
+++ b/serve/test_stop_holdback.py
@@ -0,0 +1,278 @@
+"""The end of a reply and the text held back in front of a stop string (#4576: "Nerazim" came out as "Neraz").
+
+DeepSeek-V4.1's tokenizer has no single token for Qwen's <|im_end|> / <|endoftext|>: it splits them into '<', '|', 'im', '_end',
+'|', '>' and 'end', 'of', 'text'. A server that took the ids of those pieces as its stop ids (a config without "arch") ended the
+reply in front of the first "im" it wrote: Nerazim = N, er, az, im. These tests keep three things true, for the OpenAI and the
+Anthropic route, streamed and not, with and without the family named:
+
+ * the reply that has no stop string comes out whole (every word that ends in a piece of a stop string or of an end marker);
+ * a stop string cuts the reply exactly there, whatever the chunking;
+ * the text held back for a stop string that does not come is given back at the end.
+
+ python -m unittest serve.test_stop_holdback -v
+"""
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "tools"))
+import strata_tokenizer as ST # noqa: E402
+from serve import frontend, server # noqa: E402
+from serve.frontend import ChatTemplate # noqa: E402
+from serve.server import ByteTokenizer, Detokenizer, Service, StopMatcher, end_token_ids, serve # noqa: E402
+from serve.test_dsv41 import EOS, SPECIALS, TEMPLATE, DsEngine # noqa: E402
+
+QWEN_STOPS = ["<|im_end|>", "<|im_start|>", "<|endoftext|>", "\nUser:"]
+DEEPSEEK_STOPS = ["<|end▁of▁sentence|>", "<|User|>", "</s>", "\n\nHuman:"]
+# words and endings that look like the start of a stop string or a piece of an end marker
+WORDS = ["Nerazim", "Zim", "swim", "Neraz", "im", "im_end", "Nerazimville", "a|b|c|", "x > y and a < b", "the end of the text",
+ "<", "<|", "<|im", "<|im_end", "|>", "friend", "legend", "\nUser", "\n\nHuman", "</", "</s", "<|User", "<|", "Nerazim<",
+ "Nerazim|", "Nerazim <|im_start|"]
+
+
+def nerazim_tokenizer() -> ST.Tokenizer:
+ """DeepSeek's specials, the 256 byte tokens and the pieces of the real vocabulary that matter here: N er az im ('im' is id
+ 328 in the real one), and the pieces its tokenizer cuts Qwen's end markers into."""
+ b2u = ST.bytes_to_unicode()
+ extra = ["er", "az", "im", "_e", "_en", "_end", "en", "end", "of", "te", "tex", "text"]
+ merges = ["e r", "a z", "i m", "_ e", "_e n", "_en d", "e n", "en d", "o f", "t e", "te x", "tex t"]
+ tokens = [s for s, _ in SPECIALS] + [b2u[b] for b in range(256)] + extra
+ types = [t for _, t in SPECIALS] + [1] * (256 + len(extra))
+ return ST.Tokenizer(tokens, merges, types, "deepseek3", {"bos": 0, "eos": EOS})
+
+
+class Fixture(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tok = nerazim_tokenizer()
+
+ def test_the_fixture_has_the_real_tokenizer_shape(self):
+ self.assertEqual([self.tok.tokens[i] for i in self.tok.encode("Nerazim")], ["N", "er", "az", "im"])
+ pieces = [self.tok.tokens[i] for i in self.tok.encode("<|im_end|>", parse_special=True)]
+ self.assertEqual(pieces, ["<", "|", "im", "_end", "|", ">"])
+ pieces = [self.tok.tokens[i] for i in self.tok.encode("<|endoftext|>", parse_special=True)]
+ self.assertEqual(pieces, ["<", "|", "end", "of", "text", "|", ">"])
+
+
+class EndTokenIds(Fixture):
+ def test_pieces_of_the_qwen_markers_are_not_stop_ids(self):
+ ids = end_token_ids(self.tok)
+ self.assertEqual(ids, {EOS})
+ pieces = set(self.tok.encode("<|im_end|>", parse_special=True)) | set(self.tok.encode("<|endoftext|>", parse_special=True))
+ self.assertFalse(ids & pieces)
+
+ def test_qwen_markers_that_are_one_token_each_are_the_stop_ids(self):
+ tok = ByteTokenizer()
+ self.assertEqual(end_token_ids(tok), set(tok.encode("<|im_end|>", parse_special=True))
+ | set(tok.encode("<|endoftext|>", parse_special=True)))
+ self.assertEqual(len(end_token_ids(tok)), 2)
+
+ def test_no_marker_and_no_eos_means_no_stop_id(self):
+ class Bare:
+ def encode(self, text, parse_special=False):
+ return list(text.encode())
+ self.assertEqual(end_token_ids(Bare()), set())
+
+ def test_a_service_without_a_family_does_not_stop_on_im(self):
+ svc = Service(DsEngine(self.tok, "x"), self.tok, ChatTemplate(TEMPLATE))
+ im = self.tok.encode("im")[0]
+ for piece in ("im", "<", "|", ">", "end", "of", "text", "_end"):
+ self.assertNotIn(self.tok.encode(piece)[0], svc.stop_ids, piece)
+ self.assertIn(im, set(self.tok.encode("<|im_end|>", parse_special=True)))
+ self.assertEqual(svc.stop_ids, {EOS})
+
+
+class StopMatcherSplits(unittest.TestCase):
+ """StopMatcher alone: every way to cut the text in two, and one character at a time."""
+
+ @staticmethod
+ def run_matcher(stops, chunks):
+ m = StopMatcher(stops)
+ out = "".join(m.push(c) for c in chunks)
+ if m.hit is None:
+ out += m.flush()
+ return out, m.hit
+
+ def chunkings(self, text):
+ yield [text]
+ for i in range(1, len(text)):
+ yield [text[:i], text[i:]]
+ yield list(text)
+
+ def test_text_without_a_stop_string_comes_out_whole_at_every_split(self):
+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS, ["im_end"], ["im"], ["zim"], ["|", ">"], ["end"]):
+ for word in WORDS:
+ text = "My friend " + word
+ if any(s in text for s in stops):
+ continue
+ for chunks in self.chunkings(text):
+ with self.subTest(stops=stops, text=text, chunks=chunks):
+ self.assertEqual(self.run_matcher(stops, chunks), (text, None))
+
+ def test_a_stop_string_cuts_exactly_there_at_every_split(self):
+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS):
+ for stop in stops:
+ for word in ("Nerazim", "swim"):
+ text = f"My friend {word}{stop}never seen"
+ for chunks in self.chunkings(text):
+ with self.subTest(stops=stops, text=text, chunks=chunks):
+ first = min((text.find(s), s) for s in stops if s in text)
+ self.assertEqual(self.run_matcher(stops, chunks), (text[:first[0]], first[1]))
+
+ def test_the_held_back_tail_is_given_back_at_the_end(self):
+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS):
+ for text in ("Nerazim <|im", "Nerazim <|User", "Nerazim\nUser", "Nerazim <", "ends with a pipe |"):
+ with self.subTest(stops=stops, text=text):
+ self.assertEqual(self.run_matcher(stops, list(text)), (text, None))
+
+ def test_a_stop_string_that_starts_like_the_word_costs_nothing_but_the_wait(self):
+ # "im_end" as a stop: "Nerazim" holds "im" until the next character says it is not the stop string
+ m = StopMatcher(["im_end"])
+ self.assertEqual(m.push("Neraz"), "Neraz")
+ self.assertEqual(m.push("im"), "")
+ self.assertEqual(m.held, "im")
+ self.assertEqual(m.flush(), "im")
+
+
+class ByteWise(Fixture):
+ """The text path as the server runs it: one token per byte through the Detokenizer (a UTF-8 character split across tokens is held
+ until it is whole), then the StopMatcher."""
+
+ def run_bytes(self, stops, text):
+ tok = ByteTokenizer()
+ detok, m, out = Detokenizer(tok), StopMatcher(stops), ""
+ for b in text.encode("utf-8"):
+ out += m.push(detok.push(b))
+ if m.hit is not None:
+ break
+ if m.hit is None:
+ out += m.flush()
+ return out, m.hit
+
+ def test_every_word_comes_out_whole_byte_by_byte(self):
+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS):
+ for word in WORDS + ["Zażółć", "日本語の名前"]:
+ text = "Hello " + word
+ if any(s in text for s in stops):
+ continue
+ with self.subTest(stops=stops, text=text):
+ self.assertEqual(self.run_bytes(stops, text), (text, None))
+
+ def test_a_multibyte_stop_string_is_cut_byte_by_byte(self):
+ text = "Nerazim<|end▁of▁sentence|>not shown"
+ self.assertEqual(self.run_bytes(DEEPSEEK_STOPS, text), ("Nerazim", "<|end▁of▁sentence|>"))
+
+
+class Routes(Fixture):
+ """The four ways out of the server (OpenAI and Anthropic, streamed and whole), with the family named and without it."""
+
+ def start(self, script, named):
+ """A server for `named` (one per test and setting, reused); the engine replays `script` for the next request."""
+ if named in self.servers:
+ engine = self.servers[named]
+ engine.script = self.tok.encode(script) + [EOS] # (a marker in the script is text the model wrote)
+ self.base = self.bases[named]
+ return engine
+ engine = self.servers[named] = DsEngine(self.tok, script)
+ engine.script = self.tok.encode(script) + [EOS]
+ svc = Service(engine, self.tok, ChatTemplate(TEMPLATE), model_name="deepseek-v4.1-flash")
+ if named:
+ svc.model_arch = "deepseek41"
+ svc.set_family("deepseek41")
+ httpd = serve(svc, port=0)
+ self.addCleanup(httpd.server_close)
+ self.addCleanup(httpd.shutdown)
+ self.addCleanup(frontend.use_family, None)
+ self.base = self.bases[named] = f"http://127.0.0.1:{httpd.server_address[1]}"
+ return engine
+
+ def send(self, route, stream, stops):
+ if route == "openai":
+ path = "/v1/chat/completions"
+ body = {"model": "m", "max_tokens": 200, "stream": stream, "chat_template_kwargs": {"enable_thinking": False},
+ "messages": [{"role": "user", "content": "Use the name Nerazim in a sentence."}]}
+ if stops:
+ body["stop"] = stops[:4]
+ else:
+ path = "/v1/messages"
+ body = {"model": "m", "max_tokens": 200, "stream": stream, "thinking": {"type": "disabled"},
+ "messages": [{"role": "user", "content": "Use the name Nerazim in a sentence."}]}
+ if stops:
+ body["stop_sequences"] = stops
+ req = urllib.request.Request(self.base + path, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"})
+ with urllib.request.urlopen(req, timeout=30) as r:
+ raw = r.read().decode("utf-8")
+ text, finish, stop_seq = "", None, None
+ if not stream:
+ d = json.loads(raw)
+ if route == "openai":
+ text, finish = d["choices"][0]["message"].get("content") or "", d["choices"][0]["finish_reason"]
+ else:
+ text = "".join(b.get("text", "") for b in d["content"] if b["type"] == "text")
+ finish, stop_seq = d["stop_reason"], d.get("stop_sequence")
+ return text, finish, stop_seq
+ for line in raw.splitlines():
+ if not line.startswith("data:") or line.strip() == "data: [DONE]":
+ continue
+ d = json.loads(line[5:])
+ if route == "openai":
+ for ch in d.get("choices", []):
+ text += (ch.get("delta") or {}).get("content") or ""
+ finish = ch.get("finish_reason") or finish
+ elif d.get("type") == "content_block_delta":
+ text += d["delta"].get("text", "")
+ elif d.get("type") == "message_delta":
+ finish, stop_seq = d["delta"].get("stop_reason"), d["delta"].get("stop_sequence")
+ return text, finish, stop_seq
+
+ def setUp(self):
+ self.servers = {}
+ self.bases = {}
+
+ def every_route(self):
+ for named in (False, True):
+ for route in ("openai", "anthropic"):
+ for stream in (False, True):
+ yield named, route, stream
+
+ def test_nerazim_and_its_kin_come_out_whole_with_no_stop_string(self):
+ for word in ("Nerazim", "Nerazimville and Nerazim", "swim", "Zim", "a|b|c|", "x > y", "the end of the text", "im_end", "<", "Nerazim<|im"):
+ for named, route, stream in self.every_route():
+ with self.subTest(word=word, named=named, route=route, stream=stream):
+ self.start(word, named)
+ text, finish, _ = self.send(route, stream, None)
+ self.assertEqual(text.strip(), word)
+ self.assertIn(finish, ("stop", "end_turn"))
+
+ def test_the_held_tail_is_given_back_when_the_stop_string_never_comes(self):
+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS, ["im_end", "|>"]):
+ for named, route, stream in self.every_route():
+ for word in ("My friend Nerazim", "Nerazim <|im", "Nerazim <|User", "ends with a pipe |"):
+ with self.subTest(stops=stops, word=word, named=named, route=route, stream=stream):
+ self.start(word, named)
+ text, finish, stop_seq = self.send(route, stream, stops)
+ self.assertEqual(text.strip(), word)
+ self.assertIn(finish, ("stop", "end_turn"))
+ self.assertIsNone(stop_seq)
+
+ def test_a_stop_string_cuts_the_reply_there(self):
+ for stops in (QWEN_STOPS, DEEPSEEK_STOPS):
+ for named, route, stream in self.every_route():
+ for stop in stops:
+ with self.subTest(stops=stops, stop=stop, named=named, route=route, stream=stream):
+ self.start("My friend Nerazim" + stop + "and more", named)
+ text, finish, stop_seq = self.send(route, stream, stops)
+ self.assertEqual(text.strip(), "My friend Nerazim")
+ self.assertIn(finish, ("stop", "end_turn", "stop_sequence"))
+ if route == "anthropic":
+ self.assertEqual(stop_seq, stop)
+
+
+if __name__ == "__main__":
+ unittest.main()