Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: DeepSeek-V4.1 with a short max_tokens (48-1023) caps the thinking at half of it, so a text block always arrives
Niko1221b4418e30dfa6
diff --git a/serve/server.py b/serve/server.pyindex 19e03a2be..920cd35bc 100644--- a/serve/server.py+++ b/serve/server.py@@ -201,7 +201,8 @@ VISION_START = "<|vision_start|>"# sets it; 0 turns it off.REPEAT_STOP_TOKENS = 256ANSWER_ROOM_MIN_CAP = 1024 # 0.1.45: from this max_tokens on, thinking without a budget is closed early enough to leave room to answer-ANSWER_RESERVE_MIN = 512 # #984: the tokens kept for the answer when a thinking budget is cut down to fit max_tokens+DS41_SMALL_CAP_MIN = 48 # DeepSeek-V4.1 only: from this max_tokens up to ANSWER_ROOM_MIN_CAP, thinking is capped at half of it+ANSWER_RESERVE_MIN = 512 # #984: the tokens kept for the answer when a thinking budget is cut down to fit max_tokens# #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)# #1053 (opt-in "reasoning_close_retry": true): a reply that ends on its stop token still inside <think>, with no answer# and no call, is continued once with the thinking closed (the same way the budget's wrap-up closes it)@@ -5052,6 +5053,14 @@ class Service:budget = max_new - min(max(ANSWER_RESERVE_MIN, max_new // 4), max_new // 2)if max_new <= 32768: print(f"[strata] thinking limited to {budget} of max_tokens {max_new} so the answer has room ""(\"answer_room\": false in the config turns this off)", flush=True)+ if (thinking and not budget and self.answer_room and self.family == "deepseek41"+ and max_new and DS41_SMALL_CAP_MIN <= max_new < ANSWER_ROOM_MIN_CAP):+ # DeepSeek-V4.1 only: a short max_tokens (/v1/messages and /v1/chat/completions with 200-600) was spent+ # entirely on the thinking, so no text block ever arrived. With no budget of the client's own, the thinking+ # is capped at half of max_tokens; then the wrap-up, </think> and the answer follow in the other half.+ budget = max_new // 2+ print(f"[strata] thinking limited to {budget} of max_tokens {max_new} so the answer has room "+ "(\"answer_room\": false in the config turns this off)", flush=True)if budget and max_new and max_new > 0:# #984: a budget at or near max_tokens never fires before the reply is cut off while still thinking, so# it has no answer. Close the thinking early enough to leave room for one.diff --git a/serve/test_dsv41.py b/serve/test_dsv41.pyindex bc798034c..f9f13a854 100644--- a/serve/test_dsv41.py+++ b/serve/test_dsv41.py@@ -23,7 +23,7 @@ import strata_tokenizer as ST # noqa: E402from serve import frontend # noqa: E402from serve.dsml import DsmlOutputParser, DsmlTranslator, to_dsml_force # noqa: E402from serve.frontend import ChatTemplate, OutputParser, openai_to_messages # noqa: E402-from serve.server import (FAMILY_MODEL_NAMES, MockEngine, Service, default_model_name, model_card_sampling, # noqa: E402+from serve.server import (FAMILY_MODEL_NAMES, MockEngine, REASONING_WRAP_UP, Service, default_model_name, model_card_sampling, # noqa: E402serve)FIX = ROOT / "serve" / "fixtures" / "deepseek41"@@ -296,8 +296,8 @@ class ReadyTimeout(unittest.TestCase):server.apply_ready_timeout({"engine_ready_s": bad})-class Api(unittest.TestCase):- """Both APIs against a fake engine that replays scripted DeepSeek text."""+class ApiBase(unittest.TestCase):+ """Both APIs against a fake engine that replays scripted DeepSeek text (the helpers; Api has the tests)."""@classmethoddef setUpClass(cls):@@ -334,6 +334,8 @@ class Api(unittest.TestCase):body = {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 400, **extra}return self.post("/v1/chat/completions", body)++class Api(ApiBase):# -- thinking --------------------------------------------------------------------------------def test_thinking_by_default(self):self.start("The user greets me.</think>\n\nHello!")@@ -527,6 +529,108 @@ class Api(unittest.TestCase):self.assertEqual(last["decode_tok_s"], 100.0)+class ThinkForever(DsEngine):+ """Thinks without end (random letters) until the thinking is closed in the prompt, then answers."""++ ANSWER = "Four."++ def __init__(self, tok):+ super().__init__(tok, "")+ self.prompts = []++ def generate(self, ids, max_new, sampling, cancel, embeddings=None):+ self.prompts.append(list(ids))+ self.last_prompt = list(ids)+ got = 0+ try:+ if self.tok.decode(list(ids[-14:])).endswith("</think>\n\n"):+ out = self.tok.encode(self.ANSWER) + [EOS]+ else:+ import random+ rng = random.Random(len(ids)) # (no repeated token: the server's loop guard would end it)+ out = self.tok.encode("".join(chr(97 + rng.randrange(26)) for _ in range(max_new)))+ for t in out[:max_new]:+ if cancel.is_set():+ return+ got += 1+ yield t+ finally:+ self.last = {"generated": got, "prompt_tokens": len(ids), "prompt_ms": 50.0, "decode_ms": got * 10.0,+ "finish": "stop"}+++class SmallCap(ApiBase):+ """A short max_tokens (200-600) with the thinking on: it was all spent on the thinking and no text block came.+ With no budget of the client's own the thinking is capped at half of max_tokens (this family only)."""++ def start_forever(self):+ self.start("")+ engine = ThinkForever(self.tok)+ self.svc.engine = engine+ self.engine = engine+ return engine++ def text_of(self, r):+ return "".join(c.get("text", "") for c in r["content"] if c["type"] == "text")++ def test_every_short_cap_gets_an_answer_on_both_apis(self):+ for cap in (200, 400, 600):+ for api in ("anthropic", "openai"):+ with self.subTest(cap=cap, api=api):+ self.start_forever()+ if api == "anthropic":+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": cap, "thinking": {"type": "enabled"},+ "messages": [{"role": "user", "content": "2+2?"}]})+ self.assertEqual(status, 200, r)+ self.assertEqual(self.text_of(r), ThinkForever.ANSWER)+ self.assertLessEqual(r["usage"]["output_tokens"], cap)+ else:+ status, r = self.chat(max_tokens=cap)+ self.assertEqual(status, 200, r)+ self.assertEqual(r["choices"][0]["message"]["content"], ThinkForever.ANSWER)+ self.assertLessEqual(r["usage"]["completion_tokens"], cap)+ first = self.engine.prompts[0]+ self.assertEqual(len(self.engine.prompts), 2) # the thinking was wrapped up, then answered+ self.assertEqual(len(self.engine.prompts[1]) - len(first), cap // 2 - 0 + len(self.tok.encode(+ REASONING_WRAP_UP, parse_special=True)))++ def test_thinking_off_is_untouched(self):+ for cap in (200, 400, 600):+ with self.subTest(cap=cap):+ self.start("Plain answer.")+ status, r = self.chat(max_tokens=cap, reasoning_effort="none")+ self.assertEqual(status, 200, r)+ self.assertEqual(r["choices"][0]["message"]["content"], "Plain answer.")+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": cap, "thinking": {"type": "disabled"},+ "messages": [{"role": "user", "content": "hi"}]})+ self.assertEqual(self.text_of(r), "Plain answer.")++ def test_a_client_budget_wins(self):+ self.start_forever()+ status, r = self.chat(max_tokens=600, reasoning_budget_tokens=20)+ self.assertEqual(status, 200, r)+ self.assertEqual(len(self.engine.prompts[1]) - len(self.engine.prompts[0]), 20 + len(self.tok.encode(+ REASONING_WRAP_UP, parse_special=True)))++ def test_a_tool_call_under_a_short_cap_still_parses(self):+ for cap in (200, 400, 600):+ with self.subTest(cap=cap):+ self.start("Need it.</think>" + call(("f", [("a", "1", True)])))+ status, r = self.chat(max_tokens=cap, tools=[{"type": "function", "function": {+ "name": "f", "description": "f", "parameters": {"type": "object", "properties": {"a": {"type": "string"}}}}}])+ self.assertEqual(status, 200, r)+ calls = r["choices"][0]["message"]["tool_calls"]+ self.assertEqual(calls[0]["function"]["name"], "f")+ self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"a": "1"})+ self.assertEqual(len(self.engine.last_prompt) > 0, True)++ def test_other_families_keep_the_old_behaviour(self):+ self.start_forever()+ self.svc.set_family(None)+ status, r = self.chat(max_tokens=300)+ self.assertEqual(len(self.engine.prompts), 1) # no wrap-up: Qwen's serving is as it was++class PickTemplateTests(unittest.TestCase):"""A pack made from another GGUF can ship the V4-Flash template (`tool_calls`, no space): the V4.1 model thenwrites malformed calls (P2003 install test). The server swaps in the V4.1 template for such a pack."""