Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: DeepSeek-V4.1 with a short max_tokens (48-1023) caps the thinking at half of it, so a text block always arrives

Niko1221b4418e30dfa6
diff --git a/serve/server.py b/serve/server.py
index 19e03a2be..920cd35bc 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -201,7 +201,8 @@ VISION_START = "<|vision_start|>"
# sets it; 0 turns it off.
REPEAT_STOP_TOKENS = 256
ANSWER_ROOM_MIN_CAP = 1024 # 0.1.45: from this max_tokens on, thinking without a budget is closed early enough to leave room to answer
-ANSWER_RESERVE_MIN = 512 # #984: the tokens kept for the answer when a thinking budget is cut down to fit max_tokens
+DS41_SMALL_CAP_MIN = 48 # DeepSeek-V4.1 only: from this max_tokens up to ANSWER_ROOM_MIN_CAP, thinking is capped at half of it
+ANSWER_RESERVE_MIN = 512 # #984: the tokens kept for the answer when a thinking budget is cut down to fit max_tokens
# #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)
# #1053 (opt-in "reasoning_close_retry": true): a reply that ends on its stop token still inside <think>, with no answer
# and no call, is continued once with the thinking closed (the same way the budget's wrap-up closes it)
@@ -5052,6 +5053,14 @@ class Service:
budget = max_new - min(max(ANSWER_RESERVE_MIN, max_new // 4), max_new // 2)
if max_new <= 32768: print(f"[strata] thinking limited to {budget} of max_tokens {max_new} so the answer has room "
"(\"answer_room\": false in the config turns this off)", flush=True)
+ if (thinking and not budget and self.answer_room and self.family == "deepseek41"
+ and max_new and DS41_SMALL_CAP_MIN <= max_new < ANSWER_ROOM_MIN_CAP):
+ # DeepSeek-V4.1 only: a short max_tokens (/v1/messages and /v1/chat/completions with 200-600) was spent
+ # entirely on the thinking, so no text block ever arrived. With no budget of the client's own, the thinking
+ # is capped at half of max_tokens; then the wrap-up, </think> and the answer follow in the other half.
+ budget = max_new // 2
+ print(f"[strata] thinking limited to {budget} of max_tokens {max_new} so the answer has room "
+ "(\"answer_room\": false in the config turns this off)", flush=True)
if budget and max_new and max_new > 0:
# #984: a budget at or near max_tokens never fires before the reply is cut off while still thinking, so
# it has no answer. Close the thinking early enough to leave room for one.
diff --git a/serve/test_dsv41.py b/serve/test_dsv41.py
index bc798034c..f9f13a854 100644
--- a/serve/test_dsv41.py
+++ b/serve/test_dsv41.py
@@ -23,7 +23,7 @@ import strata_tokenizer as ST # noqa: E402
from serve import frontend # noqa: E402
from serve.dsml import DsmlOutputParser, DsmlTranslator, to_dsml_force # noqa: E402
from serve.frontend import ChatTemplate, OutputParser, openai_to_messages # noqa: E402
-from serve.server import (FAMILY_MODEL_NAMES, MockEngine, Service, default_model_name, model_card_sampling, # noqa: E402
+from serve.server import (FAMILY_MODEL_NAMES, MockEngine, REASONING_WRAP_UP, Service, default_model_name, model_card_sampling, # noqa: E402
serve)
FIX = ROOT / "serve" / "fixtures" / "deepseek41"
@@ -296,8 +296,8 @@ class ReadyTimeout(unittest.TestCase):
server.apply_ready_timeout({"engine_ready_s": bad})
-class Api(unittest.TestCase):
- """Both APIs against a fake engine that replays scripted DeepSeek text."""
+class ApiBase(unittest.TestCase):
+ """Both APIs against a fake engine that replays scripted DeepSeek text (the helpers; Api has the tests)."""
@classmethod
def setUpClass(cls):
@@ -334,6 +334,8 @@ class Api(unittest.TestCase):
body = {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 400, **extra}
return self.post("/v1/chat/completions", body)
+
+class Api(ApiBase):
# -- thinking --------------------------------------------------------------------------------
def test_thinking_by_default(self):
self.start("The user greets me.</think>\n\nHello!")
@@ -527,6 +529,108 @@ class Api(unittest.TestCase):
self.assertEqual(last["decode_tok_s"], 100.0)
+class ThinkForever(DsEngine):
+ """Thinks without end (random letters) until the thinking is closed in the prompt, then answers."""
+
+ ANSWER = "Four."
+
+ def __init__(self, tok):
+ super().__init__(tok, "")
+ self.prompts = []
+
+ def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+ self.prompts.append(list(ids))
+ self.last_prompt = list(ids)
+ got = 0
+ try:
+ if self.tok.decode(list(ids[-14:])).endswith("</think>\n\n"):
+ out = self.tok.encode(self.ANSWER) + [EOS]
+ else:
+ import random
+ rng = random.Random(len(ids)) # (no repeated token: the server's loop guard would end it)
+ out = self.tok.encode("".join(chr(97 + rng.randrange(26)) for _ in range(max_new)))
+ for t in out[:max_new]:
+ if cancel.is_set():
+ return
+ got += 1
+ yield t
+ finally:
+ self.last = {"generated": got, "prompt_tokens": len(ids), "prompt_ms": 50.0, "decode_ms": got * 10.0,
+ "finish": "stop"}
+
+
+class SmallCap(ApiBase):
+ """A short max_tokens (200-600) with the thinking on: it was all spent on the thinking and no text block came.
+ With no budget of the client's own the thinking is capped at half of max_tokens (this family only)."""
+
+ def start_forever(self):
+ self.start("")
+ engine = ThinkForever(self.tok)
+ self.svc.engine = engine
+ self.engine = engine
+ return engine
+
+ def text_of(self, r):
+ return "".join(c.get("text", "") for c in r["content"] if c["type"] == "text")
+
+ def test_every_short_cap_gets_an_answer_on_both_apis(self):
+ for cap in (200, 400, 600):
+ for api in ("anthropic", "openai"):
+ with self.subTest(cap=cap, api=api):
+ self.start_forever()
+ if api == "anthropic":
+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": cap, "thinking": {"type": "enabled"},
+ "messages": [{"role": "user", "content": "2+2?"}]})
+ self.assertEqual(status, 200, r)
+ self.assertEqual(self.text_of(r), ThinkForever.ANSWER)
+ self.assertLessEqual(r["usage"]["output_tokens"], cap)
+ else:
+ status, r = self.chat(max_tokens=cap)
+ self.assertEqual(status, 200, r)
+ self.assertEqual(r["choices"][0]["message"]["content"], ThinkForever.ANSWER)
+ self.assertLessEqual(r["usage"]["completion_tokens"], cap)
+ first = self.engine.prompts[0]
+ self.assertEqual(len(self.engine.prompts), 2) # the thinking was wrapped up, then answered
+ self.assertEqual(len(self.engine.prompts[1]) - len(first), cap // 2 - 0 + len(self.tok.encode(
+ REASONING_WRAP_UP, parse_special=True)))
+
+ def test_thinking_off_is_untouched(self):
+ for cap in (200, 400, 600):
+ with self.subTest(cap=cap):
+ self.start("Plain answer.")
+ status, r = self.chat(max_tokens=cap, reasoning_effort="none")
+ self.assertEqual(status, 200, r)
+ self.assertEqual(r["choices"][0]["message"]["content"], "Plain answer.")
+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": cap, "thinking": {"type": "disabled"},
+ "messages": [{"role": "user", "content": "hi"}]})
+ self.assertEqual(self.text_of(r), "Plain answer.")
+
+ def test_a_client_budget_wins(self):
+ self.start_forever()
+ status, r = self.chat(max_tokens=600, reasoning_budget_tokens=20)
+ self.assertEqual(status, 200, r)
+ self.assertEqual(len(self.engine.prompts[1]) - len(self.engine.prompts[0]), 20 + len(self.tok.encode(
+ REASONING_WRAP_UP, parse_special=True)))
+
+ def test_a_tool_call_under_a_short_cap_still_parses(self):
+ for cap in (200, 400, 600):
+ with self.subTest(cap=cap):
+ self.start("Need it.</think>" + call(("f", [("a", "1", True)])))
+ status, r = self.chat(max_tokens=cap, tools=[{"type": "function", "function": {
+ "name": "f", "description": "f", "parameters": {"type": "object", "properties": {"a": {"type": "string"}}}}}])
+ self.assertEqual(status, 200, r)
+ calls = r["choices"][0]["message"]["tool_calls"]
+ self.assertEqual(calls[0]["function"]["name"], "f")
+ self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"a": "1"})
+ self.assertEqual(len(self.engine.last_prompt) > 0, True)
+
+ def test_other_families_keep_the_old_behaviour(self):
+ self.start_forever()
+ self.svc.set_family(None)
+ status, r = self.chat(max_tokens=300)
+ self.assertEqual(len(self.engine.prompts), 1) # no wrap-up: Qwen's serving is as it was
+
+
class PickTemplateTests(unittest.TestCase):
"""A pack made from another GGUF can ship the V4-Flash template (`tool_calls`, no space): the V4.1 model then
writes malformed calls (P2003 install test). The server swaps in the V4.1 template for such a pack."""