Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: max_tokens counts the thinking - without a budget, thinking is closed early enough to leave a quarter of the cap for the answer (cap 8192 cut the thinking off with no answer); config answer_room:false opts out; tests

Niko1221d0e990ba7840
diff --git a/serve/server.py b/serve/server.py
index a74a193c0..252d93681 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -147,6 +147,7 @@ VISION_START = "<|vision_start|>"
# broken state that answers one token forever (an issue saw 36,689 tokens of "!"). The config's "repeat_stop_tokens"
# sets it; 0 turns it off.
REPEAT_STOP_TOKENS = 256
+ANSWER_ROOM_MIN_CAP = 1024 # 0.1.45: from this max_tokens on, thinking without a budget is closed early enough to leave room to answer
ANSWER_RESERVE_MIN = 512 # #984: the tokens kept for the answer when a thinking budget is cut down to fit max_tokens
# #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)
# #1053 (opt-in "reasoning_close_retry": true): a reply that ends on its stop token still inside <think>, with no answer
@@ -3076,6 +3077,7 @@ class Service:
self.literals = literal_tags(getattr(tokenizer, "control_tokens", ())) # texts that stay text inside a message
self.fit_max_tokens = fit_max_tokens # --fit-max-tokens: clamp the output cap instead of 400
self.aliases: list[str] = [] # #297: other names of the model (the config's `aliases`)
+ self.answer_room = True # 0.1.45: see ANSWER_ROOM_MIN_CAP (the config's "answer_room": false turns it off)
self.reasoning_close_retry = False # #1053 (opt-in): close the thinking once when a reply ends inside it
self.codex_compaction_cache = False # #924 (opt-in): a Codex compaction is rendered with its conversation's tools
self.codex_thread_titles = False # #923 (opt-in): answer Codex's thread-title turns without the engine
@@ -4137,6 +4139,15 @@ class Service:
sampling = {**defaults, **req_values}
# #123: read after the merge, so a budget shared through POST /settings is seen like the other keys
budget = self.reasoning_budget(sampling) if thinking else None
+ if thinking and not budget and self.answer_room and max_new and max_new >= ANSWER_ROOM_MIN_CAP:
+ # 0.1.45: max_tokens counts the thinking too. A client that caps the reply at 8192 (the desktop agent's
+ # harness sends max_completion_tokens: 8192 on every request) and a task the model thinks long about got
+ # the cut-off in the middle of the thinking and NO answer. So thinking is closed early enough to leave
+ # the answer its quarter of the cap (the budget's wrap-up: a short closing sentence, </think>, the answer).
+ # A request with a budget of its own, or max_tokens below ANSWER_ROOM_MIN_CAP, is left as it was.
+ budget = max_new - min(max(ANSWER_RESERVE_MIN, max_new // 4), max_new // 2)
+ if max_new <= 32768: print(f"[strata] thinking limited to {budget} of max_tokens {max_new} so the answer has room "
+ "(\"answer_room\": false in the config turns this off)", flush=True)
if budget and max_new and max_new > 0:
# #984: a budget at or near max_tokens never fires before the reply is cut off while still thinking, so
# it has no answer. Close the thinking early enough to leave room for one.
@@ -6844,6 +6855,7 @@ def main() -> int:
fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)
if a.engine == "strata": # the model card's sampling when nothing else is said (see MODEL_CARD_SAMPLING)
svc.model_arch = cfg.get("arch") or "qwen4exp"
+ svc.answer_room = cfg.get("answer_room") is not False
svc.reasoning_close_retry = cfg.get("reasoning_close_retry") is True # #1053: opt-in, off by default
if cfg.get("images"): # image generation (serve/images.py), set up by setup --images
icfg = {k: (os.path.abspath(os.path.join(cfg.get("cwd") or ".", v))
diff --git a/serve/test_reasoning_loop_recovery.py b/serve/test_reasoning_loop_recovery.py
index b53ee56ac..c00fc1bee 100644
--- a/serve/test_reasoning_loop_recovery.py
+++ b/serve/test_reasoning_loop_recovery.py
@@ -58,6 +58,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
engine = RecordingEngine(self.tok, [self.loop,
'I have verified the solution.\n</think>\nComplete solution.'])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = enabled
events = list(service.run(self.ids, True, [], 40000, {}, threading.Event()))
return engine, events
@@ -66,6 +67,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
engine = RecordingEngine(self.tok, [self.loop,
'Verified.\n</think>\nComplete solution.'])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = True
request = {'temperature': .2, 'presence_penalty': 0, 'top_p': .95, 'top_k': 64, 'seed': 2}
list(service.run(self.ids, True, [], 40000, request, threading.Event()))
@@ -81,6 +83,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
def test_frozen_cohort_sampler_is_unchanged_during_recovery(self):
engine = RecordingEngine(self.tok, [self.loop, 'Done.\n</think>\nComplete solution.'])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = True
sampler = {'temperature': 1.0, 'top_p': .95, 'top_k': 20, 'presence_penalty': 1.5, 'seed': 2}
list(service.run(self.ids, True, [], 40000, sampler, threading.Event()))
@@ -91,6 +94,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
def test_a_lower_top_p_and_top_k_are_never_raised_or_lowered(self):
engine = RecordingEngine(self.tok, [self.loop, 'Verified.\n</think>\nComplete solution.'])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = "recover"
list(service.run(self.ids, True, [], 40000, {'temperature': 0, 'top_p': .5, 'top_k': 5}, threading.Event()))
self.assertEqual((engine.sampling_calls[1]['top_p'], engine.sampling_calls[1]['top_k']), (.5, 5))
@@ -110,6 +114,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
def test_stop_mode_ends_the_reply_at_the_loop(self):
engine = RecordingEngine(self.tok, [self.loop])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = "stop"
events = list(service.run(self.ids, True, [], 40000, {}, threading.Event()))
self.assertEqual(len(engine.calls), 1) # no second pass
@@ -121,6 +126,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
"""#1753: the word count never sees the digits, so only the tail's period ends the reply."""
engine = RecordingEngine(self.tok, ['9^104 = ' + '58747693387' * 3000])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = "stop"
events = list(service.run(self.ids, True, [], 40000, {}, threading.Event()))
self.assertEqual(len(engine.calls), 1) # no second pass
@@ -136,6 +142,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
def test_default_single_token_guard_does_not_detect_passage_loop(self):
engine = RecordingEngine(self.tok, [self.loop])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
events = list(service.run(self.ids, True, [], 16384, {}, threading.Event()))
self.assertEqual(service.repeat_stop_tokens, 256)
self.assertEqual(events[-1][1]['finish'], 'length')
@@ -145,6 +152,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
def test_recovery_is_once_and_never_extends_output_limit(self):
engine = RecordingEngine(self.tok, [self.loop, self.loop])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = True
events = list(service.run(self.ids, True, [], 16384, {}, threading.Event()))
self.assertEqual(len(engine.calls), 2)
@@ -155,6 +163,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
def test_repeated_final_text_does_not_change_sampler_or_prompt(self):
engine = RecordingEngine(self.tok, [self.loop])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = True
events = list(service.run(self.ids, False, [], 40000, {}, threading.Event()))
self.assertEqual(len(engine.calls), 1)
@@ -163,6 +172,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
def test_missing_native_high_hint_skips_recovery(self):
engine = RecordingEngine(self.tok, [self.loop])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = True
ids = self.tok.encode('<|im_start|>user\nTask<|im_end|>\n', parse_special=True)
events = list(service.run(ids, True, [], 40000, {}, threading.Event()))
@@ -172,6 +182,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
def test_cancel_during_reasoning_does_not_start_recovery(self):
engine = RecordingEngine(self.tok, [self.loop, 'Unused'])
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = True
cancel = threading.Event()
events = []
@@ -187,6 +198,7 @@ class ReasoningLoopRecovery(unittest.TestCase):
engine = RecordingEngine(self.tok, [self.loop, 'Done.\n</think>\nAnswer.'])
engine.batch = 2
service = Service(engine, self.tok, None)
+ service.answer_room = False # these cases test the loop guards at the cap itself
service.reasoning_loop_recovery = True
events = list(service.run(self.ids, True, [], 40000, {}, threading.Event()))
self.assertEqual(events[-1][1]['reasoning_recoveries'], 1)
diff --git a/serve/test_server.py b/serve/test_server.py
index 0e566ef5c..ba1361d8d 100644
--- a/serve/test_server.py
+++ b/serve/test_server.py
@@ -3809,6 +3809,34 @@ class ThinkingBudget(unittest.TestCase):
self.assertEqual(text, ThinkingEngine.ANSWER)
self.assertEqual(evs[-2]["delta"]["stop_reason"], "end_turn")
+ def test_a_cap_that_the_thinking_would_eat_leaves_room_to_answer(self):
+ """0.1.45: max_tokens counts the thinking; a thinking that would fill it gets closed with a quarter left (the
+ agent harness sends max_completion_tokens 8192; the pagoda's thinking took all of it and no answer came)."""
+ self.engine.THOUGHT = "Let me think step by step about two plus two. " * 40 # 1,800 reasoning tokens
+ code, b = self.post("/v1/chat/completions", {"model": "m", "max_tokens": 1024,
+ "messages": [{"role": "user", "content": "2+2?"}]})
+ self.assertEqual(code, 200, b)
+ msg = b["choices"][0]["message"]
+ self.assertEqual(msg["content"], ThinkingEngine.ANSWER)
+ self.assertEqual(b["choices"][0]["finish_reason"], "stop")
+ self.assertLess(len(msg["reasoning_content"]), 1024 - 256)
+ self.assertEqual(len(self.engine.prompts), 2) # the wrap-up continuation
+
+ def test_the_answer_room_has_its_limits(self):
+ self.engine.THOUGHT = "Let me think step by step about two plus two. " * 40
+ code, b = self.post("/v1/chat/completions", {"model": "m", "max_tokens": 300, # below the floor: as before
+ "messages": [{"role": "user", "content": "2+2?"}]})
+ self.assertEqual(b["choices"][0]["finish_reason"], "length")
+ self.assertFalse(b["choices"][0]["message"]["content"])
+ self.svc.answer_room = False # the config's "answer_room": false
+ code, b = self.post("/v1/chat/completions", {"model": "m", "max_tokens": 1024,
+ "messages": [{"role": "user", "content": "2+2?"}]})
+ self.assertEqual(b["choices"][0]["finish_reason"], "length")
+ self.svc.answer_room = True
+ code, b = self.post("/v1/chat/completions", {"model": "m", "max_tokens": 1024, "reasoning_effort": "none",
+ "messages": [{"role": "user", "content": "2+2?"}]})
+ self.assertEqual(len(self.engine.prompts), 3) # thinking off: no budget, one prompt each
+
def test_a_budget_the_thinking_stays_under(self):
code, b = self.openai(reasoning_budget_tokens=10_000)
self.assertEqual(b["choices"][0]["message"]["reasoning_content"], ThinkingEngine.THOUGHT)