Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: max_tokens counts the thinking - without a budget, thinking is closed early enough to leave a quarter of the cap for the answer (cap 8192 cut the thinking off with no answer); config answer_room:false opts out; tests
Niko1221d0e990ba7840
diff --git a/serve/server.py b/serve/server.pyindex a74a193c0..252d93681 100644--- a/serve/server.py+++ b/serve/server.py@@ -147,6 +147,7 @@ VISION_START = "<|vision_start|>"# broken state that answers one token forever (an issue saw 36,689 tokens of "!"). The config's "repeat_stop_tokens"# sets it; 0 turns it off.REPEAT_STOP_TOKENS = 256+ANSWER_ROOM_MIN_CAP = 1024 # 0.1.45: from this max_tokens on, thinking without a budget is closed early enough to leave room to answerANSWER_RESERVE_MIN = 512 # #984: the tokens kept for the answer when a thinking budget is cut down to fit max_tokens# #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)# #1053 (opt-in "reasoning_close_retry": true): a reply that ends on its stop token still inside <think>, with no answer@@ -3076,6 +3077,7 @@ class Service:self.literals = literal_tags(getattr(tokenizer, "control_tokens", ())) # texts that stay text inside a messageself.fit_max_tokens = fit_max_tokens # --fit-max-tokens: clamp the output cap instead of 400self.aliases: list[str] = [] # #297: other names of the model (the config's `aliases`)+ self.answer_room = True # 0.1.45: see ANSWER_ROOM_MIN_CAP (the config's "answer_room": false turns it off)self.reasoning_close_retry = False # #1053 (opt-in): close the thinking once when a reply ends inside itself.codex_compaction_cache = False # #924 (opt-in): a Codex compaction is rendered with its conversation's toolsself.codex_thread_titles = False # #923 (opt-in): answer Codex's thread-title turns without the engine@@ -4137,6 +4139,15 @@ class Service:sampling = {**defaults, **req_values}# #123: read after the merge, so a budget shared through POST /settings is seen like the other keysbudget = self.reasoning_budget(sampling) if thinking else None+ if thinking and not budget and self.answer_room and max_new and max_new >= ANSWER_ROOM_MIN_CAP:+ # 0.1.45: max_tokens counts the thinking too. A client that caps the reply at 8192 (the desktop agent's+ # harness sends max_completion_tokens: 8192 on every request) and a task the model thinks long about got+ # the cut-off in the middle of the thinking and NO answer. So thinking is closed early enough to leave+ # the answer its quarter of the cap (the budget's wrap-up: a short closing sentence, </think>, the answer).+ # A request with a budget of its own, or max_tokens below ANSWER_ROOM_MIN_CAP, is left as it was.+ budget = max_new - min(max(ANSWER_RESERVE_MIN, max_new // 4), max_new // 2)+ if max_new <= 32768: print(f"[strata] thinking limited to {budget} of max_tokens {max_new} so the answer has room "+ "(\"answer_room\": false in the config turns this off)", flush=True)if budget and max_new and max_new > 0:# #984: a budget at or near max_tokens never fires before the reply is cut off while still thinking, so# it has no answer. Close the thinking early enough to leave room for one.@@ -6844,6 +6855,7 @@ def main() -> int:fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)if a.engine == "strata": # the model card's sampling when nothing else is said (see MODEL_CARD_SAMPLING)svc.model_arch = cfg.get("arch") or "qwen4exp"+ svc.answer_room = cfg.get("answer_room") is not Falsesvc.reasoning_close_retry = cfg.get("reasoning_close_retry") is True # #1053: opt-in, off by defaultif cfg.get("images"): # image generation (serve/images.py), set up by setup --imagesicfg = {k: (os.path.abspath(os.path.join(cfg.get("cwd") or ".", v))diff --git a/serve/test_reasoning_loop_recovery.py b/serve/test_reasoning_loop_recovery.pyindex b53ee56ac..c00fc1bee 100644--- a/serve/test_reasoning_loop_recovery.py+++ b/serve/test_reasoning_loop_recovery.py@@ -58,6 +58,7 @@ class ReasoningLoopRecovery(unittest.TestCase):engine = RecordingEngine(self.tok, [self.loop,'I have verified the solution.\n</think>\nComplete solution.'])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = enabledevents = list(service.run(self.ids, True, [], 40000, {}, threading.Event()))return engine, events@@ -66,6 +67,7 @@ class ReasoningLoopRecovery(unittest.TestCase):engine = RecordingEngine(self.tok, [self.loop,'Verified.\n</think>\nComplete solution.'])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = Truerequest = {'temperature': .2, 'presence_penalty': 0, 'top_p': .95, 'top_k': 64, 'seed': 2}list(service.run(self.ids, True, [], 40000, request, threading.Event()))@@ -81,6 +83,7 @@ class ReasoningLoopRecovery(unittest.TestCase):def test_frozen_cohort_sampler_is_unchanged_during_recovery(self):engine = RecordingEngine(self.tok, [self.loop, 'Done.\n</think>\nComplete solution.'])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = Truesampler = {'temperature': 1.0, 'top_p': .95, 'top_k': 20, 'presence_penalty': 1.5, 'seed': 2}list(service.run(self.ids, True, [], 40000, sampler, threading.Event()))@@ -91,6 +94,7 @@ class ReasoningLoopRecovery(unittest.TestCase):def test_a_lower_top_p_and_top_k_are_never_raised_or_lowered(self):engine = RecordingEngine(self.tok, [self.loop, 'Verified.\n</think>\nComplete solution.'])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = "recover"list(service.run(self.ids, True, [], 40000, {'temperature': 0, 'top_p': .5, 'top_k': 5}, threading.Event()))self.assertEqual((engine.sampling_calls[1]['top_p'], engine.sampling_calls[1]['top_k']), (.5, 5))@@ -110,6 +114,7 @@ class ReasoningLoopRecovery(unittest.TestCase):def test_stop_mode_ends_the_reply_at_the_loop(self):engine = RecordingEngine(self.tok, [self.loop])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = "stop"events = list(service.run(self.ids, True, [], 40000, {}, threading.Event()))self.assertEqual(len(engine.calls), 1) # no second pass@@ -121,6 +126,7 @@ class ReasoningLoopRecovery(unittest.TestCase):"""#1753: the word count never sees the digits, so only the tail's period ends the reply."""engine = RecordingEngine(self.tok, ['9^104 = ' + '58747693387' * 3000])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = "stop"events = list(service.run(self.ids, True, [], 40000, {}, threading.Event()))self.assertEqual(len(engine.calls), 1) # no second pass@@ -136,6 +142,7 @@ class ReasoningLoopRecovery(unittest.TestCase):def test_default_single_token_guard_does_not_detect_passage_loop(self):engine = RecordingEngine(self.tok, [self.loop])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfevents = list(service.run(self.ids, True, [], 16384, {}, threading.Event()))self.assertEqual(service.repeat_stop_tokens, 256)self.assertEqual(events[-1][1]['finish'], 'length')@@ -145,6 +152,7 @@ class ReasoningLoopRecovery(unittest.TestCase):def test_recovery_is_once_and_never_extends_output_limit(self):engine = RecordingEngine(self.tok, [self.loop, self.loop])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = Trueevents = list(service.run(self.ids, True, [], 16384, {}, threading.Event()))self.assertEqual(len(engine.calls), 2)@@ -155,6 +163,7 @@ class ReasoningLoopRecovery(unittest.TestCase):def test_repeated_final_text_does_not_change_sampler_or_prompt(self):engine = RecordingEngine(self.tok, [self.loop])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = Trueevents = list(service.run(self.ids, False, [], 40000, {}, threading.Event()))self.assertEqual(len(engine.calls), 1)@@ -163,6 +172,7 @@ class ReasoningLoopRecovery(unittest.TestCase):def test_missing_native_high_hint_skips_recovery(self):engine = RecordingEngine(self.tok, [self.loop])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = Trueids = self.tok.encode('<|im_start|>user\nTask<|im_end|>\n', parse_special=True)events = list(service.run(ids, True, [], 40000, {}, threading.Event()))@@ -172,6 +182,7 @@ class ReasoningLoopRecovery(unittest.TestCase):def test_cancel_during_reasoning_does_not_start_recovery(self):engine = RecordingEngine(self.tok, [self.loop, 'Unused'])service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = Truecancel = threading.Event()events = []@@ -187,6 +198,7 @@ class ReasoningLoopRecovery(unittest.TestCase):engine = RecordingEngine(self.tok, [self.loop, 'Done.\n</think>\nAnswer.'])engine.batch = 2service = Service(engine, self.tok, None)+ service.answer_room = False # these cases test the loop guards at the cap itselfservice.reasoning_loop_recovery = Trueevents = list(service.run(self.ids, True, [], 40000, {}, threading.Event()))self.assertEqual(events[-1][1]['reasoning_recoveries'], 1)diff --git a/serve/test_server.py b/serve/test_server.pyindex 0e566ef5c..ba1361d8d 100644--- a/serve/test_server.py+++ b/serve/test_server.py@@ -3809,6 +3809,34 @@ class ThinkingBudget(unittest.TestCase):self.assertEqual(text, ThinkingEngine.ANSWER)self.assertEqual(evs[-2]["delta"]["stop_reason"], "end_turn")+ def test_a_cap_that_the_thinking_would_eat_leaves_room_to_answer(self):+ """0.1.45: max_tokens counts the thinking; a thinking that would fill it gets closed with a quarter left (the+ agent harness sends max_completion_tokens 8192; the pagoda's thinking took all of it and no answer came)."""+ self.engine.THOUGHT = "Let me think step by step about two plus two. " * 40 # 1,800 reasoning tokens+ code, b = self.post("/v1/chat/completions", {"model": "m", "max_tokens": 1024,+ "messages": [{"role": "user", "content": "2+2?"}]})+ self.assertEqual(code, 200, b)+ msg = b["choices"][0]["message"]+ self.assertEqual(msg["content"], ThinkingEngine.ANSWER)+ self.assertEqual(b["choices"][0]["finish_reason"], "stop")+ self.assertLess(len(msg["reasoning_content"]), 1024 - 256)+ self.assertEqual(len(self.engine.prompts), 2) # the wrap-up continuation++ def test_the_answer_room_has_its_limits(self):+ self.engine.THOUGHT = "Let me think step by step about two plus two. " * 40+ code, b = self.post("/v1/chat/completions", {"model": "m", "max_tokens": 300, # below the floor: as before+ "messages": [{"role": "user", "content": "2+2?"}]})+ self.assertEqual(b["choices"][0]["finish_reason"], "length")+ self.assertFalse(b["choices"][0]["message"]["content"])+ self.svc.answer_room = False # the config's "answer_room": false+ code, b = self.post("/v1/chat/completions", {"model": "m", "max_tokens": 1024,+ "messages": [{"role": "user", "content": "2+2?"}]})+ self.assertEqual(b["choices"][0]["finish_reason"], "length")+ self.svc.answer_room = True+ code, b = self.post("/v1/chat/completions", {"model": "m", "max_tokens": 1024, "reasoning_effort": "none",+ "messages": [{"role": "user", "content": "2+2?"}]})+ self.assertEqual(len(self.engine.prompts), 3) # thinking off: no budget, one prompt each+def test_a_budget_the_thinking_stays_under(self):code, b = self.openai(reasoning_budget_tokens=10_000)self.assertEqual(b["choices"][0]["message"]["reasoning_content"], ThinkingEngine.THOUGHT)