Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: shorten max_tokens to the room left instead of answering 400 (all three dialects)

Niko1221f31b6efd68b5
diff --git a/docs/DETAILS.md b/docs/DETAILS.md
index d46ee5a5d..39b29fc50 100644
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@ -921,17 +921,18 @@ print(r.choices[0].message.content)
```
Set `limit.context` to the context you chose in setup: OpenCode compacts the conversation before it gets there.
- Keep `limit.output` well under it: a request whose prompt plus `max_tokens` runs past the context is refused (see
- **Context** below), or add `"fit_max_tokens": true` to `strata-<model>.json`. For a hard cap on the thinking, add
+ Keep `limit.output` well under it: a request whose prompt plus `max_tokens` runs past the context has its `max_tokens`
+ shortened to the room left (see **Context** below). For a hard cap on the thinking, add
`"reasoning_budget_tokens": N` to `strata-<model>.json` (see above).
- **Claude Code** (Strata 0.1.17 or newer): set `ANTHROPIC_BASE_URL=http://127.0.0.1:8080` and
`ANTHROPIC_MODEL` to a Claude model name it knows (it refuses names it doesn't; Strata ignores the name), plus any
`ANTHROPIC_AUTH_TOKEN` (or your `api_key`, if you set one).
- **Codex CLI** (0.1.39): see [the Responses API](#the-responses-api-and-codex-cli) below.
- **Context.** Chosen in setup (8K-262K). Requests longer than that are refused, never silently cut. A request whose
- `max_tokens` would run past the context is refused too (400); agents that always ask for their full output cap
- can instead get it shortened to the room left: add `"fit_max_tokens": true` to `strata-<model>.json` (or pass
- `--fit-max-tokens` to `serve/server.py`). A prompt that leaves no room at all is still refused.
+ `max_tokens` would run past the context is answered with `max_tokens` shortened to the room left (agents such as
+ Claude Code ask for 32,000 on every request); the thinking budget follows the shortened cap. For the strict llama.cpp
+ behaviour (400) set `"fit_max_tokens": false` in `strata-<model>.json` or pass `--no-fit-max-tokens` to
+ `serve/server.py`. A prompt that leaves no room at all is always refused.
- **Model aliases** (0.1.32). `"aliases": ["qwen", "local-model"]` in `strata-<model>.json` lists the model under
those names too in `/v1/models` (each with its own `id`, and in the model's `aliases`), like llama-server's
`--alias`; a request naming one is answered under that name. Any other name is still served, as before.
diff --git a/serve/runconfig.py b/serve/runconfig.py
index 842554ab7..62aea1bb7 100644
--- a/serve/runconfig.py
+++ b/serve/runconfig.py
@@ -23,7 +23,7 @@ EDITABLE = [
("sampling.top_k", ("sampling", "1..64"), "Default top_k for requests that send none (1-64)"),
("sampling.min_p", ("sampling", "0<=x<=1"), "Default min_p for requests that send none"),
("reasoning_budget_tokens", "int>=0", "Cap the thinking of every request at this many tokens (0 or empty: no cap)"),
- ("fit_max_tokens", "bool", "Shorten a max_tokens that does not fit the context instead of answering 400"),
+ ("fit_max_tokens", "bool", "Shorten a max_tokens that does not fit the context (on by default); off: answer 400 like llama.cpp"),
("anthropic_thinking", ("enum", ["model", "on_request"]),
"Anthropic requests that do not ask for thinking: think as the model does (model) or not (on_request)"),
("effort_position", ("enum", ["start", "end"]),
diff --git a/serve/server.py b/serve/server.py
index 229e30b91..f1edd6b9c 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -14,7 +14,10 @@ Images (optional, when the config has a "vision" entry): OpenAI image_url parts
data, http(s) URLs or local file paths) go through `strata-vision` (the model's mmproj file) and reach the engine as
embeddings (`GENI`). JPEG/PNG/BMP/GIF go straight in; WebP, TIFF, AVIF, ... (agents like omp send WebP) are
converted to PNG first with Pillow.
-Requests whose prompt plus max tokens exceed the engine's context are REJECTED with 400, never truncated.
+A request whose prompt alone does not fit the engine's context is REJECTED with 400, never truncated. When the prompt
+fits but prompt plus max tokens does not (Claude Code asks for 32000, the Harness for 16384), max tokens is shortened to
+the room left (less CTX_SLACK) and the thinking budget follows it; `--no-fit-max-tokens` / `"fit_max_tokens": false`
+brings back the strict 400 of llama.cpp.
An unset (or 0, or -1) max tokens means "unlimited": whatever the prompt leaves of the context.
The engine boundary is `Engine.generate(prompt_ids, max_new, sampling, cancel) -> iterator of token ids`.
@@ -3271,10 +3274,10 @@ def slot_save_dir(value, base: str | None = None) -> str:
class Service:
def __init__(self, engine: Engine, tokenizer, template: ChatTemplate, model_name: str = "qwen3.8-flash-next",
vision: Vision | None = None, sampling_defaults: dict | None = None,
- fit_max_tokens: bool = False):
+ fit_max_tokens: bool = True):
self.engine, self.tok, self.template, self.model, self.vision = engine, tokenizer, template, model_name, vision
self.literals = literal_tags(getattr(tokenizer, "control_tokens", ())) # texts that stay text inside a message
- self.fit_max_tokens = fit_max_tokens # --fit-max-tokens: clamp the output cap instead of 400
+ self.fit_max_tokens = fit_max_tokens # default: clamp the output cap to the room left; False (--no-fit-max-tokens) = 400
self.aliases: list[str] = [] # #297: other names of the model (the config's `aliases`)
self.answer_room = True # 0.1.45: see ANSWER_ROOM_MIN_CAP (the config's "answer_room": false turns it off)
self.reasoning_close_retry = False # #1053 (opt-in): close the thinking once when a reply ends inside it
@@ -4298,7 +4301,9 @@ class Service:
f"({ctx}); requests are never truncated. Send a smaller "
f"max_tokens (at most {max(0, room)} here), or add \"fit_max_tokens\": true to the "
"model's strata-<model>.json to shorten it to the room left (#545)")
- max_new = max(1, room) # --fit-max-tokens: a shorter completion beats a 400
+ print(f"[strata] max_tokens {max_new} shortened to {max(1, room)}: the prompt ({len(ids)} tokens) leaves that much "
+ f"of the context ({ctx}); \"fit_max_tokens\": false answers 400 instead", flush=True)
+ max_new = max(1, room) # a shorter completion beats a 400 (the default)
if images:
# The request's images in one file for GENI (~10 MB a picture), written once nothing above refuses the
# request: one refused after it (the engine starting, no room) left it in the vision directory for good,
@@ -7426,7 +7431,10 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
ap.add_argument("--open", action="store_true", help="open the local page in the browser once the model is ready")
ap.add_argument("--fit-max-tokens", action="store_true",
help="clamp max_tokens to the remaining context instead of rejecting the request "
- "(default: reject with 400, like llama.cpp; also \"fit_max_tokens\": true in the config)")
+ "(the default; kept so older scripts keep working)")
+ ap.add_argument("--no-fit-max-tokens", action="store_true",
+ help="answer 400 when prompt + max_tokens does not fit the context, like llama.cpp "
+ "(also \"fit_max_tokens\": false in the config). A prompt that alone does not fit is always 400")
ap.add_argument("--api-key", default=os.environ.get("STRATA_API_KEY", ""),
help="require this key on /v1/* (Authorization: Bearer ... or x-api-key); also $STRATA_API_KEY. "
"Several keys: separate them with commas (key1,key2), as llama.cpp does")
@@ -7655,7 +7663,7 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
svc = Service(engine, tok, ChatTemplate(tpl if tpl.exists() else ROOT / "serve/chat_template.jinja"),
model_name=default_model_name(cfg), vision=vision,
sampling_defaults=sampling_defaults,
- fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)
+ fit_max_tokens=a.fit_max_tokens or (not a.no_fit_max_tokens and cfg.get("fit_max_tokens") is not False))
svc.launch_attestation = launch_attestation
svc.offline = a.offline
svc.max_queue = max_queue
diff --git a/serve/test_context_overflow.py b/serve/test_context_overflow.py
index 8c68b77e8..ff26c0a12 100644
--- a/serve/test_context_overflow.py
+++ b/serve/test_context_overflow.py
@@ -85,6 +85,23 @@ class ContextOverflow(unittest.TestCase):
else:
self.assertEqual(set(error), {"type", "message"})
+ def test_default_service_shortens_an_overshooting_budget_in_every_dialect(self):
+ # the Service default (fit_max_tokens on): a prompt that fits is answered, streaming or not; the prompt that
+ # alone leaves no room is the 400 of the strict test above
+ self.assertTrue(Service(self.engine, ByteTokenizer(), self.svc.template).fit_max_tokens)
+ self.svc.fit_max_tokens = True
+ for path, key in APIS:
+ for stream in (False, True):
+ with self.subTest(path=path, stream=stream):
+ if not stream: # (a 200 stream is SSE, not the JSON post() reads)
+ with mock.patch.object(self.engine, "generate", wraps=self.engine.generate) as generate:
+ status, _, body = self.post(path, key, 1024, CTX, stream)
+ self.assertEqual(status, 200, body)
+ self.assertEqual(generate.call_args.args[1], CTX - CTX_SLACK - 1024)
+ status, _, body = self.post(path, key, CTX - CTX_SLACK, CTX, stream)
+ self.assertEqual(status, 400, body)
+ self.assertIn("no room to answer", body["error"]["message"])
+
def test_one_token_of_room_still_answers_but_zero_is_rejected(self):
for path, key in APIS:
for fit in (False, True):
diff --git a/serve/test_monitor.py b/serve/test_monitor.py
index 0c82cb0b5..6c64b1115 100644
--- a/serve/test_monitor.py
+++ b/serve/test_monitor.py
@@ -109,7 +109,11 @@ class Monitor(unittest.TestCase):
self.assertNotIn("response", record)
def test_error_and_engine_death_are_not_successes(self):
+ # a request the server refuses is an error record: the strict mode (fit_max_tokens off) refuses a max_tokens
+ # that overshoots the context, the default shortens it (serve/test_server.py MaxTokens)
+ self.svc.fit_max_tokens = False
self.assertEqual(self.chat(max_tokens=20000)[0], 400)
+ self.svc.fit_max_tokens = True
self.assertEqual(self.latest()["state"], "error")
for api in ("openai", "anthropic"):
for stream in (False, True):
diff --git a/serve/test_server.py b/serve/test_server.py
index 9dd296d5b..b97897099 100644
--- a/serve/test_server.py
+++ b/serve/test_server.py
@@ -22,7 +22,7 @@ from unittest import mock
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from serve.frontend import ChatTemplate, literal_tags, mark_think_literals, unmark_think_literals # noqa: E402
-from serve.server import (CTX_SLACK, ByteTokenizer, EngineDied, EngineNonFinite, GpuBusy, MockEngine, # noqa: E402
+from serve.server import (ANSWER_RESERVE_MIN, CTX_SLACK, ByteTokenizer, EngineDied, EngineNonFinite, GpuBusy, MockEngine, # noqa: E402
PP_DONE_TAIL, Service,
StrataEngine, api_key_of, engine_args, key_matches, layer_split_value, prompt_progress,
prompt_tokens_seen,
@@ -137,14 +137,84 @@ class MaxTokens(unittest.TestCase):
self.assertEqual(self.engine.last_max_new, want)
self.assertEqual(ct, want)
- def test_explicit_budget_over_the_context_is_rejected(self):
- for api in ("openai", "anthropic"):
+ def responses(self, text="hi", **budget):
+ """-> (status, body, prompt tokens, completion tokens) for /v1/responses."""
+ s, b = self.post("/v1/responses", {"model": "m", "input": text, **budget})
+ u = b.get("usage", {}) if isinstance(b, dict) else {}
+ return s, b, u.get("input_tokens"), u.get("output_tokens")
+
+ def call_any(self, api, text="hi", **budget):
+ """The three dialects; `budget` is given in the chat/messages spelling (max_tokens)."""
+ if api == "responses":
+ if "max_tokens" in budget:
+ budget["max_output_tokens"] = budget.pop("max_tokens")
+ return self.responses(text, **budget)
+ return self.call(api, text, **budget)
+
+ def test_explicit_budget_over_the_context_is_shortened_to_the_room_left(self):
+ # Claude Code asks for max_tokens 32000 on every request: a prompt that fits must be answered, with the output
+ # cap cut to what the context has left (less CTX_SLACK), not refused.
+ for api in ("openai", "anthropic", "responses"):
+ with self.subTest(api=api):
+ s, b, pt, ct = self.call_any(api, max_tokens=CTX)
+ self.assertEqual(s, 200, b)
+ self.assertEqual(self.engine.last_max_new, CTX - CTX_SLACK - pt)
+ self.assertLessEqual(pt + ct, CTX - CTX_SLACK)
+
+ def test_budget_that_fits_exactly_or_less_is_untouched(self):
+ for api in ("openai", "anthropic", "responses"):
+ _, _, pt, _ = self.call_any(api, max_tokens=1)
+ room = CTX - CTX_SLACK - pt
+ for budget in (50, room):
+ with self.subTest(api=api, budget=budget):
+ s, b, _, _ = self.call_any(api, max_tokens=budget)
+ self.assertEqual(s, 200, b)
+ self.assertEqual(self.engine.last_max_new, budget) # exactly what the client asked for
+ with self.subTest(api=api, budget="room + 1"):
+ s, b, _, _ = self.call_any(api, max_tokens=room + 1)
+ self.assertEqual(s, 200, b)
+ self.assertEqual(self.engine.last_max_new, room) # one over: cut by one
+
+ def test_a_prompt_that_alone_does_not_fit_is_still_a_400(self):
+ _, _, pt0, _ = self.call("openai", max_tokens=1)
+ overhead = pt0 - len("hi")
+ for api in ("openai", "anthropic", "responses"):
with self.subTest(api=api):
- s, b, _, _ = self.call(api, max_tokens=CTX)
- self.assertEqual(s, 400)
- self.assertIn("exceeds the context", b["error"]["message"])
- self.assertIn("\"fit_max_tokens\": true", b["error"]["message"]) # #545: says how to get past it
- self.assertRegex(b["error"]["message"], r"at most \d+ here")
+ s, b, _, _ = self.call_any(api, text="y" * (CTX - overhead + 50), max_tokens=CTX)
+ self.assertEqual(s, 400, b)
+ self.assertIn("exceeds the context window", b["error"]["message"])
+ self.assertIn("no room to answer", b["error"]["message"])
+ self.assertIn("never truncated", b["error"]["message"])
+
+ def test_thinking_budget_follows_the_shortened_cap(self):
+ # answer_room: thinking gets 3/4 of the cap; with the cap cut to the room left the budget is cut with it
+ import contextlib
+ import io
+ for api in ("openai", "anthropic", "responses"):
+ with self.subTest(api=api):
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out):
+ s, b, pt, _ = self.call_any(api, max_tokens=CTX)
+ self.assertEqual(s, 200, b)
+ room = CTX - CTX_SLACK - pt
+ text = out.getvalue()
+ self.assertIn(f"shortened to {room}", text)
+ if api != "anthropic": # Anthropic thinks only when asked (#278)
+ cap = room - min(max(ANSWER_RESERVE_MIN, room // 4), room // 2)
+ self.assertIn(f"thinking limited to {cap} of max_tokens {room}", text)
+
+ def test_strict_mode_keeps_the_llama_cpp_400(self):
+ self.svc.fit_max_tokens = False
+ try:
+ for api in ("openai", "anthropic", "responses"):
+ with self.subTest(api=api):
+ s, b, _, _ = self.call_any(api, max_tokens=CTX)
+ self.assertEqual(s, 400)
+ self.assertIn("exceeds the context", b["error"]["message"])
+ self.assertIn("\"fit_max_tokens\": true", b["error"]["message"]) # #545: says how to get past it
+ self.assertRegex(b["error"]["message"], r"at most \d+ here")
+ finally:
+ self.svc.fit_max_tokens = True
def test_unset_budget_with_a_near_full_prompt(self):
_, _, pt0, _ = self.call("openai", max_tokens=1)