Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: a request with no sampling gets the model card's (thinking 1.0/0.95/20; not thinking 0.7/0.8/20 + presence 1.5; Qwen3.6 too); greedy only when asked (#2685)

Niko1221a8091f4d2e7e
diff --git a/docs/DETAILS.md b/docs/DETAILS.md
index 045be3021..aa0ec7a39 100644
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@ -1272,8 +1272,13 @@ to 4; Anthropic `stop_sequences`): the answer ends before the first one, which i
there (`finish_reason` "stop"; `stop_reason` "stop_sequence" with `stop_sequence` set to the one found); with the default adaptive expert tier a sampled result
is not reproducible run to run - for seed-reproducible output add `--adapt-every 100000` (static residency) to the
engine arguments. The run config's optional `sampling` block sets the defaults for requests that leave the fields out
-(`"sampling": {"temperature": 1.0, "top_p": 0.95, "top_k": 20}`); a request's own fields always win, and with no
-block at all a request without sampling keys decodes greedy. The penalties (`presence_penalty`, `frequency_penalty`,
+(`"sampling": {"temperature": 1.0, "top_p": 0.95, "top_k": 20}`); a request's own fields always win. **With no
+block at all (0.1.45) a request without sampling keys gets the sampling the model card recommends**, picked by the
+request's thinking state: Qwen3.8-Flash-Next (and Gyro) thinking temperature 1.0, top_p 0.95, top_k 20; not thinking temperature 0.7,
+top_p 0.8, top_k 20, presence_penalty 1.5. Qwen3.6-35B-A3B thinking 1.0 / 0.95 / 20 with presence_penalty 1.5, not thinking
+0.7 / 0.8 / 20 with 1.5 (min_p 0 and repetition_penalty 1 are "off"). Order of precedence: the request, the web page's shared Chat settings,
+the config's `sampling` block, the card. Greedy is only for who asks for it: `temperature: 0` in the request, or `"sampling": {"temperature": 0}` in the config
+(before 0.1.45 a config without a block ran greedy, and greedy decoding is what makes small local models loop; #2685). The penalties (`presence_penalty`, `frequency_penalty`,
`repetition_penalty`, with `penalty_last_n` capping how many recent tokens they count over, default 64 when any
penalty is set; llama.cpp's names `repeat_penalty` and `repeat_last_n` work too, in the config's `sampling` block and in a request, #1819) ride the same path; they count the tokens the request has consumed, so a repetition penalty
suppresses what the model itself just said, not the prompt alone. Since engine 0.1.19 they apply to every token
diff --git a/serve/runconfig.py b/serve/runconfig.py
index 472b00212..842554ab7 100644
--- a/serve/runconfig.py
+++ b/serve/runconfig.py
@@ -18,7 +18,7 @@ from pathlib import Path
# of the "sampling" block, ("arg", flag) for an engine option kept in "args".
EDITABLE = [
("sampling.temperature", ("sampling", "num>=0"),
- "Default temperature for requests that send none (0 = greedy, the default without a sampling block)"),
+ "Default temperature for requests that send none (0 = greedy; without it the model card's: 1.0 thinking, 0.7 not)"),
("sampling.top_p", ("sampling", "0<x<=1"), "Default top_p for requests that send none"),
("sampling.top_k", ("sampling", "1..64"), "Default top_k for requests that send none (1-64)"),
("sampling.min_p", ("sampling", "0<=x<=1"), "Default min_p for requests that send none"),
diff --git a/serve/server.py b/serve/server.py
index 3ca03b03b..a74a193c0 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -3080,6 +3080,7 @@ class Service:
self.codex_compaction_cache = False # #924 (opt-in): a Codex compaction is rendered with its conversation's tools
self.codex_thread_titles = False # #923 (opt-in): answer Codex's thread-title turns without the engine
self.sampling_defaults = dict(sampling_defaults or {}) # the run config's `sampling` block
+ self.model_arch = None # the config's family ("qwen4exp", "qwen35moe"): None = no model-card defaults
self.shared = {} # the web app's Chat settings for every client (POST /settings)
self.shared_path = None # where they are kept between starts (next to the config)
self.config_effort = None # #1641: the config's "reasoning_effort": the level for requests with none
@@ -4128,7 +4129,9 @@ class Service:
def _run(self, ids, thinking, tools, max_new, sampling, cancel, force=None) -> Iterator[tuple[str, object]]:
"""Yields ("event", Event) as text arrives, then ("done", {"finish": .., "completion_tokens": ..}).
`force` (forced_call): the opening of the call the reply must make - see prepare()."""
- defaults = {**self.sampling_defaults, **self.shared} # the config's, then the Chat settings shared with apps
+ # the model card's sampling for this thinking state, then the config's block, then the Chat settings shared with
+ # apps; the request's own fields win over all of them
+ defaults = {**model_card_sampling(self.model_arch, thinking), **self.sampling_defaults, **self.shared}
if defaults: # the request's own fields win (explicit 0 stays greedy)
req_values = {k: v for k, v in (sampling or {}).items() if v is not None}
sampling = {**defaults, **req_values}
@@ -5664,7 +5667,7 @@ def make_handler(svc: Service):
if not svc.loaded() and not getattr(svc.engine, "unloaded", False):
self._json(503, {"error": {"message": "the engine is not running"}})
return
- defaults = {**svc.sampling_defaults, **svc.shared}
+ defaults = {**model_card_sampling(svc.model_arch, True), **svc.sampling_defaults, **svc.shared}
names = {"repetition_penalty": "repeat_penalty", "penalty_last_n": "repeat_last_n"}
params = {names.get(k, k): v for k, v in defaults.items()
if k in ("temperature", "top_p", "top_k", "min_p", "seed", "repetition_penalty",
@@ -6667,6 +6670,25 @@ def check_request_sampling(req) -> None:
FAMILY_MODEL_NAMES = {"qwen4exp": "qwen3.8-flash-next", "qwen35moe": "qwen3.6-35b-a3b"}
+# The sampling each model card recommends ("Best Practices"), used when neither the request, the Chat settings nor the
+# run config's `sampling` block says otherwise. Greedy is only for a request or config that asks for temperature 0.
+# min_p 0, repetition_penalty 1 and presence_penalty 0 are the engine's own "off", so they are not listed.
+MODEL_CARD_SAMPLING = {
+ # Qwen3.8-Flash-Next (and Gyro, the same base model)
+ "qwen4exp": {"thinking": {"temperature": 1.0, "top_p": 0.95, "top_k": 20},
+ "instruct": {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "presence_penalty": 1.5}},
+ # Qwen3.6-35B-A3B: the general-task sets (the card's precise-coding set is temperature 0.6, presence_penalty 0)
+ "qwen35moe": {"thinking": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "presence_penalty": 1.5},
+ "instruct": {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "presence_penalty": 1.5}},
+}
+
+
+def model_card_sampling(arch, thinking) -> dict:
+ """The model card's sampling for the family `arch` and the thinking state ({} for an unknown or unset family)."""
+ sets = MODEL_CARD_SAMPLING.get(arch) if arch else None
+ return dict(sets["thinking" if thinking else "instruct"]) if sets else {}
+
+
def default_model_name(cfg: dict) -> str:
"""The config's "model_name", else its family's name ("arch": qwen4exp, qwen35moe), else Flash-Next's."""
return cfg.get("model_name") or FAMILY_MODEL_NAMES.get(cfg.get("arch") or "qwen4exp", "qwen3.8-flash-next")
@@ -6820,6 +6842,8 @@ def main() -> int:
model_name=default_model_name(cfg), vision=vision,
sampling_defaults=sampling_defaults,
fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)
+ if a.engine == "strata": # the model card's sampling when nothing else is said (see MODEL_CARD_SAMPLING)
+ svc.model_arch = cfg.get("arch") or "qwen4exp"
svc.reasoning_close_retry = cfg.get("reasoning_close_retry") is True # #1053: opt-in, off by default
if cfg.get("images"): # image generation (serve/images.py), set up by setup --images
icfg = {k: (os.path.abspath(os.path.join(cfg.get("cwd") or ".", v))
diff --git a/serve/test_model_card_sampling.py b/serve/test_model_card_sampling.py
new file mode 100644
index 000000000..ee491506b
--- /dev/null
+++ b/serve/test_model_card_sampling.py
@@ -0,0 +1,108 @@
+"""serve/test_model_card_sampling.py - a request that sends no sampling gets the model card's, not greedy (mock engine).
+
+Qwen's "Best Practices": thinking mode temperature 1.0 / top_p 0.95 / top_k 20, non-thinking 0.7 / 0.8 / 20 with
+presence_penalty 1.5. The request's own fields win, then the Chat settings shared with apps, then the config's
+`sampling` block, then the card; temperature 0 asked for is still greedy.
+
+ python -m unittest serve.test_model_card_sampling -v
+"""
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+import urllib.request
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from serve.frontend import ChatTemplate # noqa: E402
+from serve.server import ByteTokenizer, MockEngine, Service, StrataEngine, model_card_sampling, serve # noqa: E402
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class Recording(MockEngine):
+ def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+ self.sampling = dict(sampling or {})
+ yield from super().generate(ids, max_new, sampling, cancel, embeddings)
+
+
+class ModelCardSampling(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tok = ByteTokenizer()
+ cls.engine = Recording(cls.tok, "<think>\nhm\n</think>\n\nok", max_context=8192)
+ cls.svc = Service(cls.engine, cls.tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+ cls.httpd = serve(cls.svc, port=0)
+ cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}"
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.httpd.shutdown()
+ cls.httpd.server_close()
+
+ def setUp(self):
+ self.svc.model_arch = "qwen4exp"
+ self.svc.sampling_defaults = {}
+ self.svc.shared = {}
+ self.svc.config_effort = None
+
+ def tearDown(self):
+ self.svc.model_arch = None
+
+ def keys(self, **body):
+ body = {"model": "m", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 8, **body}
+ self.engine.sampling = {}
+ req = urllib.request.Request(self.base + "/v1/chat/completions", data=json.dumps(body).encode(),
+ headers={"Content-Type": "application/json"})
+ urllib.request.urlopen(req, timeout=10).read()
+ return StrataEngine.sampling_keys(self.engine.sampling)
+
+ def test_tables(self):
+ self.assertEqual(model_card_sampling("qwen4exp", True), {"temperature": 1.0, "top_p": 0.95, "top_k": 20})
+ self.assertEqual(model_card_sampling("qwen4exp", False),
+ {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "presence_penalty": 1.5})
+ self.assertEqual(model_card_sampling("qwen35moe", True)["presence_penalty"], 1.5)
+ self.assertEqual(model_card_sampling(None, True), {})
+ self.assertEqual(model_card_sampling("unknown", False), {})
+
+ def test_thinking_default(self):
+ k = self.keys()
+ self.assertIn("temperature=1.0", k)
+ self.assertIn("top_p=0.95", k)
+ self.assertIn("top_k=20", k)
+ self.assertNotIn("penalty_present", k)
+
+ def test_non_thinking_default(self):
+ k = self.keys(reasoning_effort="none")
+ self.assertIn("temperature=0.7", k)
+ self.assertIn("top_p=0.8", k)
+ self.assertIn("penalty_present=1.5", k)
+
+ def test_request_wins_and_zero_is_greedy(self):
+ k = self.keys(temperature=0)
+ self.assertNotIn("temperature", k)
+ self.assertIn("top_k=20", k)
+ k = self.keys(temperature=0.3, top_p=0.5)
+ self.assertIn("temperature=0.3", k)
+ self.assertIn("top_p=0.5", k)
+
+ def test_config_block_and_shared_settings_win_over_the_card(self):
+ self.svc.sampling_defaults = {"temperature": 0.6, "top_k": 40}
+ k = self.keys()
+ self.assertIn("temperature=0.6", k)
+ self.assertIn("top_k=40", k)
+ self.assertIn("top_p=0.95", k)
+ self.svc.shared = {"temperature": 0.2}
+ self.assertIn("temperature=0.2", self.keys())
+ self.svc.sampling_defaults = {"temperature": 0}
+ self.svc.shared = {}
+ self.assertNotIn("temperature", self.keys()) # a config that asks for greedy gets it
+
+ def test_no_family_no_defaults(self):
+ self.svc.model_arch = None
+ self.assertEqual(self.keys(), "")
+
+
+if __name__ == "__main__":
+ unittest.main()