Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Pin the multi-turn determinism probe to a backend that can be deterministic (#10234)

Daniel Han9d3fcd197e6a
diff --git a/.github/scripts/studio_smoke/multi_turn_chat.py b/.github/scripts/studio_smoke/multi_turn_chat.py
index 089b85a4a6d..a5ddd839f61 100644
--- a/.github/scripts/studio_smoke/multi_turn_chat.py
+++ b/.github/scripts/studio_smoke/multi_turn_chat.py
@@ -5,8 +5,13 @@
Two properties at once. The conversation is built so that turns 2 and 4 are only
answerable from the earlier turns, which exercises the history wiring; and the whole
-conversation is run twice at temperature 0.0 with a fixed seed, which is the only check
-anywhere that greedy decoding is reproducible.
+conversation is run twice at temperature 0.0 against a slot with speculative decoding
+off, which is the only check anywhere that greedy decoding is reproducible.
+
+The seed is sent but does no sampling work: at temperature 0 llama.cpp collapses to
+argmax and never reaches an RNG. What it does do is make the Studio send
+cache_prompt=False, which is one of the two things the comparison below needs. The
+other is the load pin that assert_reproducible_backend() checks.
This lived inline in three workflows, and being three copies is what let one of them
stop checking. On 2026-05-22 an unrelated event-loop fix (#5669) relaxed the Linux copy
@@ -21,9 +26,11 @@ between the three callers: each boots its server on its own port.
from __future__ import annotations
+import json
import os
import re
import sys
+import urllib.request
SEED = 3407
MAX_TOKENS = 80
@@ -68,6 +75,63 @@ def _server() -> tuple[str, str]:
return os.environ["BASE_URL"], os.environ["TOKEN"] # a JWT is accepted as Bearer
+def _read_backend_status() -> dict:
+ """What the server says it loaded. Split out so the assertion below is reachable
+ from a test without standing a server up, the same reason _server() exists."""
+ base, token = _server()
+ request = urllib.request.Request(
+ f"{base}/api/inference/status", headers = {"Authorization": f"Bearer {token}"}
+ )
+ with urllib.request.urlopen(request, timeout = 30) as response:
+ return json.load(response)
+
+
+def assert_reproducible_backend() -> None:
+ """Refuse to assert determinism against a server that cannot provide it.
+
+ llama-server's own README says it, about cache_prompt: "the logits are not
+ guaranteed to be bit-for-bit identical for different batch" sizes. Two things in
+ the launch the Studio derives change the batch between one replay and the next,
+ and both are per-LOAD settings, so the three workflows pin them and this checks
+ the pin took.
+
+ Speculative decoding is the one that actually bites. A non-MTP GGUF like
+ gemma-3-270m-it takes the `--spec-default` branch, which llama.cpp's arg parser
+ turns into n-gram-mod drafting, and the server logs prove it is live:
+
+ slot print_timing: id 3 | task 100 | draft acceptance = 0.46875
+ (30 accepted / 64 generated), mean len = 31.00
+
+ The draft pool is built from text the server has already seen, so the FIRST turn-1
+ request after a load drafts nothing and decodes one token at a time, while every
+ later one verifies a ~32-token draft in a single batch. Same tokens, different
+ arithmetic. That is not a near-tie a retry rides out: `check(runner(), runner())`
+ means attempt 1 always compares the one cold replay against a warm one, and the
+ divergences it produces are nested prefixes of each other, which is what a 270M
+ model near-tied on whether to stop looks like, not a broken sampler.
+
+ --parallel > 1 is the second: it also brings --kv-unified, one shared KV pool
+ whose occupancy is another input to the batch.
+
+ Checked, not assumed. A load that quietly dropped the pin would leave this
+ printing OK for a server it never pinned, which is the same shape of hole #5669
+ left on the Linux leg for three months.
+ """
+ status = _read_backend_status()
+ speculative, slots = status.get("speculative_type"), status.get("parallel_slots")
+ assert speculative in ("off", "none"), (
+ f"this probe needs speculative decoding off and the server reports "
+ f'{speculative!r}. Load with "speculative_type": "off": a drafted batch '
+ f"replacing sequential decode is not bit-identical, so the greedy output "
+ f"below would not be reproducible however many times it is retried."
+ )
+ assert slots == 1, (
+ f"this probe needs one decode slot and the server reports "
+ f'parallel_slots={slots!r}. Load with "n_parallel": 1: more slots bring '
+ f"--kv-unified, whose shared-pool occupancy is another input to the batch."
+ )
+
+
def run_openai() -> list[str]:
from openai import OpenAI
@@ -168,6 +232,9 @@ def check(label: str, first: list[str], second: list[str]) -> None:
def main() -> int:
+ # Before any generation: a divergence below is only evidence of a fault if the
+ # server was in a configuration that could have avoided one.
+ assert_reproducible_backend()
for label, runner in (("openai", run_openai), ("anthropic", run_anthropic)):
# Only a replay disagreement is retried, and only here. Two replays are not two
# identical requests: each turn is built from the reply before it, and the second
diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml
index c82fa2712d3..769b42b8424 100644
--- a/.github/workflows/studio-inference-smoke.yml
+++ b/.github/workflows/studio-inference-smoke.yml
@@ -318,10 +318,23 @@ jobs:
timeout-minutes: 10
if: ${{ !cancelled() && steps.sdks.outcome == 'success' && steps.rotate-api.outcome == 'success' }}
run: |
+ # speculative_type=off and n_parallel=1 are what make the determinism
+ # assertion in the next step answerable. llama-server's README: "the
+ # logits are not guaranteed to be bit-for-bit identical for different
+ # batch" sizes. A non-MTP GGUF otherwise takes the --spec-default branch,
+ # which is live n-gram drafting, not the absence of a drafter: the server
+ # logs "draft acceptance = 0.46875 (30 accepted / 64 generated)". Its
+ # draft pool is built from text the server has already seen, so the FIRST
+ # turn-1 request after a load drafts nothing and decodes one token at a
+ # time while every later one verifies a ~32-token draft in one batch, and
+ # check(runner(), runner()) always compares the cold replay against a warm
+ # one. n_parallel=1 also drops --kv-unified, whose shared-pool occupancy is
+ # a second input to the batch. The other phases of this workflow keep both
+ # defaults, so the shipped path stays covered.
curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
-H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
--max-time 600 \
- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \
+ -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048,\"speculative_type\":\"off\",\"n_parallel\":1}" \
| jq '{status, display_name, is_gguf, context_length}'
- name: Multi-turn determinism via OpenAI + Anthropic SDKs
diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml
index cdf1a34b0e4..b3c78f1459f 100644
--- a/.github/workflows/studio-mac-ui-smoke.yml
+++ b/.github/workflows/studio-mac-ui-smoke.yml
@@ -711,10 +711,23 @@ jobs:
# curl already caps itself at --max-time 600; +2 for process overhead.
timeout-minutes: 12
run: |
+ # speculative_type=off and n_parallel=1 are what make the determinism
+ # assertion in the next step answerable. llama-server's README: "the
+ # logits are not guaranteed to be bit-for-bit identical for different
+ # batch" sizes. A non-MTP GGUF otherwise takes the --spec-default branch,
+ # which is live n-gram drafting, not the absence of a drafter: the server
+ # logs "draft acceptance = 0.46875 (30 accepted / 64 generated)". Its
+ # draft pool is built from text the server has already seen, so the FIRST
+ # turn-1 request after a load drafts nothing and decodes one token at a
+ # time while every later one verifies a ~32-token draft in one batch, and
+ # check(runner(), runner()) always compares the cold replay against a warm
+ # one. n_parallel=1 also drops --kv-unified, whose shared-pool occupancy is
+ # a second input to the batch. The other phases of this workflow keep both
+ # defaults, so the shipped path stays covered.
curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
-H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
--max-time 600 \
- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \
+ -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048,\"speculative_type\":\"off\",\"n_parallel\":1}" \
| jq '{status, display_name, is_gguf, context_length}'
- name: Multi-turn determinism via OpenAI + Anthropic SDKs
diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml
index a0ccb70b7fc..276477b1651 100644
--- a/.github/workflows/studio-windows-inference-smoke.yml
+++ b/.github/workflows/studio-windows-inference-smoke.yml
@@ -430,13 +430,27 @@ jobs:
# the whole job. The Unsloth backend's _wait_for_health now
# catches httpx.ReadError too; this retry layer covers the
# cases the backend can't recover from on its own.
+ #
+ # speculative_type=off and n_parallel=1 are what make the determinism
+ # assertion in the next step answerable. llama-server's README: "the
+ # logits are not guaranteed to be bit-for-bit identical for different
+ # batch" sizes. A non-MTP GGUF otherwise takes the --spec-default branch,
+ # which is live n-gram drafting, not the absence of a drafter: the server
+ # logs "draft acceptance = 0.46875 (30 accepted / 64 generated)". Its
+ # draft pool is built from text the server has already seen, so the FIRST
+ # turn-1 request after a load drafts nothing and decodes one token at a
+ # time while every later one verifies a ~32-token draft in one batch, and
+ # check(runner(), runner()) always compares the cold replay against a warm
+ # one. n_parallel=1 also drops --kv-unified, whose shared-pool occupancy is
+ # a second input to the batch. The other phases of this workflow keep both
+ # defaults, so the shipped path stays covered.
LOAD_OK=0
for attempt in 1 2 3; do
HTTP=$(curl -s -o /tmp/load.json -w '%{http_code}' \
-X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
-H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
--max-time 600 \
- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}")
+ -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048,\"speculative_type\":\"off\",\"n_parallel\":1}")
if [ "$HTTP" = "200" ]; then LOAD_OK=1; break; fi
echo "::warning::/api/inference/load attempt $attempt returned $HTTP; response:"
cat /tmp/load.json || true
diff --git a/tests/studio/test_smoke_workflows_share_one_script.py b/tests/studio/test_smoke_workflows_share_one_script.py
index 2b62c69d88a..d52bb4098a1 100644
--- a/tests/studio/test_smoke_workflows_share_one_script.py
+++ b/tests/studio/test_smoke_workflows_share_one_script.py
@@ -179,6 +179,7 @@ def test_the_replay_retry_cannot_pass_a_truly_nondeterministic_server(script, mo
# A different last turn every call, so no two replays ever agree.
return ["58 + 27 = 95", "the answer was 95", "paris", f"paris {calls['n']}"]
+ monkeypatch.setattr(script, "assert_reproducible_backend", lambda: None)
monkeypatch.setattr(script, "run_openai", always_divergent)
monkeypatch.setattr(script, "run_anthropic", always_divergent)
with pytest.raises(AssertionError, match = "non-deterministic"):
@@ -209,8 +210,69 @@ def test_a_non_divergence_failure_is_not_retried(script, monkeypatch):
calls["n"] += 1
return ["58 + 27 = 95", "b", "paris", "Okay, I'm ready."]
+ monkeypatch.setattr(script, "assert_reproducible_backend", lambda: None)
monkeypatch.setattr(script, "run_openai", no_history)
monkeypatch.setattr(script, "run_anthropic", no_history)
with pytest.raises(AssertionError, match = "history reached the model"):
script.main()
assert calls["n"] == 2, f"retried a non-divergence failure {calls['n']} times"
+
+
+def test_every_leg_pins_the_probe_load_to_a_reproducible_backend():
+ """The determinism assertion is only answerable against a pinned load.
+
+ A non-MTP GGUF like gemma-3-270m-it takes llama.cpp's `--spec-default` branch,
+ which is live n-gram drafting, not the absence of a drafter: the server logs
+ `draft acceptance = 0.46875 (30 accepted / 64 generated)`. The draft pool is
+ built from text the server has already seen, so the first turn-1 request after a
+ load drafts nothing and decodes one token at a time while every later one
+ verifies a ~32-token draft in one batch, and llama-server's README is explicit
+ that logits are not bit-for-bit identical across batch shapes. Since
+ check(runner(), runner()) always compares the cold replay against a warm one,
+ that made attempt 1 structurally unable to agree, and the whole check a coin
+ flip the retry usually won.
+
+ Only the load that feeds the multi-turn probe is pinned. The tool-calling and
+ vision phases deliberately keep the defaults, so the shipped path stays covered.
+ """
+ for name in LEGS:
+ text = _workflow(name)
+ # rsplit, not split: two of the legs also name the script in their `paths:`
+ # filter, long before the step that runs it.
+ probe = text.rsplit("studio_smoke/multi_turn_chat.py", 1)[0]
+ assert '\\"speculative_type\\":\\"off\\"' in probe, (
+ f"{name} loads the multi-turn probe's model without pinning "
+ f"speculative_type=off, so a drafted batch can replace sequential decode "
+ f"between the two replays and greedy output stops being reproducible"
+ )
+ assert '\\"n_parallel\\":1' in probe, (
+ f"{name} loads the multi-turn probe's model without pinning n_parallel=1, "
+ f"so --kv-unified shares one KV pool across slots and its occupancy is "
+ f"another input to the batch"
+ )
+
+
+def test_the_probe_refuses_a_backend_that_cannot_be_reproducible(script, monkeypatch):
+ """The workflow pin is worth nothing if a load can silently drop it.
+
+ A divergence below is only evidence of a fault if the server was in a
+ configuration that could have avoided one, so main() checks before it generates
+ and names what is wrong instead of reporting a mystery disagreement.
+ """
+ clean = ["58 + 27 = 95", "the answer was 95", "paris", "paris"]
+ monkeypatch.setattr(script, "run_openai", lambda: list(clean))
+ monkeypatch.setattr(script, "run_anthropic", lambda: list(clean))
+
+ def _with(status):
+ monkeypatch.setattr(script, "_read_backend_status", lambda: status)
+
+ _with({"speculative_type": "default", "parallel_slots": 1})
+ with pytest.raises(AssertionError, match = "speculative decoding off"):
+ script.main()
+
+ _with({"speculative_type": "off", "parallel_slots": 4})
+ with pytest.raises(AssertionError, match = "one decode slot"):
+ script.main()
+
+ _with({"speculative_type": "off", "parallel_slots": 1})
+ assert script.main() == 0