Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Pin the multi-turn determinism probe to a backend that can be deterministic (#10234)
Daniel Han9d3fcd197e6a
diff --git a/.github/scripts/studio_smoke/multi_turn_chat.py b/.github/scripts/studio_smoke/multi_turn_chat.pyindex 089b85a4a6d..a5ddd839f61 100644--- a/.github/scripts/studio_smoke/multi_turn_chat.py+++ b/.github/scripts/studio_smoke/multi_turn_chat.py@@ -5,8 +5,13 @@Two properties at once. The conversation is built so that turns 2 and 4 are onlyanswerable from the earlier turns, which exercises the history wiring; and the whole-conversation is run twice at temperature 0.0 with a fixed seed, which is the only check-anywhere that greedy decoding is reproducible.+conversation is run twice at temperature 0.0 against a slot with speculative decoding+off, which is the only check anywhere that greedy decoding is reproducible.++The seed is sent but does no sampling work: at temperature 0 llama.cpp collapses to+argmax and never reaches an RNG. What it does do is make the Studio send+cache_prompt=False, which is one of the two things the comparison below needs. The+other is the load pin that assert_reproducible_backend() checks.This lived inline in three workflows, and being three copies is what let one of themstop checking. On 2026-05-22 an unrelated event-loop fix (#5669) relaxed the Linux copy@@ -21,9 +26,11 @@ between the three callers: each boots its server on its own port.from __future__ import annotations+import jsonimport osimport reimport sys+import urllib.requestSEED = 3407MAX_TOKENS = 80@@ -68,6 +75,63 @@ def _server() -> tuple[str, str]:return os.environ["BASE_URL"], os.environ["TOKEN"] # a JWT is accepted as Bearer+def _read_backend_status() -> dict:+ """What the server says it loaded. Split out so the assertion below is reachable+ from a test without standing a server up, the same reason _server() exists."""+ base, token = _server()+ request = urllib.request.Request(+ f"{base}/api/inference/status", headers = {"Authorization": f"Bearer {token}"}+ )+ with urllib.request.urlopen(request, timeout = 30) as response:+ return json.load(response)+++def assert_reproducible_backend() -> None:+ """Refuse to assert determinism against a server that cannot provide it.++ llama-server's own README says it, about cache_prompt: "the logits are not+ guaranteed to be bit-for-bit identical for different batch" sizes. Two things in+ the launch the Studio derives change the batch between one replay and the next,+ and both are per-LOAD settings, so the three workflows pin them and this checks+ the pin took.++ Speculative decoding is the one that actually bites. A non-MTP GGUF like+ gemma-3-270m-it takes the `--spec-default` branch, which llama.cpp's arg parser+ turns into n-gram-mod drafting, and the server logs prove it is live:++ slot print_timing: id 3 | task 100 | draft acceptance = 0.46875+ (30 accepted / 64 generated), mean len = 31.00++ The draft pool is built from text the server has already seen, so the FIRST turn-1+ request after a load drafts nothing and decodes one token at a time, while every+ later one verifies a ~32-token draft in a single batch. Same tokens, different+ arithmetic. That is not a near-tie a retry rides out: `check(runner(), runner())`+ means attempt 1 always compares the one cold replay against a warm one, and the+ divergences it produces are nested prefixes of each other, which is what a 270M+ model near-tied on whether to stop looks like, not a broken sampler.++ --parallel > 1 is the second: it also brings --kv-unified, one shared KV pool+ whose occupancy is another input to the batch.++ Checked, not assumed. A load that quietly dropped the pin would leave this+ printing OK for a server it never pinned, which is the same shape of hole #5669+ left on the Linux leg for three months.+ """+ status = _read_backend_status()+ speculative, slots = status.get("speculative_type"), status.get("parallel_slots")+ assert speculative in ("off", "none"), (+ f"this probe needs speculative decoding off and the server reports "+ f'{speculative!r}. Load with "speculative_type": "off": a drafted batch '+ f"replacing sequential decode is not bit-identical, so the greedy output "+ f"below would not be reproducible however many times it is retried."+ )+ assert slots == 1, (+ f"this probe needs one decode slot and the server reports "+ f'parallel_slots={slots!r}. Load with "n_parallel": 1: more slots bring '+ f"--kv-unified, whose shared-pool occupancy is another input to the batch."+ )++def run_openai() -> list[str]:from openai import OpenAI@@ -168,6 +232,9 @@ def check(label: str, first: list[str], second: list[str]) -> None:def main() -> int:+ # Before any generation: a divergence below is only evidence of a fault if the+ # server was in a configuration that could have avoided one.+ assert_reproducible_backend()for label, runner in (("openai", run_openai), ("anthropic", run_anthropic)):# Only a replay disagreement is retried, and only here. Two replays are not two# identical requests: each turn is built from the reply before it, and the seconddiff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.ymlindex c82fa2712d3..769b42b8424 100644--- a/.github/workflows/studio-inference-smoke.yml+++ b/.github/workflows/studio-inference-smoke.yml@@ -318,10 +318,23 @@ jobs:timeout-minutes: 10if: ${{ !cancelled() && steps.sdks.outcome == 'success' && steps.rotate-api.outcome == 'success' }}run: |+ # speculative_type=off and n_parallel=1 are what make the determinism+ # assertion in the next step answerable. llama-server's README: "the+ # logits are not guaranteed to be bit-for-bit identical for different+ # batch" sizes. A non-MTP GGUF otherwise takes the --spec-default branch,+ # which is live n-gram drafting, not the absence of a drafter: the server+ # logs "draft acceptance = 0.46875 (30 accepted / 64 generated)". Its+ # draft pool is built from text the server has already seen, so the FIRST+ # turn-1 request after a load drafts nothing and decodes one token at a+ # time while every later one verifies a ~32-token draft in one batch, and+ # check(runner(), runner()) always compares the cold replay against a warm+ # one. n_parallel=1 also drops --kv-unified, whose shared-pool occupancy is+ # a second input to the batch. The other phases of this workflow keep both+ # defaults, so the shipped path stays covered.curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \-H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \--max-time 600 \- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \+ -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048,\"speculative_type\":\"off\",\"n_parallel\":1}" \| jq '{status, display_name, is_gguf, context_length}'- name: Multi-turn determinism via OpenAI + Anthropic SDKsdiff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.ymlindex cdf1a34b0e4..b3c78f1459f 100644--- a/.github/workflows/studio-mac-ui-smoke.yml+++ b/.github/workflows/studio-mac-ui-smoke.yml@@ -711,10 +711,23 @@ jobs:# curl already caps itself at --max-time 600; +2 for process overhead.timeout-minutes: 12run: |+ # speculative_type=off and n_parallel=1 are what make the determinism+ # assertion in the next step answerable. llama-server's README: "the+ # logits are not guaranteed to be bit-for-bit identical for different+ # batch" sizes. A non-MTP GGUF otherwise takes the --spec-default branch,+ # which is live n-gram drafting, not the absence of a drafter: the server+ # logs "draft acceptance = 0.46875 (30 accepted / 64 generated)". Its+ # draft pool is built from text the server has already seen, so the FIRST+ # turn-1 request after a load drafts nothing and decodes one token at a+ # time while every later one verifies a ~32-token draft in one batch, and+ # check(runner(), runner()) always compares the cold replay against a warm+ # one. n_parallel=1 also drops --kv-unified, whose shared-pool occupancy is+ # a second input to the batch. The other phases of this workflow keep both+ # defaults, so the shipped path stays covered.curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \-H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \--max-time 600 \- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \+ -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048,\"speculative_type\":\"off\",\"n_parallel\":1}" \| jq '{status, display_name, is_gguf, context_length}'- name: Multi-turn determinism via OpenAI + Anthropic SDKsdiff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.ymlindex a0ccb70b7fc..276477b1651 100644--- a/.github/workflows/studio-windows-inference-smoke.yml+++ b/.github/workflows/studio-windows-inference-smoke.yml@@ -430,13 +430,27 @@ jobs:# the whole job. The Unsloth backend's _wait_for_health now# catches httpx.ReadError too; this retry layer covers the# cases the backend can't recover from on its own.+ #+ # speculative_type=off and n_parallel=1 are what make the determinism+ # assertion in the next step answerable. llama-server's README: "the+ # logits are not guaranteed to be bit-for-bit identical for different+ # batch" sizes. A non-MTP GGUF otherwise takes the --spec-default branch,+ # which is live n-gram drafting, not the absence of a drafter: the server+ # logs "draft acceptance = 0.46875 (30 accepted / 64 generated)". Its+ # draft pool is built from text the server has already seen, so the FIRST+ # turn-1 request after a load drafts nothing and decodes one token at a+ # time while every later one verifies a ~32-token draft in one batch, and+ # check(runner(), runner()) always compares the cold replay against a warm+ # one. n_parallel=1 also drops --kv-unified, whose shared-pool occupancy is+ # a second input to the batch. The other phases of this workflow keep both+ # defaults, so the shipped path stays covered.LOAD_OK=0for attempt in 1 2 3; doHTTP=$(curl -s -o /tmp/load.json -w '%{http_code}' \-X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \-H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \--max-time 600 \- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}")+ -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048,\"speculative_type\":\"off\",\"n_parallel\":1}")if [ "$HTTP" = "200" ]; then LOAD_OK=1; break; fiecho "::warning::/api/inference/load attempt $attempt returned $HTTP; response:"cat /tmp/load.json || truediff --git a/tests/studio/test_smoke_workflows_share_one_script.py b/tests/studio/test_smoke_workflows_share_one_script.pyindex 2b62c69d88a..d52bb4098a1 100644--- a/tests/studio/test_smoke_workflows_share_one_script.py+++ b/tests/studio/test_smoke_workflows_share_one_script.py@@ -179,6 +179,7 @@ def test_the_replay_retry_cannot_pass_a_truly_nondeterministic_server(script, mo# A different last turn every call, so no two replays ever agree.return ["58 + 27 = 95", "the answer was 95", "paris", f"paris {calls['n']}"]+ monkeypatch.setattr(script, "assert_reproducible_backend", lambda: None)monkeypatch.setattr(script, "run_openai", always_divergent)monkeypatch.setattr(script, "run_anthropic", always_divergent)with pytest.raises(AssertionError, match = "non-deterministic"):@@ -209,8 +210,69 @@ def test_a_non_divergence_failure_is_not_retried(script, monkeypatch):calls["n"] += 1return ["58 + 27 = 95", "b", "paris", "Okay, I'm ready."]+ monkeypatch.setattr(script, "assert_reproducible_backend", lambda: None)monkeypatch.setattr(script, "run_openai", no_history)monkeypatch.setattr(script, "run_anthropic", no_history)with pytest.raises(AssertionError, match = "history reached the model"):script.main()assert calls["n"] == 2, f"retried a non-divergence failure {calls['n']} times"+++def test_every_leg_pins_the_probe_load_to_a_reproducible_backend():+ """The determinism assertion is only answerable against a pinned load.++ A non-MTP GGUF like gemma-3-270m-it takes llama.cpp's `--spec-default` branch,+ which is live n-gram drafting, not the absence of a drafter: the server logs+ `draft acceptance = 0.46875 (30 accepted / 64 generated)`. The draft pool is+ built from text the server has already seen, so the first turn-1 request after a+ load drafts nothing and decodes one token at a time while every later one+ verifies a ~32-token draft in one batch, and llama-server's README is explicit+ that logits are not bit-for-bit identical across batch shapes. Since+ check(runner(), runner()) always compares the cold replay against a warm one,+ that made attempt 1 structurally unable to agree, and the whole check a coin+ flip the retry usually won.++ Only the load that feeds the multi-turn probe is pinned. The tool-calling and+ vision phases deliberately keep the defaults, so the shipped path stays covered.+ """+ for name in LEGS:+ text = _workflow(name)+ # rsplit, not split: two of the legs also name the script in their `paths:`+ # filter, long before the step that runs it.+ probe = text.rsplit("studio_smoke/multi_turn_chat.py", 1)[0]+ assert '\\"speculative_type\\":\\"off\\"' in probe, (+ f"{name} loads the multi-turn probe's model without pinning "+ f"speculative_type=off, so a drafted batch can replace sequential decode "+ f"between the two replays and greedy output stops being reproducible"+ )+ assert '\\"n_parallel\\":1' in probe, (+ f"{name} loads the multi-turn probe's model without pinning n_parallel=1, "+ f"so --kv-unified shares one KV pool across slots and its occupancy is "+ f"another input to the batch"+ )+++def test_the_probe_refuses_a_backend_that_cannot_be_reproducible(script, monkeypatch):+ """The workflow pin is worth nothing if a load can silently drop it.++ A divergence below is only evidence of a fault if the server was in a+ configuration that could have avoided one, so main() checks before it generates+ and names what is wrong instead of reporting a mystery disagreement.+ """+ clean = ["58 + 27 = 95", "the answer was 95", "paris", "paris"]+ monkeypatch.setattr(script, "run_openai", lambda: list(clean))+ monkeypatch.setattr(script, "run_anthropic", lambda: list(clean))++ def _with(status):+ monkeypatch.setattr(script, "_read_backend_status", lambda: status)++ _with({"speculative_type": "default", "parallel_slots": 1})+ with pytest.raises(AssertionError, match = "speculative decoding off"):+ script.main()++ _with({"speculative_type": "off", "parallel_slots": 4})+ with pytest.raises(AssertionError, match = "one decode slot"):+ script.main()++ _with({"speculative_type": "off", "parallel_slots": 1})+ assert script.main() == 0