Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: step streaming generators on a dedicated thread instead of the pool (#15590)
Daniel Han975715cb440f
diff --git a/studio/backend/core/inference/gen_stepper.py b/studio/backend/core/inference/gen_stepper.pynew file mode 100644index 00000000000..71ced0e1a13--- /dev/null+++ b/studio/backend/core/inference/gen_stepper.py@@ -0,0 +1,95 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Advance a blocking generator off the event loop, one step per await, on its own thread."""++from __future__ import annotations++import asyncio+import contextvars+import queue+import threading+import weakref++# A stepper whose consumer stops asking (slow client) frees its thread after this.+_IDLE_EXIT_S = 30.0+++def _resolve(future: asyncio.Future, result, error) -> None:+ if future.cancelled():+ return+ if error is None:+ future.set_result(result)+ else:+ future.set_exception(error)+++def _waker(requests: queue.SimpleQueue):+ # SimpleQueue.put is reentrant, so this is safe from a weakref callback run by the GC.+ return lambda _ref: requests.put(None)+++def _work(ref, requests: queue.SimpleQueue) -> None:+ # Holds the stepper only during a step, so a dropped stream's generator is finalized at once.+ while True:+ try:+ request = requests.get(timeout = _IDLE_EXIT_S)+ except queue.Empty:+ stepper = ref()+ if stepper is None:+ return+ with stepper._lock:+ if requests.empty():+ stepper._running = False+ return+ del stepper+ continue+ if request is None:+ return+ loop, future, context, default = request+ stepper = ref()+ if stepper is None:+ return+ try:+ result, error = context.run(next, stepper._gen, default), None+ except BaseException as exc:+ result, error = None, exc+ del stepper+ try:+ loop.call_soon_threadsafe(_resolve, future, result, error)+ except RuntimeError:+ pass+ del request, loop, future, context, default, result, error+++class ThreadStepper:+ """``await stepper.next(default)`` behaves like ``await asyncio.to_thread(next, gen, default)``.++ Same context copy, exception and cancellation behaviour (a cancelled await abandons the step,+ which still finishes), but every step runs on one dedicated thread instead of a thread-pool+ round trip, and steps run one at a time in request order.+ """++ __slots__ = ("_gen", "_lock", "_requests", "_running", "__weakref__")++ def __init__(self, gen):+ self._gen = gen+ self._lock = threading.Lock()+ self._requests = queue.SimpleQueue()+ self._running = False++ async def next(self, default):+ loop = asyncio.get_running_loop()+ future = loop.create_future()+ request = (loop, future, contextvars.copy_context(), default)+ with self._lock:+ if not self._running:+ threading.Thread(+ target = _work,+ args = (weakref.ref(self, _waker(self._requests)), self._requests),+ name = "studio-gen-step",+ daemon = True,+ ).start()+ self._running = True+ self._requests.put(request)+ return await futurediff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex f1bdab4e85c..6f1123addcf 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -131,6 +131,7 @@ from core.inference.audio_errors import ()from core.inference.audio_cpp_outputs import NO_STEMSfrom core.inference import context_refusal+from core.inference.gen_stepper import ThreadStepperfrom core.inference.context_window import (estimate_message_tokens as _estimate_message_tokens,estimate_messages_tokens as _estimate_messages_tokens,@@ -21437,6 +21438,7 @@ async def generate_stream(presence_penalty = request.presence_penalty,cancel_event = cancel_event,)+ gen_step = ThreadStepper(gen).next_DONE = object()while True:if cancel_event.is_set():@@ -21445,7 +21447,7 @@ async def generate_stream(# Flag failed so the middleware does not claim the model on a cancelled stream.mark_response_failed(_gs_scope)break- chunk = await asyncio.to_thread(next, gen, _DONE)+ chunk = await gen_step(_DONE)if chunk is _DONE:if cancel_event.is_set():mark_response_failed(_gs_scope)@@ -30633,6 +30635,7 @@ async def produce_openai_chat_completions(yield _chat_role_chunk(completion_id, created, model_name)gen = audio_input_generate()+ gen_step = ThreadStepper(gen).next_DONE = object()cancelled = Falsewhile True:@@ -30646,7 +30649,7 @@ async def produce_openai_chat_completions(api_monitor.finish(monitor_id, "cancelled")mark_current_response_failed()return- chunk_text = await asyncio.to_thread(next, gen, _DONE)+ chunk_text = await gen_step(_DONE)if chunk_text is _DONE:if cancel_event.is_set():cancelled = True@@ -31502,6 +31505,7 @@ async def produce_openai_chat_completions(context_refusal.open_slot()# In a thread so the event loop stays free for disconnect detection.gen = gguf_generate_with_tools()+ gen_step = ThreadStepper(gen).nextprev_text = ""_gguf_display_strip = _StreamingDisplayStrip(auto_heal_tool_calls = _gguf_auto_heal_tool_calls,@@ -31546,9 +31550,7 @@ async def produce_openai_chat_completions(resume_before_next = Falseawait _park_admission(False)- next_task = asyncio.create_task(- asyncio.to_thread(next, gen, _tool_sentinel)- )+ next_task = asyncio.create_task(gen_step(_tool_sentinel))try:# asyncio.wait never cancels next_task, matching the finally-drain shield.wait_timeout = (@@ -32224,6 +32226,7 @@ async def produce_openai_chat_completions(# In a thread so the event loop stays free for disconnect detection.gen = gguf_generate()+ gen_step = ThreadStepper(gen).nextprev_text = ""reasoning_extractor = _new_chat_reasoning_extractor()_stream_usage = None@@ -32238,9 +32241,7 @@ async def produce_openai_chat_completions(mark_current_response_failed()api_monitor.finish(monitor_id, "cancelled")return- next_task = asyncio.create_task(- asyncio.to_thread(next, gen, _gguf_sentinel)- )+ next_task = asyncio.create_task(gen_step(_gguf_sentinel))try:# Stall-timeout wait: keepalive while the generator stays# silent (e.g. no-tool prefill). asyncio.wait never cancels@@ -33370,6 +33371,7 @@ async def produce_openai_chat_completions(yield _chat_role_chunk(completion_id, created, model_name)gen = sf_generate_with_tools()+ gen_step = ThreadStepper(gen).nextprev_text = ""_sf_display_strip = _StreamingDisplayStrip(auto_heal_tool_calls = _sf_auto_heal_tool_calls,@@ -33404,9 +33406,7 @@ async def produce_openai_chat_completions(return# Keepalive so silent backend segments do not exceed proxy timeouts.- _sf_next_task = asyncio.create_task(- asyncio.to_thread(next, gen, _sf_tool_sentinel)- )+ _sf_next_task = asyncio.create_task(gen_step(_sf_tool_sentinel))wait_timeout = (TOOL_APPROVAL_FLUSH_DELAY_Sif approval_flush_pending@@ -34065,13 +34065,14 @@ async def produce_openai_chat_completions(# Worker thread: a blocking _gen_lock in compare mode would otherwise freeze the loop._DONE = object() # sentinel for generator exhaustiongen = generate()+ gen_step = ThreadStepper(gen).nextwhile True:if cancel_event.is_set():backend.reset_generation_state(cancel_event)mark_current_response_failed()break# next(gen, _DONE) avoids StopIteration, which can't cross asyncio futures.- _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _DONE))+ _next_task = asyncio.create_task(gen_step(_DONE))while True:_done_tasks, _ = await asyncio.wait({_next_task},@@ -36463,11 +36464,12 @@ async def _mlx_completions(request, current_subject, backend, body: dict, refusa)_DONE = object()gen = generate(prompt_ids) if budget != 0 else iter(())+ gen_step = ThreadStepper(gen).nextprev = ""while True:if cancel_event.is_set():break- next_task = asyncio.create_task(asyncio.to_thread(next, gen, _DONE))+ next_task = asyncio.create_task(gen_step(_DONE))while not (await asyncio.wait({next_task}, timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S@@ -43636,6 +43638,7 @@ async def _anthropic_tool_stream(# The respawn refit records the refusal from the worker thread, so open the slot out here first.context_refusal.open_slot()gen = run_gen()+ gen_step = ThreadStepper(gen).next_next_task = None# The in-loop poll fires only between events, so a mid-prefill disconnect needs a watcher.disconnect_watcher = asyncio.create_task(@@ -43649,7 +43652,7 @@ async def _anthropic_tool_stream(mark_response_failed(getattr(request, "scope", None))return# Stall keepalive: silent backend segments must not idle the SSE stream past proxy timeouts.- _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel))+ _next_task = asyncio.create_task(gen_step(_sentinel))while True:_done_tasks, _ = await asyncio.wait({_next_task},@@ -43730,6 +43733,7 @@ async def _anthropic_plain_stream(captured_finish_reason = Nonegen = run_gen()+ gen_step = ThreadStepper(gen).next_next_task = None# Watcher cancels on disconnect since the in-loop poll only runs between chunks, not mid-prefill.disconnect_watcher = asyncio.create_task(@@ -43743,7 +43747,7 @@ async def _anthropic_plain_stream(mark_response_failed(getattr(request, "scope", None))return# Stall keepalive each window while next(gen) runs in a worker.- _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel))+ _next_task = asyncio.create_task(gen_step(_sentinel))while True:_done_tasks, _ = await asyncio.wait({_next_task},diff --git a/studio/backend/tests/test_gen_stepper.py b/studio/backend/tests/test_gen_stepper.pynew file mode 100644index 00000000000..a54b2da6bd6--- /dev/null+++ b/studio/backend/tests/test_gen_stepper.py@@ -0,0 +1,170 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""ThreadStepper matches asyncio.to_thread(next, gen, default) step for step."""++import asyncio+import contextvars+import threading+import time++import pytest++from core.inference import gen_stepper+from core.inference.gen_stepper import ThreadStepper++_VAR = contextvars.ContextVar("_VAR", default = "unset")+++def _drive(step, n):+ async def run():+ return [await step(None) for _ in range(n)]++ return asyncio.run(run())+++def test_steps_match_to_thread():+ def gen():+ yield from ("a", "b", "c")++ async def via_to_thread(default):+ return await asyncio.to_thread(next, base, default)++ base = gen()+ assert (+ _drive(ThreadStepper(gen()).next, 5)+ == _drive(via_to_thread, 5)+ == ["a", "b", "c", None, None]+ )+++def test_one_thread_runs_every_step_in_order():+ seen = []++ def gen():+ for i in range(50):+ seen.append(threading.get_ident())+ yield i++ assert _drive(ThreadStepper(gen()).next, 51) == list(range(50)) + [None]+ assert len(set(seen)) == 1 and seen[0] != threading.get_ident()+++def test_exception_propagates_and_generator_finishes():+ def gen():+ yield 1+ raise KeyError("boom")++ async def run():+ step = ThreadStepper(gen()).next+ first = await step(None)+ with pytest.raises(KeyError):+ await step(None)+ return first, await step("done")++ assert asyncio.run(run()) == (1, "done")+++def test_context_is_copied_per_step():+ def gen():+ while True:+ yield _VAR.get()++ async def run():+ step = ThreadStepper(gen()).next+ _VAR.set("one")+ a = await step(None)+ _VAR.set("two")+ return a, await step(None)++ assert asyncio.run(run()) == ("one", "two")+++def test_cancelled_wait_abandons_the_step_only():+ release = threading.Event()++ def gen():+ release.wait(5)+ yield "slow"+ yield "next"++ async def run():+ step = ThreadStepper(gen()).next+ task = asyncio.create_task(step(None))+ await asyncio.sleep(0.05)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ release.set()+ return await step(None)++ assert asyncio.run(run()) == "next"+++def test_idle_thread_exits_and_restarts(monkeypatch):+ monkeypatch.setattr(gen_stepper, "_IDLE_EXIT_S", 0.05)++ def gen():+ yield from range(3)++ stepper = ThreadStepper(gen())++ async def run():+ first = await stepper.next(None)+ deadline = time.monotonic() + 5+ while stepper._running and time.monotonic() < deadline:+ await asyncio.sleep(0.02)+ assert not stepper._running+ return first, await stepper.next(None)++ assert asyncio.run(run()) == (0, 1)+++def test_idle_thread_does_not_keep_a_dropped_generator_alive():+ closed = threading.Event()++ def gen():+ try:+ yield from range(10)+ finally:+ closed.set()++ async def run():+ stepper = ThreadStepper(gen())+ first = await stepper.next(None)+ del stepper+ return first++ assert asyncio.run(run()) == 0+ import gc++ gc.collect()+ # The idle worker waits up to _IDLE_EXIT_S, but must not hold the generator meanwhile.+ assert closed.wait(2)+++def test_dropping_a_stepper_frees_its_thread_before_the_idle_timeout():+ import gc++ def workers():+ return [t for t in threading.enumerate() if t.name == "studio-gen-step"]++ # By identity, not count: a worker woken by an earlier test may still be exiting.+ before = set(workers())+ started = []++ async def run():+ steppers = [ThreadStepper(iter(range(3))) for _ in range(32)]+ assert [await stepper.next(None) for stepper in steppers] == [0] * 32+ started.extend(t for t in workers() if t not in before)+ assert len(started) == 32+ steppers.clear()++ asyncio.run(run())+ gc.collect()+ # _IDLE_EXIT_S is 30 s; a dropped stepper's worker is woken instead of left to wait it out.+ deadline = time.monotonic() + 5+ for t in started:+ t.join(max(0.0, deadline - time.monotonic()))+ alive = [t for t in started if t.is_alive()]+ assert not alive, len(alive)diff --git a/tests/studio/test_stream_cancel_registration_timing.py b/tests/studio/test_stream_cancel_registration_timing.pyindex 9c2f4f9fd22..b9c7e1d11dd 100644--- a/tests/studio/test_stream_cancel_registration_timing.py+++ b/tests/studio/test_stream_cancel_registration_timing.py@@ -540,6 +540,26 @@ def test_streaming_generators_check_cancel_event_in_loop():)+def _steps_generator_off_loop(fn) -> bool:+ for sub in ast.walk(fn):+ if not isinstance(sub, ast.Call):+ continue+ f = sub.func+ if isinstance(f, ast.Name) and f.id == "ThreadStepper":+ return True+ if (+ isinstance(f, ast.Attribute)+ and f.attr == "to_thread"+ and isinstance(f.value, ast.Name)+ and f.value.id == "asyncio"+ and sub.args+ and isinstance(sub.args[0], ast.Name)+ and sub.args[0].id == "next"+ ):+ return True+ return False++def test_audio_input_stream_offloads_blocking_next_to_thread():audio = Nonefor fn in ast.walk(_TREE):@@ -553,29 +573,14 @@ def test_audio_input_stream_offloads_blocking_next_to_thread():it_src = ast.unparse(sub.iter)assert "audio_input_generate" not in it_src, ("audio_input_stream must not iterate audio_input_generate() "- "directly -- that blocks the event loop. Use "- "`await asyncio.to_thread(next, gen, _DONE)` inside a "- "`while True` loop instead"+ "directly -- that blocks the event loop. Step it with "+ "`ThreadStepper(gen).next` inside a `while True` loop instead")- found_to_thread_next = False- for sub in ast.walk(audio):- if not isinstance(sub, ast.Call):- continue- fn_expr = sub.func- if not (- isinstance(fn_expr, ast.Attribute)- and fn_expr.attr == "to_thread"- and isinstance(fn_expr.value, ast.Name)- and fn_expr.value.id == "asyncio"- ):- continue- if sub.args and isinstance(sub.args[0], ast.Name) and sub.args[0].id == "next":- found_to_thread_next = True- break- assert found_to_thread_next, (- "audio_input_stream must call `asyncio.to_thread(next, gen, ...)` "- "to keep the event loop free while whisper yields the next chunk"+ assert _steps_generator_off_loop(audio), (+ "audio_input_stream must step its generator with `ThreadStepper(gen).next` "+ "or `asyncio.to_thread(next, gen, ...)` to keep the event loop free while "+ "whisper yields the next chunk")@@ -600,30 +605,14 @@ def test_generate_stream_offloads_blocking_next_to_thread():assert "generate_chat_response" not in it_src, ("generate_stream's inner stream() must not iterate ""backend.generate_chat_response() directly -- that blocks the event "- "loop on every blocking subprocess read between tokens. Use "- "`await asyncio.to_thread(next, gen, _DONE)` inside a `while True` "- "loop instead"+ "loop on every blocking subprocess read between tokens. Step it with "+ "`ThreadStepper(gen).next` inside a `while True` loop instead")- found_to_thread_next = False- for sub in ast.walk(inner):- if not isinstance(sub, ast.Call):- continue- fn_expr = sub.func- if not (- isinstance(fn_expr, ast.Attribute)- and fn_expr.attr == "to_thread"- and isinstance(fn_expr.value, ast.Name)- and fn_expr.value.id == "asyncio"- ):- continue- if sub.args and isinstance(sub.args[0], ast.Name) and sub.args[0].id == "next":- found_to_thread_next = True- break- assert found_to_thread_next, (- "generate_stream's inner stream() must call "- "`asyncio.to_thread(next, gen, _DONE)` to keep the event loop free while the "- "worker subprocess produces the next token"+ assert _steps_generator_off_loop(inner), (+ "generate_stream's inner stream() must step its generator with "+ "`ThreadStepper(gen).next` or `asyncio.to_thread(next, gen, _DONE)` to keep "+ "the event loop free while the worker subprocess produces the next token")