Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: step streaming generators on a dedicated thread instead of the pool (#15590)

Daniel Han975715cb440f
diff --git a/studio/backend/core/inference/gen_stepper.py b/studio/backend/core/inference/gen_stepper.py
new file mode 100644
index 00000000000..71ced0e1a13
--- /dev/null
+++ b/studio/backend/core/inference/gen_stepper.py
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Advance a blocking generator off the event loop, one step per await, on its own thread."""
+
+from __future__ import annotations
+
+import asyncio
+import contextvars
+import queue
+import threading
+import weakref
+
+# A stepper whose consumer stops asking (slow client) frees its thread after this.
+_IDLE_EXIT_S = 30.0
+
+
+def _resolve(future: asyncio.Future, result, error) -> None:
+ if future.cancelled():
+ return
+ if error is None:
+ future.set_result(result)
+ else:
+ future.set_exception(error)
+
+
+def _waker(requests: queue.SimpleQueue):
+ # SimpleQueue.put is reentrant, so this is safe from a weakref callback run by the GC.
+ return lambda _ref: requests.put(None)
+
+
+def _work(ref, requests: queue.SimpleQueue) -> None:
+ # Holds the stepper only during a step, so a dropped stream's generator is finalized at once.
+ while True:
+ try:
+ request = requests.get(timeout = _IDLE_EXIT_S)
+ except queue.Empty:
+ stepper = ref()
+ if stepper is None:
+ return
+ with stepper._lock:
+ if requests.empty():
+ stepper._running = False
+ return
+ del stepper
+ continue
+ if request is None:
+ return
+ loop, future, context, default = request
+ stepper = ref()
+ if stepper is None:
+ return
+ try:
+ result, error = context.run(next, stepper._gen, default), None
+ except BaseException as exc:
+ result, error = None, exc
+ del stepper
+ try:
+ loop.call_soon_threadsafe(_resolve, future, result, error)
+ except RuntimeError:
+ pass
+ del request, loop, future, context, default, result, error
+
+
+class ThreadStepper:
+ """``await stepper.next(default)`` behaves like ``await asyncio.to_thread(next, gen, default)``.
+
+ Same context copy, exception and cancellation behaviour (a cancelled await abandons the step,
+ which still finishes), but every step runs on one dedicated thread instead of a thread-pool
+ round trip, and steps run one at a time in request order.
+ """
+
+ __slots__ = ("_gen", "_lock", "_requests", "_running", "__weakref__")
+
+ def __init__(self, gen):
+ self._gen = gen
+ self._lock = threading.Lock()
+ self._requests = queue.SimpleQueue()
+ self._running = False
+
+ async def next(self, default):
+ loop = asyncio.get_running_loop()
+ future = loop.create_future()
+ request = (loop, future, contextvars.copy_context(), default)
+ with self._lock:
+ if not self._running:
+ threading.Thread(
+ target = _work,
+ args = (weakref.ref(self, _waker(self._requests)), self._requests),
+ name = "studio-gen-step",
+ daemon = True,
+ ).start()
+ self._running = True
+ self._requests.put(request)
+ return await future
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index f1bdab4e85c..6f1123addcf 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -131,6 +131,7 @@ from core.inference.audio_errors import (
)
from core.inference.audio_cpp_outputs import NO_STEMS
from core.inference import context_refusal
+from core.inference.gen_stepper import ThreadStepper
from core.inference.context_window import (
estimate_message_tokens as _estimate_message_tokens,
estimate_messages_tokens as _estimate_messages_tokens,
@@ -21437,6 +21438,7 @@ async def generate_stream(
presence_penalty = request.presence_penalty,
cancel_event = cancel_event,
)
+ gen_step = ThreadStepper(gen).next
_DONE = object()
while True:
if cancel_event.is_set():
@@ -21445,7 +21447,7 @@ async def generate_stream(
# Flag failed so the middleware does not claim the model on a cancelled stream.
mark_response_failed(_gs_scope)
break
- chunk = await asyncio.to_thread(next, gen, _DONE)
+ chunk = await gen_step(_DONE)
if chunk is _DONE:
if cancel_event.is_set():
mark_response_failed(_gs_scope)
@@ -30633,6 +30635,7 @@ async def produce_openai_chat_completions(
yield _chat_role_chunk(completion_id, created, model_name)
gen = audio_input_generate()
+ gen_step = ThreadStepper(gen).next
_DONE = object()
cancelled = False
while True:
@@ -30646,7 +30649,7 @@ async def produce_openai_chat_completions(
api_monitor.finish(monitor_id, "cancelled")
mark_current_response_failed()
return
- chunk_text = await asyncio.to_thread(next, gen, _DONE)
+ chunk_text = await gen_step(_DONE)
if chunk_text is _DONE:
if cancel_event.is_set():
cancelled = True
@@ -31502,6 +31505,7 @@ async def produce_openai_chat_completions(
context_refusal.open_slot()
# In a thread so the event loop stays free for disconnect detection.
gen = gguf_generate_with_tools()
+ gen_step = ThreadStepper(gen).next
prev_text = ""
_gguf_display_strip = _StreamingDisplayStrip(
auto_heal_tool_calls = _gguf_auto_heal_tool_calls,
@@ -31546,9 +31550,7 @@ async def produce_openai_chat_completions(
resume_before_next = False
await _park_admission(False)
- next_task = asyncio.create_task(
- asyncio.to_thread(next, gen, _tool_sentinel)
- )
+ next_task = asyncio.create_task(gen_step(_tool_sentinel))
try:
# asyncio.wait never cancels next_task, matching the finally-drain shield.
wait_timeout = (
@@ -32224,6 +32226,7 @@ async def produce_openai_chat_completions(
# In a thread so the event loop stays free for disconnect detection.
gen = gguf_generate()
+ gen_step = ThreadStepper(gen).next
prev_text = ""
reasoning_extractor = _new_chat_reasoning_extractor()
_stream_usage = None
@@ -32238,9 +32241,7 @@ async def produce_openai_chat_completions(
mark_current_response_failed()
api_monitor.finish(monitor_id, "cancelled")
return
- next_task = asyncio.create_task(
- asyncio.to_thread(next, gen, _gguf_sentinel)
- )
+ next_task = asyncio.create_task(gen_step(_gguf_sentinel))
try:
# Stall-timeout wait: keepalive while the generator stays
# silent (e.g. no-tool prefill). asyncio.wait never cancels
@@ -33370,6 +33371,7 @@ async def produce_openai_chat_completions(
yield _chat_role_chunk(completion_id, created, model_name)
gen = sf_generate_with_tools()
+ gen_step = ThreadStepper(gen).next
prev_text = ""
_sf_display_strip = _StreamingDisplayStrip(
auto_heal_tool_calls = _sf_auto_heal_tool_calls,
@@ -33404,9 +33406,7 @@ async def produce_openai_chat_completions(
return
# Keepalive so silent backend segments do not exceed proxy timeouts.
- _sf_next_task = asyncio.create_task(
- asyncio.to_thread(next, gen, _sf_tool_sentinel)
- )
+ _sf_next_task = asyncio.create_task(gen_step(_sf_tool_sentinel))
wait_timeout = (
TOOL_APPROVAL_FLUSH_DELAY_S
if approval_flush_pending
@@ -34065,13 +34065,14 @@ async def produce_openai_chat_completions(
# Worker thread: a blocking _gen_lock in compare mode would otherwise freeze the loop.
_DONE = object() # sentinel for generator exhaustion
gen = generate()
+ gen_step = ThreadStepper(gen).next
while True:
if cancel_event.is_set():
backend.reset_generation_state(cancel_event)
mark_current_response_failed()
break
# next(gen, _DONE) avoids StopIteration, which can't cross asyncio futures.
- _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _DONE))
+ _next_task = asyncio.create_task(gen_step(_DONE))
while True:
_done_tasks, _ = await asyncio.wait(
{_next_task},
@@ -36463,11 +36464,12 @@ async def _mlx_completions(request, current_subject, backend, body: dict, refusa
)
_DONE = object()
gen = generate(prompt_ids) if budget != 0 else iter(())
+ gen_step = ThreadStepper(gen).next
prev = ""
while True:
if cancel_event.is_set():
break
- next_task = asyncio.create_task(asyncio.to_thread(next, gen, _DONE))
+ next_task = asyncio.create_task(gen_step(_DONE))
while not (
await asyncio.wait(
{next_task}, timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S
@@ -43636,6 +43638,7 @@ async def _anthropic_tool_stream(
# The respawn refit records the refusal from the worker thread, so open the slot out here first.
context_refusal.open_slot()
gen = run_gen()
+ gen_step = ThreadStepper(gen).next
_next_task = None
# The in-loop poll fires only between events, so a mid-prefill disconnect needs a watcher.
disconnect_watcher = asyncio.create_task(
@@ -43649,7 +43652,7 @@ async def _anthropic_tool_stream(
mark_response_failed(getattr(request, "scope", None))
return
# Stall keepalive: silent backend segments must not idle the SSE stream past proxy timeouts.
- _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel))
+ _next_task = asyncio.create_task(gen_step(_sentinel))
while True:
_done_tasks, _ = await asyncio.wait(
{_next_task},
@@ -43730,6 +43733,7 @@ async def _anthropic_plain_stream(
captured_finish_reason = None
gen = run_gen()
+ gen_step = ThreadStepper(gen).next
_next_task = None
# Watcher cancels on disconnect since the in-loop poll only runs between chunks, not mid-prefill.
disconnect_watcher = asyncio.create_task(
@@ -43743,7 +43747,7 @@ async def _anthropic_plain_stream(
mark_response_failed(getattr(request, "scope", None))
return
# Stall keepalive each window while next(gen) runs in a worker.
- _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel))
+ _next_task = asyncio.create_task(gen_step(_sentinel))
while True:
_done_tasks, _ = await asyncio.wait(
{_next_task},
diff --git a/studio/backend/tests/test_gen_stepper.py b/studio/backend/tests/test_gen_stepper.py
new file mode 100644
index 00000000000..a54b2da6bd6
--- /dev/null
+++ b/studio/backend/tests/test_gen_stepper.py
@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""ThreadStepper matches asyncio.to_thread(next, gen, default) step for step."""
+
+import asyncio
+import contextvars
+import threading
+import time
+
+import pytest
+
+from core.inference import gen_stepper
+from core.inference.gen_stepper import ThreadStepper
+
+_VAR = contextvars.ContextVar("_VAR", default = "unset")
+
+
+def _drive(step, n):
+ async def run():
+ return [await step(None) for _ in range(n)]
+
+ return asyncio.run(run())
+
+
+def test_steps_match_to_thread():
+ def gen():
+ yield from ("a", "b", "c")
+
+ async def via_to_thread(default):
+ return await asyncio.to_thread(next, base, default)
+
+ base = gen()
+ assert (
+ _drive(ThreadStepper(gen()).next, 5)
+ == _drive(via_to_thread, 5)
+ == ["a", "b", "c", None, None]
+ )
+
+
+def test_one_thread_runs_every_step_in_order():
+ seen = []
+
+ def gen():
+ for i in range(50):
+ seen.append(threading.get_ident())
+ yield i
+
+ assert _drive(ThreadStepper(gen()).next, 51) == list(range(50)) + [None]
+ assert len(set(seen)) == 1 and seen[0] != threading.get_ident()
+
+
+def test_exception_propagates_and_generator_finishes():
+ def gen():
+ yield 1
+ raise KeyError("boom")
+
+ async def run():
+ step = ThreadStepper(gen()).next
+ first = await step(None)
+ with pytest.raises(KeyError):
+ await step(None)
+ return first, await step("done")
+
+ assert asyncio.run(run()) == (1, "done")
+
+
+def test_context_is_copied_per_step():
+ def gen():
+ while True:
+ yield _VAR.get()
+
+ async def run():
+ step = ThreadStepper(gen()).next
+ _VAR.set("one")
+ a = await step(None)
+ _VAR.set("two")
+ return a, await step(None)
+
+ assert asyncio.run(run()) == ("one", "two")
+
+
+def test_cancelled_wait_abandons_the_step_only():
+ release = threading.Event()
+
+ def gen():
+ release.wait(5)
+ yield "slow"
+ yield "next"
+
+ async def run():
+ step = ThreadStepper(gen()).next
+ task = asyncio.create_task(step(None))
+ await asyncio.sleep(0.05)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ release.set()
+ return await step(None)
+
+ assert asyncio.run(run()) == "next"
+
+
+def test_idle_thread_exits_and_restarts(monkeypatch):
+ monkeypatch.setattr(gen_stepper, "_IDLE_EXIT_S", 0.05)
+
+ def gen():
+ yield from range(3)
+
+ stepper = ThreadStepper(gen())
+
+ async def run():
+ first = await stepper.next(None)
+ deadline = time.monotonic() + 5
+ while stepper._running and time.monotonic() < deadline:
+ await asyncio.sleep(0.02)
+ assert not stepper._running
+ return first, await stepper.next(None)
+
+ assert asyncio.run(run()) == (0, 1)
+
+
+def test_idle_thread_does_not_keep_a_dropped_generator_alive():
+ closed = threading.Event()
+
+ def gen():
+ try:
+ yield from range(10)
+ finally:
+ closed.set()
+
+ async def run():
+ stepper = ThreadStepper(gen())
+ first = await stepper.next(None)
+ del stepper
+ return first
+
+ assert asyncio.run(run()) == 0
+ import gc
+
+ gc.collect()
+ # The idle worker waits up to _IDLE_EXIT_S, but must not hold the generator meanwhile.
+ assert closed.wait(2)
+
+
+def test_dropping_a_stepper_frees_its_thread_before_the_idle_timeout():
+ import gc
+
+ def workers():
+ return [t for t in threading.enumerate() if t.name == "studio-gen-step"]
+
+ # By identity, not count: a worker woken by an earlier test may still be exiting.
+ before = set(workers())
+ started = []
+
+ async def run():
+ steppers = [ThreadStepper(iter(range(3))) for _ in range(32)]
+ assert [await stepper.next(None) for stepper in steppers] == [0] * 32
+ started.extend(t for t in workers() if t not in before)
+ assert len(started) == 32
+ steppers.clear()
+
+ asyncio.run(run())
+ gc.collect()
+ # _IDLE_EXIT_S is 30 s; a dropped stepper's worker is woken instead of left to wait it out.
+ deadline = time.monotonic() + 5
+ for t in started:
+ t.join(max(0.0, deadline - time.monotonic()))
+ alive = [t for t in started if t.is_alive()]
+ assert not alive, len(alive)
diff --git a/tests/studio/test_stream_cancel_registration_timing.py b/tests/studio/test_stream_cancel_registration_timing.py
index 9c2f4f9fd22..b9c7e1d11dd 100644
--- a/tests/studio/test_stream_cancel_registration_timing.py
+++ b/tests/studio/test_stream_cancel_registration_timing.py
@@ -540,6 +540,26 @@ def test_streaming_generators_check_cancel_event_in_loop():
)
+def _steps_generator_off_loop(fn) -> bool:
+ for sub in ast.walk(fn):
+ if not isinstance(sub, ast.Call):
+ continue
+ f = sub.func
+ if isinstance(f, ast.Name) and f.id == "ThreadStepper":
+ return True
+ if (
+ isinstance(f, ast.Attribute)
+ and f.attr == "to_thread"
+ and isinstance(f.value, ast.Name)
+ and f.value.id == "asyncio"
+ and sub.args
+ and isinstance(sub.args[0], ast.Name)
+ and sub.args[0].id == "next"
+ ):
+ return True
+ return False
+
+
def test_audio_input_stream_offloads_blocking_next_to_thread():
audio = None
for fn in ast.walk(_TREE):
@@ -553,29 +573,14 @@ def test_audio_input_stream_offloads_blocking_next_to_thread():
it_src = ast.unparse(sub.iter)
assert "audio_input_generate" not in it_src, (
"audio_input_stream must not iterate audio_input_generate() "
- "directly -- that blocks the event loop. Use "
- "`await asyncio.to_thread(next, gen, _DONE)` inside a "
- "`while True` loop instead"
+ "directly -- that blocks the event loop. Step it with "
+ "`ThreadStepper(gen).next` inside a `while True` loop instead"
)
- found_to_thread_next = False
- for sub in ast.walk(audio):
- if not isinstance(sub, ast.Call):
- continue
- fn_expr = sub.func
- if not (
- isinstance(fn_expr, ast.Attribute)
- and fn_expr.attr == "to_thread"
- and isinstance(fn_expr.value, ast.Name)
- and fn_expr.value.id == "asyncio"
- ):
- continue
- if sub.args and isinstance(sub.args[0], ast.Name) and sub.args[0].id == "next":
- found_to_thread_next = True
- break
- assert found_to_thread_next, (
- "audio_input_stream must call `asyncio.to_thread(next, gen, ...)` "
- "to keep the event loop free while whisper yields the next chunk"
+ assert _steps_generator_off_loop(audio), (
+ "audio_input_stream must step its generator with `ThreadStepper(gen).next` "
+ "or `asyncio.to_thread(next, gen, ...)` to keep the event loop free while "
+ "whisper yields the next chunk"
)
@@ -600,30 +605,14 @@ def test_generate_stream_offloads_blocking_next_to_thread():
assert "generate_chat_response" not in it_src, (
"generate_stream's inner stream() must not iterate "
"backend.generate_chat_response() directly -- that blocks the event "
- "loop on every blocking subprocess read between tokens. Use "
- "`await asyncio.to_thread(next, gen, _DONE)` inside a `while True` "
- "loop instead"
+ "loop on every blocking subprocess read between tokens. Step it with "
+ "`ThreadStepper(gen).next` inside a `while True` loop instead"
)
- found_to_thread_next = False
- for sub in ast.walk(inner):
- if not isinstance(sub, ast.Call):
- continue
- fn_expr = sub.func
- if not (
- isinstance(fn_expr, ast.Attribute)
- and fn_expr.attr == "to_thread"
- and isinstance(fn_expr.value, ast.Name)
- and fn_expr.value.id == "asyncio"
- ):
- continue
- if sub.args and isinstance(sub.args[0], ast.Name) and sub.args[0].id == "next":
- found_to_thread_next = True
- break
- assert found_to_thread_next, (
- "generate_stream's inner stream() must call "
- "`asyncio.to_thread(next, gen, _DONE)` to keep the event loop free while the "
- "worker subprocess produces the next token"
+ assert _steps_generator_off_loop(inner), (
+ "generate_stream's inner stream() must step its generator with "
+ "`ThreadStepper(gen).next` or `asyncio.to_thread(next, gen, _DONE)` to keep "
+ "the event loop free while the worker subprocess produces the next token"
)