Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Follow the multi-model refactor in two guards that went red on main (#12653)

Daniel Han3afc0b570da3
diff --git a/studio/backend/tests/test_async_singleton_access.py b/studio/backend/tests/test_async_singleton_access.py
index ded8d3d075d..e661c123279 100644
--- a/studio/backend/tests/test_async_singleton_access.py
+++ b/studio/backend/tests/test_async_singleton_access.py
@@ -18,6 +18,8 @@ import ast
import sys
from pathlib import Path
+import pytest
+
_BACKEND = Path(__file__).resolve().parent.parent
if str(_BACKEND) not in sys.path:
sys.path.insert(0, str(_BACKEND))
@@ -96,6 +98,78 @@ def _sync_helpers_that_build_the_singleton(rel: str) -> set[str]:
return names
+# Workers that call a callable they are handed, on the worker thread. to_thread runs only its
+# first argument; a lambda further along is just passed to it, so it counts as off the loop only
+# when that worker is known to invoke it there. Pinned below by reading the worker itself.
+# Keyed by the qualified name the routes call it by, so an unrelated in_slot elsewhere is not
+# trusted: "module.function" -> (its file, the name and position of the parameter it calls).
+_WORKERS_THAT_RUN_THEIR_CALLBACK = {
+ "model_slots.in_slot": ("core/inference/model_slots.py", "fn", 1),
+}
+
+
+def _executed_calls(lam: ast.Lambda) -> list[ast.Call]:
+ # A lambda with a yield in its own body is a generator function: calling it runs nothing.
+ if any(isinstance(n, (ast.Yield, ast.YieldFrom)) for n in _nodes_in_scope([lam.body])):
+ return []
+ return _calls_run_in_scope([lam.body])
+
+
+def _nodes_in_scope(roots: list[ast.AST]) -> list[ast.AST]:
+ found, stack = [], list(roots)
+ while stack:
+ node = stack.pop()
+ if isinstance(node, ast.GeneratorExp):
+ # Only the outermost iterable is evaluated when the generator is built.
+ stack.append(node.generators[0].iter)
+ continue
+ if isinstance(node, (ast.Lambda, ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue
+ found.append(node)
+ stack.extend(ast.iter_child_nodes(node))
+ return found
+
+
+def _calls_run_in_scope(roots: list[ast.AST]) -> list[ast.Call]:
+ """Calls these nodes make when their own scope runs. A lambda, def or generator expression
+ nested inside is only created there, and could be returned and run later on the loop, so it
+ is not descended."""
+ return [n for n in _nodes_in_scope(roots) if isinstance(n, ast.Call)]
+
+
+def _calls_inside_offloaded_lambdas(fn: ast.AST) -> set[int]:
+ """ids of the Call nodes in a lambda that asyncio.to_thread runs on its worker thread: the
+ lambda is to_thread's first argument, or a later argument to a worker listed in
+ _WORKERS_THAT_RUN_THEIR_CALLBACK. Any other lambda, stored or handed to a worker that might
+ return it, is not exempt."""
+ inside = set()
+ for node in ast.walk(fn):
+ if not (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "to_thread"
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id == "asyncio"
+ and node.args
+ ):
+ continue
+ worker, *rest = node.args
+ lambdas = [worker] if isinstance(worker, ast.Lambda) else []
+ worker_name = (
+ f"{worker.value.id}.{worker.attr}"
+ if isinstance(worker, ast.Attribute) and isinstance(worker.value, ast.Name)
+ else None
+ )
+ if worker_name in _WORKERS_THAT_RUN_THEIR_CALLBACK:
+ _file, param, position = _WORKERS_THAT_RUN_THEIR_CALLBACK[worker_name]
+ callback = [rest[position]] if len(rest) > position else []
+ callback += [kw.value for kw in node.keywords if kw.arg == param]
+ lambdas += [a for a in callback if isinstance(a, ast.Lambda)]
+ for lam in lambdas:
+ inside.update(id(n) for n in _executed_calls(lam))
+ return inside
+
+
def test_no_async_handler_reaches_the_singleton_through_a_sync_helper():
"""The direct sweep is not enough: a sync helper hides the same stall. _loaded_satisfies
calls get_inference_backend() inline, so an async handler calling it on the loop pays
@@ -109,13 +183,17 @@ def test_no_async_handler_reaches_the_singleton_through_a_sync_helper():
for fn in ast.walk(tree):
if not isinstance(fn, ast.AsyncFunctionDef):
continue
+ off_loop = _calls_inside_offloaded_lambdas(fn)
for sub in ast.walk(fn):
# A bare Call to the helper runs it on the loop; passing it to
- # to_thread makes it an ast.Name argument, never a Call.
+ # to_thread makes it an ast.Name argument, never a Call. A call inside a
+ # lambda handed to to_thread runs on the worker thread too (#11591's
+ # get_active_generations does `to_thread(..., lambda: _loaded_satisfies(model))`).
if (
isinstance(sub, ast.Call)
and isinstance(sub.func, ast.Name)
and sub.func.id in helpers
+ and id(sub) not in off_loop
):
offenders.append(f"{rel}:{sub.lineno} async {fn.name} -> {sub.func.id}()")
@@ -202,3 +280,91 @@ def test_read_only_endpoints_never_construct_the_singleton():
"read-only paths construct the inference singleton, so a status poll or a "
"metadata-only delete imports torch on a warm-disabled host:\n " + "\n ".join(offenders)
)
+
+
+def test_a_lambda_handed_to_to_thread_is_off_the_loop_but_an_inline_call_is_not():
+ """The exemption is exactly as wide as the offload: the same helper call on the loop, in a
+ stored lambda, or in a lambda handed to a worker that may only return it is still reported."""
+ fn = ast.parse(
+ "async def handler(model):\n"
+ " await asyncio.to_thread(slots.in_slot, None, lambda: helper(model))\n"
+ " await asyncio.to_thread(lambda: helper(model))\n"
+ " stored = lambda: helper(model)\n"
+ " helper(model)\n"
+ " (await asyncio.to_thread(identity, lambda: helper(model)))()\n"
+ " (await asyncio.to_thread(lambda: lambda: helper(model)))()\n"
+ " await asyncio.to_thread(model_slots.in_slot, None, lambda: helper(model))\n"
+ " await asyncio.to_thread(other.in_slot, None, lambda: helper(model))\n"
+ " (await asyncio.to_thread(lambda: (helper(model) for _ in range(1)))).__next__()\n"
+ " await dispatcher.to_thread(lambda: helper(model))\n"
+ " (await asyncio.to_thread(lambda: (yield helper(model)))).__next__()\n"
+ " await asyncio.to_thread(lambda: (x for x in helper(model)))\n"
+ ).body[0]
+ off_loop = _calls_inside_offloaded_lambdas(fn)
+ calls = [
+ n
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "helper"
+ ]
+ assert sorted(n.lineno for n in calls if id(n) in off_loop) == [3, 8, 13]
+ assert sorted(n.lineno for n in calls if id(n) not in off_loop) == [
+ 2,
+ 4,
+ 5,
+ 6,
+ 7,
+ 9,
+ 10,
+ 11,
+ 12,
+ ]
+
+
[email protected]("worker", sorted(_WORKERS_THAT_RUN_THEIR_CALLBACK))
+def test_each_listed_worker_still_runs_the_callable_it_is_handed(worker):
+ """The exemption trusts these workers to call their callback; read that off their source."""
+ rel, param, position = _WORKERS_THAT_RUN_THEIR_CALLBACK[worker]
+ tree = ast.parse((_BACKEND / rel).read_text(encoding = "utf-8"))
+ module, name = worker.split(".")
+ assert Path(rel).stem == module
+ # Every scanned route that calls it by that name must have imported that module.
+ for route in _ROUTE_FILES:
+ source = (_BACKEND / route).read_text(encoding = "utf-8")
+ if f"{worker}(" in source or f"{worker}," in source:
+ assert (
+ f"from core.inference import {module}\n" in source
+ ), f"{route} calls {worker} but does not import it from {rel}"
+ fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == name)
+ assert [a.arg for a in fn.args.args][position] == param, f"{worker}'s callback moved"
+ # A name bound to contextvars.copy_context() inside the worker: its .run is synchronous.
+ contexts = {
+ target.id
+ for n in _nodes_in_scope(fn.body)
+ if isinstance(n, ast.Assign)
+ and isinstance(n.value, ast.Call)
+ and isinstance(n.value.func, ast.Attribute)
+ and n.value.func.attr == "copy_context"
+ and isinstance(n.value.func.value, ast.Name)
+ and n.value.func.value.id == "contextvars"
+ for target in n.targets
+ if isinstance(target, ast.Name)
+ }
+ # Called directly, or run through contextvars' Context.run: forwarding it anywhere else could
+ # return it uncalled, so it does not count.
+ runs = [
+ n
+ for n in _calls_run_in_scope(fn.body)
+ if (
+ (isinstance(n.func, ast.Name) and n.func.id == param)
+ or (
+ isinstance(n.func, ast.Attribute)
+ and n.func.attr == "run"
+ and isinstance(n.func.value, ast.Name)
+ and n.func.value.id in contexts
+ and n.args[:1]
+ and isinstance(n.args[0], ast.Name)
+ and n.args[0].id == param
+ )
+ )
+ ]
+ assert runs, f"{worker} no longer runs its {param} argument; drop it from the list"
diff --git a/tests/studio/test_multi_chat_prompt_queue_contract.py b/tests/studio/test_multi_chat_prompt_queue_contract.py
index 4bbbceab2f7..096988e6034 100644
--- a/tests/studio/test_multi_chat_prompt_queue_contract.py
+++ b/tests/studio/test_multi_chat_prompt_queue_contract.py
@@ -716,12 +716,23 @@ def test_queued_settings_are_thread_scoped_without_cross_chat_fallback():
< side_two.index("handle2.startRun()")
)
assert "requestLocalPromptQueueStop" in eject
+ # #11591 moved the stop into stopQueuedRuns, shared with unloading a kept model, so the order
+ # is read across the call: eject confirms first and then stops through the helper, and the
+ # helper's whole-runtime branch cancels pre-stream reservations before stopping the queue.
assert (
eject.index('beginModelLoading("unloading")')
< eject.index("await confirmStopRunningChatsIfNeeded(")
- < eject.index("cancelPreStreamRunReservations(stopDecision.preStreamRunTokens)")
- < eject.index("requestLocalPromptQueueStop(stopDecision.promptQueueThreadIds)")
+ < eject.index("stopQueuedRuns(stopDecision, Boolean(scope))")
)
+ stop_queued = _between(
+ MODEL_RUNTIME,
+ "function stopQueuedRuns(decision: StopRunningChatsDecision, scoped: boolean): void {",
+ "\n}\n",
+ )
+ whole_runtime = stop_queued.split("if (scoped) {", 1)[1].split("\n }\n", 1)[1]
+ assert whole_runtime.index(
+ "cancelPreStreamRunReservations(decision.preStreamRunTokens)"
+ ) < whole_runtime.index("requestLocalPromptQueueStop(decision.promptQueueThreadIds)")
assert "function promptQueueRunUsesLocalModel(run: PromptQueueRun)" in THREAD
assert ".slice(Math.max(run.index, 0))" in THREAD
assert ".some((item) => item.target.usesLocalModel)" in THREAD