Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Follow the multi-model refactor in two guards that went red on main (#12653)
Daniel Han3afc0b570da3
diff --git a/studio/backend/tests/test_async_singleton_access.py b/studio/backend/tests/test_async_singleton_access.pyindex ded8d3d075d..e661c123279 100644--- a/studio/backend/tests/test_async_singleton_access.py+++ b/studio/backend/tests/test_async_singleton_access.py@@ -18,6 +18,8 @@ import astimport sysfrom pathlib import Path+import pytest+_BACKEND = Path(__file__).resolve().parent.parentif str(_BACKEND) not in sys.path:sys.path.insert(0, str(_BACKEND))@@ -96,6 +98,78 @@ def _sync_helpers_that_build_the_singleton(rel: str) -> set[str]:return names+# Workers that call a callable they are handed, on the worker thread. to_thread runs only its+# first argument; a lambda further along is just passed to it, so it counts as off the loop only+# when that worker is known to invoke it there. Pinned below by reading the worker itself.+# Keyed by the qualified name the routes call it by, so an unrelated in_slot elsewhere is not+# trusted: "module.function" -> (its file, the name and position of the parameter it calls).+_WORKERS_THAT_RUN_THEIR_CALLBACK = {+ "model_slots.in_slot": ("core/inference/model_slots.py", "fn", 1),+}+++def _executed_calls(lam: ast.Lambda) -> list[ast.Call]:+ # A lambda with a yield in its own body is a generator function: calling it runs nothing.+ if any(isinstance(n, (ast.Yield, ast.YieldFrom)) for n in _nodes_in_scope([lam.body])):+ return []+ return _calls_run_in_scope([lam.body])+++def _nodes_in_scope(roots: list[ast.AST]) -> list[ast.AST]:+ found, stack = [], list(roots)+ while stack:+ node = stack.pop()+ if isinstance(node, ast.GeneratorExp):+ # Only the outermost iterable is evaluated when the generator is built.+ stack.append(node.generators[0].iter)+ continue+ if isinstance(node, (ast.Lambda, ast.FunctionDef, ast.AsyncFunctionDef)):+ continue+ found.append(node)+ stack.extend(ast.iter_child_nodes(node))+ return found+++def _calls_run_in_scope(roots: list[ast.AST]) -> list[ast.Call]:+ """Calls these nodes make when their own scope runs. A lambda, def or generator expression+ nested inside is only created there, and could be returned and run later on the loop, so it+ is not descended."""+ return [n for n in _nodes_in_scope(roots) if isinstance(n, ast.Call)]+++def _calls_inside_offloaded_lambdas(fn: ast.AST) -> set[int]:+ """ids of the Call nodes in a lambda that asyncio.to_thread runs on its worker thread: the+ lambda is to_thread's first argument, or a later argument to a worker listed in+ _WORKERS_THAT_RUN_THEIR_CALLBACK. Any other lambda, stored or handed to a worker that might+ return it, is not exempt."""+ inside = set()+ for node in ast.walk(fn):+ if not (+ isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "to_thread"+ and isinstance(node.func.value, ast.Name)+ and node.func.value.id == "asyncio"+ and node.args+ ):+ continue+ worker, *rest = node.args+ lambdas = [worker] if isinstance(worker, ast.Lambda) else []+ worker_name = (+ f"{worker.value.id}.{worker.attr}"+ if isinstance(worker, ast.Attribute) and isinstance(worker.value, ast.Name)+ else None+ )+ if worker_name in _WORKERS_THAT_RUN_THEIR_CALLBACK:+ _file, param, position = _WORKERS_THAT_RUN_THEIR_CALLBACK[worker_name]+ callback = [rest[position]] if len(rest) > position else []+ callback += [kw.value for kw in node.keywords if kw.arg == param]+ lambdas += [a for a in callback if isinstance(a, ast.Lambda)]+ for lam in lambdas:+ inside.update(id(n) for n in _executed_calls(lam))+ return inside++def test_no_async_handler_reaches_the_singleton_through_a_sync_helper():"""The direct sweep is not enough: a sync helper hides the same stall. _loaded_satisfiescalls get_inference_backend() inline, so an async handler calling it on the loop pays@@ -109,13 +183,17 @@ def test_no_async_handler_reaches_the_singleton_through_a_sync_helper():for fn in ast.walk(tree):if not isinstance(fn, ast.AsyncFunctionDef):continue+ off_loop = _calls_inside_offloaded_lambdas(fn)for sub in ast.walk(fn):# A bare Call to the helper runs it on the loop; passing it to- # to_thread makes it an ast.Name argument, never a Call.+ # to_thread makes it an ast.Name argument, never a Call. A call inside a+ # lambda handed to to_thread runs on the worker thread too (#11591's+ # get_active_generations does `to_thread(..., lambda: _loaded_satisfies(model))`).if (isinstance(sub, ast.Call)and isinstance(sub.func, ast.Name)and sub.func.id in helpers+ and id(sub) not in off_loop):offenders.append(f"{rel}:{sub.lineno} async {fn.name} -> {sub.func.id}()")@@ -202,3 +280,91 @@ def test_read_only_endpoints_never_construct_the_singleton():"read-only paths construct the inference singleton, so a status poll or a ""metadata-only delete imports torch on a warm-disabled host:\n " + "\n ".join(offenders))+++def test_a_lambda_handed_to_to_thread_is_off_the_loop_but_an_inline_call_is_not():+ """The exemption is exactly as wide as the offload: the same helper call on the loop, in a+ stored lambda, or in a lambda handed to a worker that may only return it is still reported."""+ fn = ast.parse(+ "async def handler(model):\n"+ " await asyncio.to_thread(slots.in_slot, None, lambda: helper(model))\n"+ " await asyncio.to_thread(lambda: helper(model))\n"+ " stored = lambda: helper(model)\n"+ " helper(model)\n"+ " (await asyncio.to_thread(identity, lambda: helper(model)))()\n"+ " (await asyncio.to_thread(lambda: lambda: helper(model)))()\n"+ " await asyncio.to_thread(model_slots.in_slot, None, lambda: helper(model))\n"+ " await asyncio.to_thread(other.in_slot, None, lambda: helper(model))\n"+ " (await asyncio.to_thread(lambda: (helper(model) for _ in range(1)))).__next__()\n"+ " await dispatcher.to_thread(lambda: helper(model))\n"+ " (await asyncio.to_thread(lambda: (yield helper(model)))).__next__()\n"+ " await asyncio.to_thread(lambda: (x for x in helper(model)))\n"+ ).body[0]+ off_loop = _calls_inside_offloaded_lambdas(fn)+ calls = [+ n+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "helper"+ ]+ assert sorted(n.lineno for n in calls if id(n) in off_loop) == [3, 8, 13]+ assert sorted(n.lineno for n in calls if id(n) not in off_loop) == [+ 2,+ 4,+ 5,+ 6,+ 7,+ 9,+ 10,+ 11,+ 12,+ ]++[email protected]("worker", sorted(_WORKERS_THAT_RUN_THEIR_CALLBACK))+def test_each_listed_worker_still_runs_the_callable_it_is_handed(worker):+ """The exemption trusts these workers to call their callback; read that off their source."""+ rel, param, position = _WORKERS_THAT_RUN_THEIR_CALLBACK[worker]+ tree = ast.parse((_BACKEND / rel).read_text(encoding = "utf-8"))+ module, name = worker.split(".")+ assert Path(rel).stem == module+ # Every scanned route that calls it by that name must have imported that module.+ for route in _ROUTE_FILES:+ source = (_BACKEND / route).read_text(encoding = "utf-8")+ if f"{worker}(" in source or f"{worker}," in source:+ assert (+ f"from core.inference import {module}\n" in source+ ), f"{route} calls {worker} but does not import it from {rel}"+ fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == name)+ assert [a.arg for a in fn.args.args][position] == param, f"{worker}'s callback moved"+ # A name bound to contextvars.copy_context() inside the worker: its .run is synchronous.+ contexts = {+ target.id+ for n in _nodes_in_scope(fn.body)+ if isinstance(n, ast.Assign)+ and isinstance(n.value, ast.Call)+ and isinstance(n.value.func, ast.Attribute)+ and n.value.func.attr == "copy_context"+ and isinstance(n.value.func.value, ast.Name)+ and n.value.func.value.id == "contextvars"+ for target in n.targets+ if isinstance(target, ast.Name)+ }+ # Called directly, or run through contextvars' Context.run: forwarding it anywhere else could+ # return it uncalled, so it does not count.+ runs = [+ n+ for n in _calls_run_in_scope(fn.body)+ if (+ (isinstance(n.func, ast.Name) and n.func.id == param)+ or (+ isinstance(n.func, ast.Attribute)+ and n.func.attr == "run"+ and isinstance(n.func.value, ast.Name)+ and n.func.value.id in contexts+ and n.args[:1]+ and isinstance(n.args[0], ast.Name)+ and n.args[0].id == param+ )+ )+ ]+ assert runs, f"{worker} no longer runs its {param} argument; drop it from the list"diff --git a/tests/studio/test_multi_chat_prompt_queue_contract.py b/tests/studio/test_multi_chat_prompt_queue_contract.pyindex 4bbbceab2f7..096988e6034 100644--- a/tests/studio/test_multi_chat_prompt_queue_contract.py+++ b/tests/studio/test_multi_chat_prompt_queue_contract.py@@ -716,12 +716,23 @@ def test_queued_settings_are_thread_scoped_without_cross_chat_fallback():< side_two.index("handle2.startRun()"))assert "requestLocalPromptQueueStop" in eject+ # #11591 moved the stop into stopQueuedRuns, shared with unloading a kept model, so the order+ # is read across the call: eject confirms first and then stops through the helper, and the+ # helper's whole-runtime branch cancels pre-stream reservations before stopping the queue.assert (eject.index('beginModelLoading("unloading")')< eject.index("await confirmStopRunningChatsIfNeeded(")- < eject.index("cancelPreStreamRunReservations(stopDecision.preStreamRunTokens)")- < eject.index("requestLocalPromptQueueStop(stopDecision.promptQueueThreadIds)")+ < eject.index("stopQueuedRuns(stopDecision, Boolean(scope))"))+ stop_queued = _between(+ MODEL_RUNTIME,+ "function stopQueuedRuns(decision: StopRunningChatsDecision, scoped: boolean): void {",+ "\n}\n",+ )+ whole_runtime = stop_queued.split("if (scoped) {", 1)[1].split("\n }\n", 1)[1]+ assert whole_runtime.index(+ "cancelPreStreamRunReservations(decision.preStreamRunTokens)"+ ) < whole_runtime.index("requestLocalPromptQueueStop(decision.promptQueueThreadIds)")assert "function promptQueueRunUsesLocalModel(run: PromptQueueRun)" in THREADassert ".slice(Math.max(run.index, 0))" in THREADassert ".some((item) => item.target.usesLocalModel)" in THREAD