Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
"""TOCTOU atomicity guards for the cancel path: single _CANCEL_LOCK critical sections; parallel cancel-POST vs __enter__ never drops a cancel."""
from __future__ import annotations
import ast
import importlib.util
import sys
import random
import threading
from pathlib import Path
SOURCE_PATH = Path(__file__).resolve().parents[2] / "studio" / "backend" / "routes" / "inference.py"
_SRC = SOURCE_PATH.read_text(encoding = "utf-8")
_TREE = ast.parse(_SRC)
def _find_function(name: str) -> ast.FunctionDef | ast.AsyncFunctionDef:
for node in ast.walk(_TREE):
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == name:
return node
raise AssertionError(f"function {name!r} not found")
def _find_class(name: str) -> ast.ClassDef:
for node in ast.walk(_TREE):
if isinstance(node, ast.ClassDef) and node.name == name:
return node
raise AssertionError(f"class {name!r} not found")
def _count_with_cancel_lock_blocks(node: ast.AST) -> int:
n = 0
for sub in ast.walk(node):
if not isinstance(sub, ast.With):
continue
for item in sub.items:
ctx = item.context_expr
if isinstance(ctx, ast.Name) and ctx.id == "_CANCEL_LOCK":
n += 1
break
return n
def test_cancel_by_cancel_id_or_stash_is_single_lock_critical_section():
fn = _find_function("_cancel_by_cancel_id_or_stash")
assert _count_with_cancel_lock_blocks(fn) == 1, (
"_cancel_by_cancel_id_or_stash must use exactly one `with "
"_CANCEL_LOCK:` block; splitting into two acquisitions reopens "
"the TOCTOU race with _TrackedCancel.__enter__"
)
src = ast.unparse(fn)
assert "_CANCEL_REGISTRY.get(cancel_id)" in src
assert "_PENDING_CANCELS[cancel_id]" in src
def test_tracked_cancel_enter_registers_and_consumes_pending_under_one_lock():
cls = _find_class("_TrackedCancel")
enter = None
for n in cls.body:
if isinstance(n, ast.FunctionDef) and n.name == "__enter__":
enter = n
break
assert enter is not None
assert _count_with_cancel_lock_blocks(enter) == 1, (
"_TrackedCancel.__enter__ must acquire _CANCEL_LOCK exactly once. "
"A second acquisition for consume-pending lets a concurrent "
"cancel POST stash after consume sees an empty map, silently "
"dropping the cancel"
)
with_block = None
for sub in ast.walk(enter):
if isinstance(sub, ast.With) and any(
isinstance(i.context_expr, ast.Name) and i.context_expr.id == "_CANCEL_LOCK"
for i in sub.items
):
with_block = sub
break
assert with_block is not None
block_src = "\n".join(ast.unparse(s) for s in with_block.body)
assert "_CANCEL_REGISTRY.setdefault" in block_src
assert "_PENDING_CANCELS.pop" in block_src, (
"__enter__ critical section must consume from _PENDING_CANCELS "
"inside the same lock, not a later re-acquisition"
)
def test_cancel_inference_uses_atomic_helper_for_cancel_id_path():
fn = _find_function("cancel_inference")
src = ast.unparse(fn)
assert "_cancel_by_cancel_id_or_stash" in src
assert "_remember_pending_cancel(cancel_id)" not in src, (
"two-step _cancel_by_keys + _remember_pending_cancel produced "
"the TOCTOU race and must not return"
)
_WANTED = {
"_CANCEL_REGISTRY",
"_CANCEL_LOCK",
"_PENDING_CANCELS",
"_PENDING_CANCEL_TTL_S",
"_prune_pending",
"_remember_pending_cancel",
"_TrackedCancel",
"_slot_lock",
"_cancel_by_keys",
"_cancel_by_cancel_id_or_stash",
}
def _load_active_generations():
"""Loads the real _TrackedCancel registry from disk, avoiding the route package and sys.path."""
path = SOURCE_PATH.parents[1] / "state" / "active_generations.py"
spec = importlib.util.spec_from_file_location("studio_active_generations", path)
module = importlib.util.module_from_spec(spec)
# Its one import resolves from studio/backend; expose that only for this load.
backend = str(SOURCE_PATH.parents[1])
sys.path.insert(0, backend)
try:
spec.loader.exec_module(module)
finally:
sys.path.remove(backend)
return module
_REGISTRY_SOURCE = None
def _registry_source():
"""Joins the _WANTED top-level definitions verbatim, cached per process because slicing is slow."""
global _REGISTRY_SOURCE
if _REGISTRY_SOURCE is not None:
return _REGISTRY_SOURCE
chunks = []
for n in _TREE.body:
if isinstance(n, (ast.FunctionDef, ast.ClassDef)):
wanted = n.name in _WANTED
elif isinstance(n, ast.Assign):
wanted = any(t.id in _WANTED for t in n.targets if isinstance(t, ast.Name))
elif isinstance(n, ast.AnnAssign):
wanted = isinstance(n.target, ast.Name) and n.target.id in _WANTED
else:
wanted = False
if not wanted:
continue
seg = ast.get_source_segment(_SRC, n)
if seg is None:
continue
chunks.append(seg)
_REGISTRY_SOURCE = "\n\n".join(chunks)
return _REGISTRY_SOURCE
def _load_registry_module():
mod = {"active_generations": _load_active_generations()}
exec(
'import contextvars\nrouted_slot = contextvars.ContextVar("routed_slot", default = None)\n'
"import threading, time\nfrom typing import Optional\n_account_cancel_key = lambda key: key\n"
+ _registry_source(),
mod,
)
return mod
def test_parallel_cancel_vs_register_never_drops():
m = _load_registry_module()
trials = 500
dropped = 0
for i in range(trials):
m["_CANCEL_REGISTRY"].clear()
m["_PENDING_CANCELS"].clear()
cid = f"cid-{i}"
ev = threading.Event()
tracker = m["_TrackedCancel"](ev, cid, "thread")
start = threading.Event()
def do_cancel():
start.wait()
m["_cancel_by_cancel_id_or_stash"](cid)
def do_enter():
start.wait()
tracker.__enter__()
threads = [
threading.Thread(target = do_cancel),
threading.Thread(target = do_enter),
]
random.shuffle(threads)
for t in threads:
t.start()
start.set()
for t in threads:
t.join(timeout = 5.0)
assert not t.is_alive()
if not ev.is_set():
dropped += 1
tracker.__exit__(None, None, None)
assert dropped == 0, (
f"TOCTOU regression: {dropped}/{trials} parallel trials silently " f"dropped the cancel"
)
def test_cancel_before_register_replays_atomically():
m = _load_registry_module()
cid = "early-cid"
ev = threading.Event()
tracker = m["_TrackedCancel"](ev, cid, "thread-x")
assert m["_cancel_by_cancel_id_or_stash"](cid) == 0
assert cid in m["_PENDING_CANCELS"]
tracker.__enter__()
assert ev.is_set()
assert cid not in m["_PENDING_CANCELS"]
tracker.__exit__(None, None, None)
def test_cancel_after_register_signals_without_stash():
m = _load_registry_module()
cid = "post-cid"
ev = threading.Event()
tracker = m["_TrackedCancel"](ev, cid, "thread-y")
tracker.__enter__()
assert m["_cancel_by_cancel_id_or_stash"](cid) == 1
assert ev.is_set()
assert cid not in m["_PENDING_CANCELS"]
tracker.__exit__(None, None, None)
def test_cancel_by_keys_tolerates_empty_and_falsy_keys():
m = _load_registry_module()
m["_CANCEL_REGISTRY"].clear()
m["_PENDING_CANCELS"].clear()
assert m["_cancel_by_keys"]([]) == 0
assert m["_cancel_by_keys"](["", None, "unknown"]) == 0
assert m["_PENDING_CANCELS"] == {}
def test_cancel_by_keys_fans_out_to_all_streams_on_same_session():
# Compare mode shares one session_id across concurrent streams; one cancel must hit all.
m = _load_registry_module()
m["_CANCEL_REGISTRY"].clear()
m["_PENDING_CANCELS"].clear()
session = "shared-thread"
ev_a = threading.Event()
ev_b = threading.Event()
tracker_a = m["_TrackedCancel"](ev_a, "cancel-a", session, "chatcmpl-a")
tracker_b = m["_TrackedCancel"](ev_b, "cancel-b", session, "chatcmpl-b")
tracker_a.__enter__()
tracker_b.__enter__()
try:
assert m["_cancel_by_keys"]([session]) == 2
assert ev_a.is_set() and ev_b.is_set()
finally:
tracker_a.__exit__(None, None, None)
tracker_b.__exit__(None, None, None)
assert session not in m["_CANCEL_REGISTRY"]
def test_cancel_by_cancel_id_is_exclusive_to_single_run():
m = _load_registry_module()
m["_CANCEL_REGISTRY"].clear()
m["_PENDING_CANCELS"].clear()
session = "shared-thread-2"
ev_a = threading.Event()
ev_b = threading.Event()
tracker_a = m["_TrackedCancel"](ev_a, "cancel-only-a", session, "chatcmpl-a")
tracker_b = m["_TrackedCancel"](ev_b, "cancel-only-b", session, "chatcmpl-b")
tracker_a.__enter__()
tracker_b.__enter__()
try:
assert m["_cancel_by_cancel_id_or_stash"]("cancel-only-a") == 1
assert ev_a.is_set()
assert not ev_b.is_set()
finally:
tracker_a.__exit__(None, None, None)
tracker_b.__exit__(None, None, None)
def test_tracked_cancel_exit_is_idempotent():
# The outer except and the generator's finally may both call __exit__; must not raise.
m = _load_registry_module()
m["_CANCEL_REGISTRY"].clear()
m["_PENDING_CANCELS"].clear()
ev = threading.Event()
tracker = m["_TrackedCancel"](ev, "cid", "sess", "chatcmpl-x")
tracker.__enter__()
tracker.__exit__(None, None, None)
tracker.__exit__(None, None, None)
tracker.__exit__(None, None, None)
assert not m["_CANCEL_REGISTRY"]