Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""In-flight chats: /load and /unload 409 unless opted in to cancel; GET /inference/active-
generations lists them; cancel_all spares llama-server"""
from __future__ import annotations
import threading
import time
import uuid
from typing import Any, Collection, Optional
from utils.account_context import current_account_id
# Keyed by handle: a tool continuation can register before the previous leg unregisters.
_ACTIVE: dict[str, dict[str, Any]] = {}
_LOCK = threading.Lock()
# Disabled or retired accounts: a generation registering late for one starts cancelled.
_FENCED: set[str] = set()
class ActiveGeneration:
"""uses a unique handle so overlapping registrations cannot clobber each other."""
__slots__ = (
"thread_id",
"run_id",
"cancel_event",
"model",
"kind",
"account_id",
"external",
"_handle",
"_borrowed",
)
def __init__(
self,
cancel_event: threading.Event,
*,
thread_id: Optional[str] = None,
run_id: Optional[str] = None,
model: Optional[str] = None,
kind: str = "chat",
account_id: Optional[str] = None,
external: bool = False,
):
self.account_id = account_id or current_account_id()
self.external = external
self.thread_id = thread_id or None
self.run_id = run_id or None
self.cancel_event = cancel_event
self.model = model or None
self.kind = kind
self._handle: Optional[str] = None
self._borrowed = False
def __enter__(self) -> "ActiveGeneration":
with _LOCK:
# borrow the supervisor's pre-load registration when the route reuses its event and run.
if self.run_id:
for entry in _ACTIVE.values():
if (
entry["run_id"] != self.run_id
or entry["event"] is not self.cancel_event
or entry["account_id"] != self.account_id
):
continue
if self.thread_id:
entry["thread_id"] = self.thread_id
if self.model:
entry["model"] = self.model
if self.kind:
entry["kind"] = self.kind
self._borrowed = True
return self
self._handle = uuid.uuid4().hex
_ACTIVE[self._handle] = {
"handle": self._handle,
"thread_id": self.thread_id,
"run_id": self.run_id,
"model": self.model,
"kind": self.kind,
"account_id": self.account_id,
"external": self.external,
"started_at": time.time(),
"event": self.cancel_event,
}
if self.account_id in _FENCED:
self.cancel_event.set()
return self
def __exit__(self, *exc) -> bool:
if self._borrowed:
self._borrowed = False
return False
handle, self._handle = self._handle, None
if handle is not None:
with _LOCK:
_ACTIVE.pop(handle, None)
return False
def _matching(
account_id: Optional[str],
exclude: Collection[threading.Event],
only: Optional[Collection[threading.Event]] = None,
*,
include_external: bool = True,
) -> list[dict[str, Any]]:
return [
e
for e in _ACTIVE.values()
if (account_id is None or e["account_id"] == account_id)
and e["event"] not in exclude
and (only is None or e["event"] in only)
and (include_external or not e["external"])
]
def snapshot(
account_id: Optional[str] = None,
exclude: Collection[threading.Event] = (),
only: Optional[Collection[threading.Event]] = None,
*,
include_external: bool = True,
) -> list[dict[str, Any]]:
"""in-flight generations, newest last; ``account_id=None`` is for shutdown or the arbiter."""
with _LOCK:
entries = _matching(account_id, exclude, only, include_external = include_external)
entries.sort(key = lambda e: e["started_at"])
return [
{
"handle": e["handle"],
"thread_id": e["thread_id"],
"run_id": e["run_id"],
"model": e["model"],
"kind": e["kind"],
"account_id": e["account_id"],
"started_at": e["started_at"],
}
for e in entries
]
def active_thread_ids(
account_id: Optional[str] = None,
exclude: Collection[threading.Event] = (),
only: Optional[Collection[threading.Event]] = None,
*,
include_external: bool = True,
) -> list[str]:
"""distinct conversation ids in start order; an unpersisted first turn cannot be named."""
seen: list[str] = []
for e in snapshot(account_id, exclude, only, include_external = include_external):
tid = e["thread_id"]
if tid and tid not in seen:
seen.append(tid)
return seen
def count(
account_id: Optional[str] = None,
exclude: Collection[threading.Event] = (),
*,
include_external: bool = True,
) -> int:
with _LOCK:
if account_id is None and not exclude and include_external:
return len(_ACTIVE)
return len(_matching(account_id, exclude, include_external = include_external))
def foreign_count(account_id: str) -> int:
"""generations for other accounts that load and unload must not interrupt."""
with _LOCK:
return sum(
1 for e in _ACTIVE.values() if e["account_id"] != account_id and not e["external"]
)
def cancel_all(
account_id: Optional[str] = None,
exclude: Collection[threading.Event] = (),
*,
include_external: bool = True,
) -> int:
"""request callers must pass ``account_id``; None cancels every account during shutdown."""
with _LOCK:
events = [
e["event"] for e in _matching(account_id, exclude, include_external = include_external)
]
for ev in events:
try:
ev.set()
except Exception:
pass
return len(events)
def cancel_thread(thread_id: str, account_id: Optional[str] = None) -> int:
"""thread ids are client-chosen, so cancellation is scoped by account."""
if not thread_id:
return 0
scope = account_id or current_account_id()
with _LOCK:
events = [
e["event"]
for e in _ACTIVE.values()
if e["thread_id"] == thread_id and e["account_id"] == scope
]
for ev in events:
try:
ev.set()
except Exception:
pass
return len(events)
def cancel_run(run_id: str, account_id: Optional[str] = None) -> int:
if not run_id:
return 0
scope = account_id or current_account_id()
with _LOCK:
events = [
e["event"]
for e in _ACTIVE.values()
if e["run_id"] == run_id and e["account_id"] == scope
]
for ev in events:
try:
ev.set()
except Exception:
pass
return len(events)
def fence(account_id: str) -> None:
"""Deactivation or retirement: work registering after ``cancel_all`` is cancelled on entry."""
with _LOCK:
_FENCED.add(account_id)
def lift_fence(account_id: str) -> None:
with _LOCK:
_FENCED.discard(account_id)
def fenced(account_id: str) -> bool:
with _LOCK:
return account_id in _FENCED
def reset_for_tests() -> None:
"""Drop every entry. Test-only; never called from request paths."""
with _LOCK:
_ACTIVE.clear()
_FENCED.clear()