Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

7.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""In-flight chats: /load and /unload 409 unless opted in to cancel; GET /inference/active-
generations lists them; cancel_all spares llama-server"""

from __future__ import annotations

import threading
import time
import uuid
from typing import Any, Collection, Optional

from utils.account_context import current_account_id

# Keyed by handle: a tool continuation can register before the previous leg unregisters.
_ACTIVE: dict[str, dict[str, Any]] = {}
_LOCK = threading.Lock()
# Disabled or retired accounts: a generation registering late for one starts cancelled.
_FENCED: set[str] = set()


class ActiveGeneration:
    """uses a unique handle so overlapping registrations cannot clobber each other."""

    __slots__ = (
        "thread_id",
        "run_id",
        "cancel_event",
        "model",
        "kind",
        "account_id",
        "external",
        "_handle",
        "_borrowed",
    )

    def __init__(
        self,
        cancel_event: threading.Event,
        *,
        thread_id: Optional[str] = None,
        run_id: Optional[str] = None,
        model: Optional[str] = None,
        kind: str = "chat",
        account_id: Optional[str] = None,
        external: bool = False,
    ):
        self.account_id = account_id or current_account_id()
        self.external = external
        self.thread_id = thread_id or None
        self.run_id = run_id or None
        self.cancel_event = cancel_event
        self.model = model or None
        self.kind = kind
        self._handle: Optional[str] = None
        self._borrowed = False

    def __enter__(self) -> "ActiveGeneration":
        with _LOCK:
            # borrow the supervisor's pre-load registration when the route reuses its event and run.
            if self.run_id:
                for entry in _ACTIVE.values():
                    if (
                        entry["run_id"] != self.run_id
                        or entry["event"] is not self.cancel_event
                        or entry["account_id"] != self.account_id
                    ):
                        continue
                    if self.thread_id:
                        entry["thread_id"] = self.thread_id
                    if self.model:
                        entry["model"] = self.model
                    if self.kind:
                        entry["kind"] = self.kind
                    self._borrowed = True
                    return self
            self._handle = uuid.uuid4().hex
            _ACTIVE[self._handle] = {
                "handle": self._handle,
                "thread_id": self.thread_id,
                "run_id": self.run_id,
                "model": self.model,
                "kind": self.kind,
                "account_id": self.account_id,
                "external": self.external,
                "started_at": time.time(),
                "event": self.cancel_event,
            }
            if self.account_id in _FENCED:
                self.cancel_event.set()
        return self

    def __exit__(self, *exc) -> bool:
        if self._borrowed:
            self._borrowed = False
            return False
        handle, self._handle = self._handle, None
        if handle is not None:
            with _LOCK:
                _ACTIVE.pop(handle, None)
        return False


def _matching(
    account_id: Optional[str],
    exclude: Collection[threading.Event],
    only: Optional[Collection[threading.Event]] = None,
    *,
    include_external: bool = True,
) -> list[dict[str, Any]]:
    return [
        e
        for e in _ACTIVE.values()
        if (account_id is None or e["account_id"] == account_id)
        and e["event"] not in exclude
        and (only is None or e["event"] in only)
        and (include_external or not e["external"])
    ]


def snapshot(
    account_id: Optional[str] = None,
    exclude: Collection[threading.Event] = (),
    only: Optional[Collection[threading.Event]] = None,
    *,
    include_external: bool = True,
) -> list[dict[str, Any]]:
    """in-flight generations, newest last; ``account_id=None`` is for shutdown or the arbiter."""
    with _LOCK:
        entries = _matching(account_id, exclude, only, include_external = include_external)
    entries.sort(key = lambda e: e["started_at"])
    return [
        {
            "handle": e["handle"],
            "thread_id": e["thread_id"],
            "run_id": e["run_id"],
            "model": e["model"],
            "kind": e["kind"],
            "account_id": e["account_id"],
            "started_at": e["started_at"],
        }
        for e in entries
    ]


def active_thread_ids(
    account_id: Optional[str] = None,
    exclude: Collection[threading.Event] = (),
    only: Optional[Collection[threading.Event]] = None,
    *,
    include_external: bool = True,
) -> list[str]:
    """distinct conversation ids in start order; an unpersisted first turn cannot be named."""
    seen: list[str] = []
    for e in snapshot(account_id, exclude, only, include_external = include_external):
        tid = e["thread_id"]
        if tid and tid not in seen:
            seen.append(tid)
    return seen


def count(
    account_id: Optional[str] = None,
    exclude: Collection[threading.Event] = (),
    *,
    include_external: bool = True,
) -> int:
    with _LOCK:
        if account_id is None and not exclude and include_external:
            return len(_ACTIVE)
        return len(_matching(account_id, exclude, include_external = include_external))


def foreign_count(account_id: str) -> int:
    """generations for other accounts that load and unload must not interrupt."""
    with _LOCK:
        return sum(
            1 for e in _ACTIVE.values() if e["account_id"] != account_id and not e["external"]
        )


def cancel_all(
    account_id: Optional[str] = None,
    exclude: Collection[threading.Event] = (),
    *,
    include_external: bool = True,
) -> int:
    """request callers must pass ``account_id``; None cancels every account during shutdown."""
    with _LOCK:
        events = [
            e["event"] for e in _matching(account_id, exclude, include_external = include_external)
        ]
    for ev in events:
        try:
            ev.set()
        except Exception:
            pass
    return len(events)


def cancel_thread(thread_id: str, account_id: Optional[str] = None) -> int:
    """thread ids are client-chosen, so cancellation is scoped by account."""
    if not thread_id:
        return 0
    scope = account_id or current_account_id()
    with _LOCK:
        events = [
            e["event"]
            for e in _ACTIVE.values()
            if e["thread_id"] == thread_id and e["account_id"] == scope
        ]
    for ev in events:
        try:
            ev.set()
        except Exception:
            pass
    return len(events)


def cancel_run(run_id: str, account_id: Optional[str] = None) -> int:
    if not run_id:
        return 0
    scope = account_id or current_account_id()
    with _LOCK:
        events = [
            e["event"]
            for e in _ACTIVE.values()
            if e["run_id"] == run_id and e["account_id"] == scope
        ]
    for ev in events:
        try:
            ev.set()
        except Exception:
            pass
    return len(events)


def fence(account_id: str) -> None:
    """Deactivation or retirement: work registering after ``cancel_all`` is cancelled on entry."""
    with _LOCK:
        _FENCED.add(account_id)


def lift_fence(account_id: str) -> None:
    with _LOCK:
        _FENCED.discard(account_id)


def fenced(account_id: str) -> bool:
    with _LOCK:
        return account_id in _FENCED


def reset_for_tests() -> None:
    """Drop every entry. Test-only; never called from request paths."""
    with _LOCK:
        _ACTIVE.clear()
        _FENCED.clear()