Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

43.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""In-memory API monitor with an optional content-free terminal receipt sink."""

from __future__ import annotations

import hashlib
import json
import logging
import math
import os
import sys
import threading
import time
import uuid
from collections import deque
from dataclasses import dataclass, field
from typing import Any, Callable, Optional

from utils.account_context import current_account_id
from storage.api_usage_db import (
    MAX_ENDPOINT_CHARS,
    MAX_STATUS_CHARS,
    ApiUsageReceipt,
    canonical_api_model,
    canonical_api_subject,
)


logger = logging.getLogger(__name__)
TerminalCallback = Callable[[ApiUsageReceipt], None]


_MAX_ENTRIES = 50
_MAX_PROMPT_BYTES = 64 * 1024 * 1024
_MAX_REPLY_CHARS = 12000
_PREVIEW_CHARS = 360
_MAX_STREAM_TOOL_CALLS = 64
_MAX_STREAM_TOOL_FIELD_CHARS = 501
_MAX_DECODE_MS = 24 * 60 * 60 * 1000
# Far above any real context window; larger means a broken upstream payload.
_MAX_TOKEN_COUNT = 1 << 40

# Opt-in startup kill switch for Unsloth's in-memory API monitor.
_DISABLE_ENV = "UNSLOTH_STUDIO_DISABLE_API_MONITOR"
_TRUE_VALUES = frozenset({"1", "true", "yes", "on"})


def _api_monitor_disabled() -> bool:
    return os.environ.get(_DISABLE_ENV, "").strip().lower() in _TRUE_VALUES


def _token_count_or_none(value: Any) -> Optional[int]:
    try:
        count = int(value)
    except (TypeError, ValueError, OverflowError):
        return None
    # snapshot() divides by these; an unbounded int makes that division raise too.
    if count < 0 or count > _MAX_TOKEN_COUNT:
        return None
    return count


def _finite_float_or_none(value: Any) -> Optional[float]:
    try:
        number = float(value)
    except (TypeError, ValueError, OverflowError):
        return None
    return number if math.isfinite(number) else None


def _advance_updated_at(entry: "ApiMonitorEntry", now: Optional[float] = None) -> None:
    """Advance the row's freshness key even on a coarse or regressing wall clock."""
    observed = time.time() if now is None else now
    entry.updated_at = max(observed, math.nextafter(entry.updated_at, math.inf))


def _trim(text: Optional[str], limit: int) -> str:
    if not text:
        return ""
    if len(text) <= limit:
        return text
    # Guard against limit < 3 (slice would underflow).
    if limit <= 3:
        return "..."[:limit]
    return text[: limit - 3] + "..."


@dataclass
class _OpenAIStreamToolCall:
    choice_index: int
    tool_index: int
    call_id: Optional[str] = None
    name: str = ""
    arguments: str = ""


def _stream_tool_fragment(fragment: Any, *, serialize_json: bool = False) -> Optional[str]:
    if fragment is None:
        return None
    if isinstance(fragment, str):
        return fragment
    return json.dumps(fragment, default = str) if serialize_json else str(fragment)


def _append_stream_tool_field(
    current: str,
    fragment: Any,
    budget: int,
    *,
    serialize_json: bool = False,
) -> str:
    if fragment is None or budget <= 0 or len(current) >= _MAX_STREAM_TOOL_FIELD_CHARS:
        return current
    fragment = _stream_tool_fragment(fragment, serialize_json = serialize_json)
    if fragment is None:
        return current
    remaining = min(_MAX_STREAM_TOOL_FIELD_CHARS - len(current), budget)
    return current + fragment[:remaining]


def _merge_stream_tool_name(current: str, fragment: Any, budget: int) -> str:
    fragment = _stream_tool_fragment(fragment)
    if fragment is None:
        return current
    if fragment.startswith(current):
        limit = min(_MAX_STREAM_TOOL_FIELD_CHARS, len(current) + max(0, budget))
        return fragment[:limit]
    return _append_stream_tool_field(current, fragment, budget)


def _stream_tool_call_is_complete(state: _OpenAIStreamToolCall) -> bool:
    if not state.name or not state.arguments:
        return False
    try:
        arguments = json.loads(state.arguments)
    except (TypeError, ValueError):
        return False
    return isinstance(arguments, (dict, list))


def _stream_tool_call_preview(state: _OpenAIStreamToolCall) -> str:
    name = state.name or "<unknown>"
    if len(name) > 500:
        name = name[:497] + "..."
    if not state.arguments:
        return f"Tool call: {name}"
    arguments = state.arguments
    if len(arguments) > 500:
        arguments = arguments[:497] + "..."
    return f"Tool call: {name}({arguments})"


def _append_stream_tool_previews(
    entry: ApiMonitorEntry, states: list[_OpenAIStreamToolCall]
) -> None:
    if not states:
        return
    text = "\n".join(_stream_tool_call_preview(state) for state in states)
    if entry.reply and not entry.reply.endswith("\n"):
        text = "\n" + text
    entry.reply = _trim(entry.reply + text, _MAX_REPLY_CHARS)


def _stream_tool_call_id(value: Any) -> Optional[str]:
    if not isinstance(value, str) or not value:
        return None
    if len(value) <= _MAX_STREAM_TOOL_FIELD_CHARS:
        return value
    return "sha256:" + hashlib.sha256(value.encode("utf-8", errors = "replace")).hexdigest()


@dataclass
class ApiMonitorEntry:
    id: str
    endpoint: str
    method: str
    model: str
    prompt: str
    status: str
    started_at: float
    updated_at: float
    # Who this row is attributed to; on a shared row it does not restrict visibility.
    subject: Optional[str] = None
    # Usernames are reusable and these in-memory rows outlive the account, so the immutable account id fences a replacement off its predecessor's traffic.
    account_id: str = field(default_factory = current_account_id)
    # True for sk-unsloth callers only: the panel auto-opens on these, not Unsloth's chat.
    via_api_key: bool = False
    # Monotonic anchors so duration math survives wall-clock steps (NTP).
    started_monotonic: float = 0.0
    finished_monotonic: Optional[float] = None
    reply: str = ""
    finished_at: Optional[float] = None
    context_length: Optional[int] = None
    prompt_tokens: Optional[int] = None
    completion_tokens: Optional[int] = None
    total_tokens: Optional[int] = None
    total_tokens_authoritative: bool = False
    error: Optional[str] = None
    # "request" (HTTP call) or "lifecycle" (model load/unload: event/reason, not a prompt; shared)
    kind: str = "request"
    event: Optional[str] = None
    reason: Optional[str] = None
    shared: bool = False
    # 0-100 for a running download row; None when not applicable.
    progress: Optional[float] = None
    running_phase: Optional[str] = None
    prompt_progress_total: Optional[int] = None
    prompt_progress_processed: Optional[int] = None
    prompt_progress_cached: Optional[int] = None
    prompt_progress_time_ms: Optional[float] = None
    # Stamped on the first reply text; snapshot() prefers it over engine timings.
    first_token_monotonic: Optional[float] = None
    # Tool cards count for TTFT but not the token-rate clock.
    first_decode_monotonic: Optional[float] = None
    prompt_ms: Optional[float] = None
    tok_per_sec: Optional[float] = None
    prompt_tok_per_sec: Optional[float] = None
    # timings.predicted_ms; set only from engine timings, so its presence marks a rateable row.
    decode_ms: Optional[float] = None
    stop_reason: Optional[str] = None
    # An n > 1 stream reports each choice in its own chunk. Not serialized.
    stop_reasons_seen: set[str] = field(default_factory = set)
    # request-local preview state, omitted from snapshots and cleared on terminal paths.
    openai_stream_tool_calls: list[_OpenAIStreamToolCall] = field(default_factory = list)
    openai_stream_last_tool_indexes: dict[int, int] = field(default_factory = dict)
    openai_stream_last_segment_was_tool: bool = False
    prompt_complete: bool = True

    def snapshot(
        self,
        *,
        include_details: bool = True,
        include_prompt: bool = True,
        attributed: bool = True,
    ) -> dict[str, Any]:
        duration_ms = None
        if self.finished_monotonic is not None:
            duration_ms = max(
                0,
                int((self.finished_monotonic - self.started_monotonic) * 1000),
            )
        elif self.finished_at is not None:
            duration_ms = max(0, int((self.finished_at - self.started_at) * 1000))
        context_usage = None
        if self.total_tokens is not None and self.context_length:
            context_usage = min(1.0, max(0.0, self.total_tokens / self.context_length))
        ttft_ms = None
        if self.first_token_monotonic is not None:
            # Preferred over prompt_ms, which misses the admission-queue wait.
            ttft_ms = max(0, int((self.first_token_monotonic - self.started_monotonic) * 1000))
        elif self.prompt_ms is not None:
            ttft_ms = max(0, int(self.prompt_ms))
        tok_per_sec = self.tok_per_sec
        if (
            tok_per_sec is None
            and self.completion_tokens
            # Clock starts at the first token, so one token has no gap to measure and yields no rate
            and self.completion_tokens > 1
            and self.finished_monotonic is not None
            and self.first_decode_monotonic is not None
        ):
            gen_s = self.finished_monotonic - self.first_decode_monotonic
            if gen_s > 0.05:
                tok_per_sec = (self.completion_tokens - 1) / gen_s
        prompt_progress = None
        if self.prompt_progress_processed is not None:
            percent = None
            if self.prompt_progress_total is not None and self.prompt_progress_total > 0:
                percent = min(
                    100.0,
                    max(
                        0.0,
                        self.prompt_progress_processed / self.prompt_progress_total * 100.0,
                    ),
                )
            prompt_progress = {
                "total": self.prompt_progress_total,
                "processed": self.prompt_progress_processed,
                "cached": self.prompt_progress_cached,
                "time_ms": (
                    round(self.prompt_progress_time_ms, 2)
                    if self.prompt_progress_time_ms is not None
                    else None
                ),
                "percent": round(percent, 1) if percent is not None else None,
            }
        payload = {
            "id": self.id,
            "endpoint": self.endpoint,
            "method": self.method,
            "model": self.model,
            # Shared rows reach unrelated subjects and auto-open the overlay, so only attributed callers see it
            "via_api_key": self.via_api_key and attributed,
            "prompt_preview": _trim(self.prompt, _PREVIEW_CHARS),
            "reply_preview": _trim(self.reply, _PREVIEW_CHARS),
            "prompt_truncated": not self.prompt_complete or len(self.prompt) > _PREVIEW_CHARS,
            "reply_truncated": len(self.reply) > _PREVIEW_CHARS,
            "status": self.status,
            "started_at": self.started_at,
            "updated_at": self.updated_at,
            "finished_at": self.finished_at,
            "duration_ms": duration_ms,
            "context_length": self.context_length,
            "context_usage": context_usage,
            "prompt_tokens": self.prompt_tokens,
            "completion_tokens": self.completion_tokens,
            "total_tokens": self.total_tokens,
            "error": self.error,
            "kind": self.kind,
            "event": self.event,
            "reason": self.reason,
            "progress": self.progress,
            "running_phase": self.running_phase,
            "prompt_progress": prompt_progress,
            "ttft_ms": ttft_ms,
            "tok_per_sec": round(tok_per_sec, 2) if tok_per_sec is not None else None,
            "prompt_tok_per_sec": (
                round(self.prompt_tok_per_sec, 2) if self.prompt_tok_per_sec is not None else None
            ),
            # Engine decode span, not the streamed window, which reasoning tokens inflate.
            "decode_ms": int(self.decode_ms) if self.decode_ms is not None else None,
            "stop_reason": self.stop_reason,
        }
        if include_details:
            if include_prompt and self.prompt_complete:
                payload["prompt"] = self.prompt
            payload["reply"] = self.reply
        return payload


class ApiMonitor:
    def __init__(
        self,
        max_entries: int = _MAX_ENTRIES,
        *,
        enabled: bool = True,
        terminal_callback: Optional[TerminalCallback] = None,
    ):
        self._entries: deque[ApiMonitorEntry] = deque()
        # Shared rows one subject cleared: deleting would erase another caller's history.
        self._hidden_shared: dict[tuple[str, str], set[str]] = {}
        self._max_entries = max(0, max_entries)
        self._lock = threading.Lock()
        self._callback_condition = threading.Condition(self._lock)
        self._enabled = enabled
        self._terminal_callback = terminal_callback
        self._terminal_callback_leases: dict[str, TerminalCallback] = {}
        self._terminal_callbacks_inflight: dict[Optional[str], int] = {}

    @property
    def enabled(self) -> bool:
        """Read-only; the kill switch is a startup env var, so live monitors cannot flip it."""
        return self._enabled

    def set_terminal_callback(self, callback: Optional[TerminalCallback]) -> None:
        """Replace the unleased terminal sink used by tests and embedders."""
        with self._lock:
            self._terminal_callback = callback

    def acquire_terminal_callback(self, callback: TerminalCallback) -> str:
        """Only the newest live lease is notified, so overlapping lifespans do not write duplicate receipts; releasing the newest falls back to the older"""
        lease = uuid.uuid4().hex
        with self._lock:
            self._terminal_callback_leases[lease] = callback
        return lease

    def release_terminal_callback(self, lease: str) -> None:
        """Remove only the terminal callback registration owned by ``lease``."""
        with self._callback_condition:
            self._terminal_callback_leases.pop(lease, None)
            # Wait out an in-flight notification's fast enqueue before the owner drains and stops the writer
            while self._terminal_callbacks_inflight.get(lease, 0):
                self._callback_condition.wait()

    def _terminal_callback_locked(self) -> tuple[Optional[str], Optional[TerminalCallback]]:
        if self._terminal_callback_leases:
            lease = next(reversed(self._terminal_callback_leases))
            return lease, self._terminal_callback_leases[lease]
        return None, self._terminal_callback

    def start(
        self,
        *,
        endpoint: str,
        method: str,
        model: str,
        prompt: str,
        context_length: Optional[int] = None,
        subject: Optional[str] = None,
        via_api_key: bool = False,
    ) -> str:
        if not self._enabled:
            return ""
        now = time.time()
        entry = ApiMonitorEntry(
            id = f"apireq_{uuid.uuid4().hex}",
            endpoint = endpoint,
            method = method,
            # str(): a raw JSON body can carry any type, and a non-string breaks the UI.
            model = str(model) if model else "default",
            prompt = prompt or "",
            status = "running",
            started_at = now,
            updated_at = now,
            subject = subject,
            via_api_key = via_api_key,
            started_monotonic = time.monotonic(),
            context_length = context_length,
        )
        with self._lock:
            self._entries.appendleft(entry)
            self._trim_terminal_locked()
            remaining = _MAX_PROMPT_BYTES
            for retained in self._entries:
                if not retained.prompt_complete or len(retained.prompt) <= _PREVIEW_CHARS:
                    continue
                prompt_bytes = sys.getsizeof(retained.prompt)
                if prompt_bytes <= remaining:
                    remaining -= prompt_bytes
                else:
                    retained.prompt = _trim(retained.prompt, _PREVIEW_CHARS)
                    retained.prompt_complete = False
        return entry.id

    def record_lifecycle(
        self,
        *,
        event: str,
        model: str,
        reason: Optional[str] = None,
        running: bool = False,
        via_api_key: bool = False,
        subject: Optional[str] = None,
    ) -> str:
        """Pass subject whenever via_api_key is set or attribution reaches nobody; it does not narrow visibility, since rows are shared with every subject"""
        if not self._enabled:
            return ""
        now = time.time()
        entry = ApiMonitorEntry(
            id = f"apievt_{uuid.uuid4().hex[:12]}",
            endpoint = f"model.{event}",
            method = "",
            model = model or "default",
            prompt = "",
            status = "running" if running else "completed",
            started_at = now,
            updated_at = now,
            started_monotonic = time.monotonic(),
            finished_at = None if running else now,
            finished_monotonic = None if running else time.monotonic(),
            kind = "lifecycle",
            event = event,
            reason = reason,
            shared = True,
            # Refused switches skip api_monitor.start, so this row is the sole trace; attribution opens overlay
            via_api_key = via_api_key,
            subject = subject,
        )
        with self._lock:
            self._entries.appendleft(entry)
            self._trim_terminal_locked()
        return entry.id

    def relabel(self, entry_id: Optional[str], model: str) -> None:
        """Rename an open lifecycle row once the load resolves its real id: up front
        the caller only has the load path, which may be an HF snapshot dir."""
        if not entry_id or not model:
            return
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is not None:
                entry.model = model
                _advance_updated_at(entry)

    def set_progress(self, entry_id: Optional[str], progress: Optional[float]) -> None:
        """Update an open download row's percentage (clamped to 0-100)."""
        if not entry_id or progress is None:
            return
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is not None and entry.status == "running":
                entry.progress = min(100.0, max(0.0, float(progress)))
                _advance_updated_at(entry)

    def set_prompt_progress(
        self,
        entry_id: Optional[str],
        *,
        total: Any = None,
        processed: Any = None,
        cached: Any = None,
        time_ms: Any = None,
    ) -> None:
        if not entry_id:
            return
        total = _token_count_or_none(total)
        processed = _token_count_or_none(processed)
        cached = _token_count_or_none(cached)
        time_ms = _finite_float_or_none(time_ms)
        if processed is None:
            return
        if total is not None and total > 0:
            processed = min(processed, total)
        if cached is not None:
            cached = min(cached, processed)
        if time_ms is not None and time_ms < 0:
            time_ms = None
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None or entry.status != "running" or entry.kind != "request":
                return
            if entry.running_phase != "prompt_processing":
                entry.prompt_progress_total = None
                entry.prompt_progress_processed = None
                entry.prompt_progress_cached = None
                entry.prompt_progress_time_ms = None
            entry.running_phase = "prompt_processing"
            if total is not None:
                entry.prompt_progress_total = total
            entry.prompt_progress_processed = processed
            if cached is not None:
                entry.prompt_progress_cached = cached
            if time_ms is not None:
                entry.prompt_progress_time_ms = time_ms
            _advance_updated_at(entry)

    def set_running_phase(self, entry_id: Optional[str], phase: Optional[str]) -> None:
        if not entry_id or phase not in ("prompt_processing", "token_generation"):
            return
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None or entry.status != "running" or entry.kind != "request":
                return
            entry.running_phase = phase
            _advance_updated_at(entry)

    def discard(self, entry_id: Optional[str]) -> None:
        """Drop a row that turned out not to be an event (an already-satisfied load)."""
        if not entry_id:
            return
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is not None:
                self._entries.remove(entry)

    def append_reply(
        self,
        entry_id: Optional[str],
        text: str,
        *,
        stamp_first_token: bool = True,
        separate_from_openai_tool: bool = False,
    ) -> None:
        if not entry_id or not text:
            return
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None or entry.finished_at is not None:
                return
            if separate_from_openai_tool and entry.openai_stream_last_segment_was_tool:
                if entry.reply and not entry.reply.endswith("\n"):
                    text = "\n" + text
                entry.openai_stream_last_segment_was_tool = False
            # Only a streaming delta stamps TTFT; a full-response append is end-to-end latency.
            if stamp_first_token:
                entry.running_phase = "token_generation"
                now = time.monotonic()
                if entry.first_token_monotonic is None:
                    entry.first_token_monotonic = now
                if entry.first_decode_monotonic is None:
                    entry.first_decode_monotonic = now
            # Once the ... marker is present the head is frozen and the re-concat skipped; a reply exactly at the cap still needs one append to add the marker
            if len(entry.reply) >= _MAX_REPLY_CHARS:
                if not entry.reply.endswith("..."):
                    entry.reply = _trim(entry.reply + text, _MAX_REPLY_CHARS)
                _advance_updated_at(entry)
                return
            entry.reply = _trim(entry.reply + text, _MAX_REPLY_CHARS)
            _advance_updated_at(entry)

    def accumulate_openai_tool_call(
        self,
        entry_id: Optional[str],
        *,
        choice_index: int,
        tool_index: Optional[int],
        tool_index_explicit: bool = True,
        call_id: Optional[str],
        name_fragment: Any,
        arguments_fragment: Any,
    ) -> None:
        """Merge one streamed OpenAI tool-call delta into its monitor row."""
        if not entry_id:
            return
        call_id = _stream_tool_call_id(call_id)
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None or entry.finished_at is not None:
                return
            same_choice = [
                state
                for state in entry.openai_stream_tool_calls
                if state.choice_index == choice_index
            ]
            if tool_index is None:
                tool_index = entry.openai_stream_last_tool_indexes.get(choice_index, 0)
            state = None
            same_index = [
                candidate for candidate in same_choice if candidate.tool_index == tool_index
            ]
            if tool_index_explicit:
                state = next(
                    (
                        candidate
                        for candidate in reversed(same_index)
                        if candidate.call_id == call_id
                    ),
                    None,
                )
                if state is None and call_id is not None:
                    state = next(
                        (
                            candidate
                            for candidate in reversed(same_index)
                            if candidate.call_id is None
                        ),
                        None,
                    )
            elif call_id is not None:
                state = next(
                    (
                        candidate
                        for candidate in reversed(same_choice)
                        if candidate.call_id == call_id
                    ),
                    None,
                )
            if state is None and call_id is None:
                state = same_index[-1] if same_index else None
            if state is None:
                pending_chars = sum(
                    len(candidate.name) + len(candidate.arguments)
                    for candidate in entry.openai_stream_tool_calls
                )
                if (
                    len(entry.openai_stream_tool_calls) >= _MAX_STREAM_TOOL_CALLS
                    or pending_chars >= _MAX_REPLY_CHARS
                ):
                    return
                state = _OpenAIStreamToolCall(
                    choice_index = choice_index,
                    tool_index = tool_index,
                    call_id = call_id,
                )
                entry.openai_stream_tool_calls.append(state)
            elif call_id is not None and state.call_id is None:
                state.call_id = call_id
            entry.openai_stream_last_tool_indexes[choice_index] = tool_index
            pending_chars = sum(
                len(candidate.name) + len(candidate.arguments)
                for candidate in entry.openai_stream_tool_calls
            )
            budget = _MAX_REPLY_CHARS - pending_chars
            previous_name_length = len(state.name)
            state.name = _merge_stream_tool_name(state.name, name_fragment, budget)
            budget -= len(state.name) - previous_name_length
            state.arguments = _append_stream_tool_field(
                state.arguments,
                arguments_fragment,
                budget,
                serialize_json = True,
            )
            entry.running_phase = "token_generation"
            now = time.monotonic()
            if entry.first_token_monotonic is None:
                entry.first_token_monotonic = now
            if entry.first_decode_monotonic is None:
                entry.first_decode_monotonic = now
            _advance_updated_at(entry)

    def take_openai_tool_calls(
        self,
        entry_id: Optional[str],
        *,
        choice_index: Optional[int] = None,
        completed_only: bool = False,
    ) -> tuple[list[tuple[str, str]], bool]:
        """Remove and return completed preview calls in first-seen order."""
        if not entry_id:
            return [], False
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None or entry.finished_at is not None:
                return [], False
            selected = [
                state
                for state in entry.openai_stream_tool_calls
                if (choice_index is None or state.choice_index == choice_index)
                and (not completed_only or _stream_tool_call_is_complete(state))
            ]
            if not selected:
                return [], False
            separate = bool(entry.reply) and not entry.reply.endswith("\n")
            entry.openai_stream_last_segment_was_tool = True
            selected_ids = {id(state) for state in selected}
            entry.openai_stream_tool_calls = [
                state for state in entry.openai_stream_tool_calls if id(state) not in selected_ids
            ]
            remaining_indexes = {
                state.choice_index: state.tool_index for state in entry.openai_stream_tool_calls
            }
            if choice_index is None:
                entry.openai_stream_last_tool_indexes = remaining_indexes
            elif choice_index not in remaining_indexes:
                entry.openai_stream_last_tool_indexes.pop(choice_index, None)
            else:
                entry.openai_stream_last_tool_indexes[choice_index] = remaining_indexes[
                    choice_index
                ]
            _advance_updated_at(entry)
            return [(state.name, state.arguments) for state in selected], separate

    def mark_first_token(
        self,
        entry_id: Optional[str],
        *,
        decoded: bool = True,
    ) -> None:
        """``decoded=False`` is for output the model did not generate: starts TTFT, not the token-
        rate clock."""
        if not entry_id:
            return
        now = time.monotonic()
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None:
                return
            if entry.first_token_monotonic is None:
                entry.first_token_monotonic = now
            if decoded and entry.first_decode_monotonic is None:
                entry.first_decode_monotonic = now
            entry.running_phase = "token_generation" if decoded else None

    def set_reply(self, entry_id: Optional[str], text: str) -> None:
        if not entry_id:
            return
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None:
                return
            entry.reply = _trim(text, _MAX_REPLY_CHARS)
            _advance_updated_at(entry)

    def set_perf(
        self,
        entry_id: Optional[str],
        *,
        tok_per_sec: Optional[float] = None,
        prompt_tok_per_sec: Optional[float] = None,
        prompt_ms: Optional[float] = None,
        decode_ms: Optional[float] = None,
        stop_reason: Optional[str] = None,
    ) -> None:
        if not entry_id:
            return
        # Coerce before locking: a raise here would truncate the streamed response.
        tok_per_sec = _finite_float_or_none(tok_per_sec)
        prompt_tok_per_sec = _finite_float_or_none(prompt_tok_per_sec)
        prompt_ms = _finite_float_or_none(prompt_ms)
        decode_ms = _finite_float_or_none(decode_ms)
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None:
                return
            if tok_per_sec is not None and tok_per_sec >= 0:
                entry.tok_per_sec = tok_per_sec
            if prompt_tok_per_sec is not None and prompt_tok_per_sec >= 0:
                entry.prompt_tok_per_sec = prompt_tok_per_sec
            if prompt_ms is not None:
                entry.prompt_ms = prompt_ms
            # Past a day of decode it is a bad reading, not a slow model.
            if decode_ms is not None and 0 <= decode_ms <= _MAX_DECODE_MS:
                entry.decode_ms = decode_ms
            if stop_reason is not None:
                entry.stop_reason = str(stop_reason)
            _advance_updated_at(entry)

    def note_stop_reason(self, entry_id: Optional[str], reason: Optional[str]) -> None:
        """Not published here: with n > 1 a later choice can disagree, so finish() resolves it."""
        if not entry_id or not reason:
            return
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None:
                return
            entry.stop_reasons_seen.add(str(reason))
            _advance_updated_at(entry)

    @staticmethod
    def _settle_stop_reason_locked(entry: ApiMonitorEntry, completed: bool) -> None:
        """Only a completed request keeps its stop reason; clearing sits here so direct set_perf writers, reached before cancellation is stamped, are caught"""
        if not completed:
            entry.stop_reason = None
            return
        seen = entry.stop_reasons_seen
        if seen:
            entry.stop_reason = next(iter(seen)) if len(seen) == 1 else None

    def set_usage(
        self,
        entry_id: Optional[str],
        *,
        prompt_tokens: Optional[int] = None,
        completion_tokens: Optional[int] = None,
        total_tokens: Optional[int] = None,
        context_length: Optional[int] = None,
    ) -> None:
        if not entry_id:
            return
        # Coerce first: snapshot() does math on these arbitrary payload values.
        prompt_tokens = _token_count_or_none(prompt_tokens)
        completion_tokens = _token_count_or_none(completion_tokens)
        total_tokens = _token_count_or_none(total_tokens)
        context_length = _token_count_or_none(context_length)
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None or entry.finished_at is not None:
                return
            if prompt_tokens is not None:
                entry.prompt_tokens = prompt_tokens
            if completion_tokens is not None:
                entry.completion_tokens = completion_tokens
            if total_tokens is not None:
                entry.total_tokens = total_tokens
                entry.total_tokens_authoritative = True
            elif not entry.total_tokens_authoritative and (
                prompt_tokens is not None or completion_tokens is not None
            ):
                # Derive total only when none is set; a later partial chunk must not overwrite a provider total
                entry.total_tokens = (entry.prompt_tokens or 0) + (entry.completion_tokens or 0)
            if context_length is not None:
                entry.context_length = context_length
            _advance_updated_at(entry)

    def finish(
        self,
        entry_id: Optional[str],
        status: str = "completed",
    ) -> None:
        if not entry_id:
            return
        notification = None
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None:
                return
            # Idempotent: second call (e.g. [DONE] after the finally block already ran) must not move finished_*.
            if entry.finished_at is not None:
                return
            _append_stream_tool_previews(entry, entry.openai_stream_tool_calls)
            entry.openai_stream_tool_calls.clear()
            entry.openai_stream_last_tool_indexes.clear()
            entry.openai_stream_last_segment_was_tool = False
            self._settle_stop_reason_locked(entry, status == "completed")
            now = time.time()
            entry.status = status
            _advance_updated_at(entry, now)
            entry.finished_at = now
            entry.finished_monotonic = time.monotonic()
            self._entries.remove(entry)
            self._entries.appendleft(entry)
            self._trim_terminal_locked()
            notification = self._terminal_notification_locked(entry)
        self._notify_terminal(notification)

    def fail_open(self, entry_id: Optional[str], error: str) -> None:
        """Fail only a still-open row: unlike :meth:`fail`, a catch-all in a
        ``finally`` cannot stamp an error onto a request that already succeeded."""
        if not entry_id:
            return
        notification = None
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None or entry.finished_at is not None:
                return
            self._fail_locked(entry, error)
            notification = self._terminal_notification_locked(entry)
        self._notify_terminal(notification)

    def fail(self, entry_id: Optional[str], error: str) -> None:
        if not entry_id:
            return
        notification = None
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None:
                return
            if entry.finished_at is not None:
                # Already terminal; refresh error text only.
                if error:
                    entry.error = _trim(error, 1000)
                    _advance_updated_at(entry)
                return
            self._fail_locked(entry, error)
            notification = self._terminal_notification_locked(entry)
        self._notify_terminal(notification)

    def _fail_locked(self, entry: ApiMonitorEntry, error: str) -> None:
        _append_stream_tool_previews(entry, entry.openai_stream_tool_calls)
        entry.openai_stream_tool_calls.clear()
        entry.openai_stream_last_tool_indexes.clear()
        entry.openai_stream_last_segment_was_tool = False
        self._settle_stop_reason_locked(entry, False)
        now = time.time()
        entry.status = "error"
        entry.error = _trim(error, 1000)
        _advance_updated_at(entry, now)
        entry.finished_at = now
        entry.finished_monotonic = time.monotonic()
        self._entries.remove(entry)
        self._entries.appendleft(entry)
        self._trim_terminal_locked()

    def _terminal_notification_locked(
        self, entry: ApiMonitorEntry
    ) -> Optional[tuple[Optional[str], TerminalCallback, ApiUsageReceipt]]:
        callback_owner, callback = self._terminal_callback_locked()
        subject = canonical_api_subject(entry.subject)
        if (
            callback is None
            or entry.kind != "request"
            or entry.via_api_key is not True
            or not subject
            or entry.finished_at is None
        ):
            return None
        self._terminal_callbacks_inflight[callback_owner] = (
            self._terminal_callbacks_inflight.get(callback_owner, 0) + 1
        )
        return (
            callback_owner,
            callback,
            ApiUsageReceipt(
                id = entry.id,
                subject = subject,
                endpoint = str(entry.endpoint)[:MAX_ENDPOINT_CHARS],
                model = canonical_api_model(entry.model),
                status = str(entry.status)[:MAX_STATUS_CHARS],
                prompt_tokens = entry.prompt_tokens or 0,
                completion_tokens = entry.completion_tokens or 0,
                total_tokens = entry.total_tokens or 0,
                created_at = int(entry.finished_at * 1000),
                kind = entry.kind,
                via_api_key = entry.via_api_key,
            ),
        )

    def _notify_terminal(
        self, notification: Optional[tuple[Optional[str], TerminalCallback, ApiUsageReceipt]]
    ) -> None:
        if notification is None:
            return
        callback_owner, callback, receipt = notification
        try:
            callback(receipt)
        except Exception:  # noqa: BLE001 - monitoring must never break inference.
            logger.warning("api_monitor.terminal_callback_failed", exc_info = True)
        finally:
            with self._callback_condition:
                remaining = self._terminal_callbacks_inflight.get(callback_owner, 0) - 1
                if remaining > 0:
                    self._terminal_callbacks_inflight[callback_owner] = remaining
                else:
                    self._terminal_callbacks_inflight.pop(callback_owner, None)
                self._callback_condition.notify_all()

    def snapshot(
        self,
        *,
        include_details: bool = True,
        subject: Optional[str] = None,
    ) -> list[dict[str, Any]]:
        with self._lock:
            return [
                entry.snapshot(
                    include_details = include_details,
                    attributed = self._attributed(entry, subject),
                )
                for entry in self._entries
                if self._visible(entry, subject)
            ]

    def get(
        self,
        entry_id: str,
        *,
        subject: Optional[str] = None,
        include_prompt: bool = True,
    ) -> Optional[dict[str, Any]]:
        with self._lock:
            entry = self._find_locked(entry_id)
            if entry is None:
                return None
            if not self._visible(entry, subject):
                return None
            return entry.snapshot(
                include_details = True,
                include_prompt = include_prompt,
                attributed = self._attributed(entry, subject),
            )

    def active_count(self, *, subject: Optional[str] = None) -> int:
        with self._lock:
            return sum(
                1
                for entry in self._entries
                if entry.status == "running"
                and entry.kind != "lifecycle"
                and self._attributed(entry, subject)
            )

    def clear(self, *, subject: Optional[str] = None) -> None:
        """Scoped to ``subject``: an unscoped clear would let one user erase every other user's history."""
        with self._lock:
            if subject is None:
                self._entries.clear()
                self._hidden_shared.clear()
                return
            # A running shared row is a load in progress, not history, so it stays.
            hidden = self._hidden_shared.setdefault((current_account_id(), subject), set())
            for entry in self._entries:
                if entry.shared and entry.status != "running":
                    hidden.add(entry.id)
            # Shared rows are hidden, never dropped; dropping an own running row loses the request.
            self._entries = deque(
                entry
                for entry in self._entries
                if entry.shared or not self._attributed(entry, subject) or entry.status == "running"
            )

    def _visible(self, entry: ApiMonitorEntry, subject: Optional[str]) -> bool:
        if subject is None:
            return True
        if entry.shared:
            # Every subject minus the cleared ones. Before ownership, so a clear hides own rows.
            if entry.id in self._hidden_shared.get((current_account_id(), subject), ()):
                return False
            return _lifecycle_row_visible_to_caller(entry, subject)
        return entry.subject == subject and entry.account_id == current_account_id()

    def _attributed(self, entry: ApiMonitorEntry, subject: Optional[str]) -> bool:
        """Unscoped reads (no subject) see the row's own flag; the overlay is for its owner only."""
        if subject is None:
            return True
        return entry.subject == subject and entry.account_id == current_account_id()

    def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]:
        for entry in self._entries:
            if entry.id == entry_id:
                return entry
        return None

    def _trim_terminal_locked(self) -> None:
        terminal_seen = 0
        kept: deque[ApiMonitorEntry] = deque()
        for entry in self._entries:
            if entry.status == "running":
                kept.append(entry)
                continue
            if terminal_seen < self._max_entries:
                kept.append(entry)
                terminal_seen += 1
        self._entries = kept
        # Keep hidden sets to live rows so they stay bounded by the ring buffer.
        live = {entry.id for entry in kept}
        for key, hidden in list(self._hidden_shared.items()):
            hidden &= live
            if not hidden:
                del self._hidden_shared[key]


api_monitor = ApiMonitor(enabled = not _api_monitor_disabled())


def _lifecycle_row_visible_to_caller(entry: "ApiMonitorEntry", subject: str) -> bool:
    """Lifecycle rows name a model path that may sit inside the loading account's workspace, so only the owner and that account see them."""
    if entry.kind != "lifecycle":
        return True
    if entry.account_id == current_account_id() and entry.subject in (None, subject):
        return True
    from utils.account_context import is_owner_context

    if is_owner_context():
        return True
    return False