Skip to content
Python
19.3 KB
Raw Blame
"""Tagged provider views: every model call recorded as a model step (observability E2, E22).

Every model call leaves the app through one provider router, shared by every session's
engine and by the OpenAI-compatible proxy. A wrapper around the router alone cannot know
which session, turn or purpose a call belongs to, so each caller holds its own **tagged
view** (technical design §3.3, TD-2):

    view = provider.tagged(purpose="summary", parent=step_id)
    turn = view.complete(model=..., messages=...)   # recorded as a model step

- `RecordingProvider` wraps the router once, where it is made. A call made through it with
  no tags is recorded as `unattributed`, so a missing tag shows instead of counting as main.
- `tagged()` on a view returns a new view: fields passed replace the old, the rest are kept.
- A view records through a `sink`: the engine's turn recording (`TurnRecording`), which knows
  the turn, or for calls outside any engine (the proxy) a sink on the server's recorder.

Each model step carries the call's purpose, provider, model, tool count, time to the first
streamed piece, finish reason and token counts (§5.2), and, from the HTTP capture
(`capture.py`), the exact exchange, `provider_meta` and `provider_facts` (§5.2.1). Recording
never breaks a call: any recording error is logged and the call goes on.
"""

from __future__ import annotations

import json
import logging
import threading
import time
from dataclasses import dataclass, replace
from types import SimpleNamespace
from typing import Any, Callable, Iterator, Optional

from ..providers.base import ProviderClient
from . import capture
from .engine_hooks import CURRENT, TurnRecording
from .recorder import short_hash

log = logging.getLogger(__name__)


@dataclass(frozen=True)
class Tags:
    purpose: str = "unattributed"
    parent: Optional[str] = None  # the step this call is nested in
    call_id: Optional[str] = None  # for auto-approver checks: the tool call checked
    shadow: bool = False
    turn: Any = CURRENT  # CURRENT = the sink's current turn; a turn id; None = no turn
    sink: Optional[TurnRecording] = None


class RecordingProvider(ProviderClient):
    """The router, wrapped once. Untagged calls are recorded as `unattributed` when a recorder
    is given; everything else (`invalidate`, `_secrets`…) passes through to the router."""

    def __init__(self, inner: ProviderClient, recorder: Any = None) -> None:
        self.inner = inner
        self.recorder = recorder
        self._sink = TurnRecording(recorder, lambda: "") if recorder is not None else None

    def tagged(self, **tags: Any) -> "ProviderView":
        if "sink" not in tags:
            tags["sink"] = self._sink
        return ProviderView(self, Tags(**tags))

    def complete(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):
        return self.tagged().complete(model=model, messages=messages, tools=tools, **settings)

    def stream(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):
        return self.tagged().stream(model=model, messages=messages, tools=tools, **settings)

    def capabilities(self, model: str):
        return self.inner.capabilities(model)

    def __getattr__(self, name: str) -> Any:
        return getattr(self.inner, name)


class ProviderView(ProviderClient):
    """A tagged handle on the shared provider. Tags are explicit; nothing relies on context
    variables crossing threads."""

    def __init__(self, base: RecordingProvider, tags: Tags) -> None:
        self.base, self.tags = base, tags

    def tagged(self, **overrides: Any) -> "ProviderView":
        return ProviderView(self.base, replace(self.tags, **overrides))

    def capabilities(self, model: str):
        return self.base.inner.capabilities(model)

    def __getattr__(self, name: str) -> Any:
        return getattr(self.base.inner, name)

    # -- calls --------------------------------------------------------------------------------
    def record_call(self, fn: Callable[[], Any], *, model: str) -> Any:
        """Record a model call made outside the provider router as a model step: a
        decision-model reviewer's request to its own service (E23, TD-16). `fn` makes the
        request on this thread; its HTTP exchange is captured, and the usage is read from
        the service's reply with the Responses-style mapping (input includes cached)."""
        call = _ModelCall.begin(self.tags, model=model, tools=None, streamed=False)
        if call is None:
            return fn()
        try:
            with call.http.active():
                result = fn()
        except BaseException as exc:
            call.failed(exc)
            raise
        call.finished(SimpleNamespace(finish_reason=None, usage=_reply_usage(call.http.entries()), served_by=None))
        return result

    def complete(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):
        call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=False, settings=settings)
        if call is None:
            return self.base.inner.complete(model=model, messages=messages, tools=tools, **settings)
        try:
            with call.http.active():
                turn = self.base.inner.complete(model=model, messages=messages, tools=tools, **settings)
        except BaseException as exc:
            call.failed(exc)
            raise
        call.finished(turn)
        return turn

    def stream(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any) -> Iterator[Any]:
        call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=True, settings=settings)
        if call is None:
            yield from self.base.inner.stream(model=model, messages=messages, tools=tools, **settings)
            return
        it = iter(())
        turn = None
        try:
            with call.http.active():
                it = iter(self.base.inner.stream(model=model, messages=messages, tools=tools, **settings))
            while True:
                # Capture is on only while the next piece is read, so a stream left half-read
                # (Stop pressed) never keeps capturing on this thread. Such a step stays open
                # and is closed by the turn's end as `interrupted` (§7.4).
                with call.http.active():
                    try:
                        chunk = next(it)
                    except StopIteration:
                        break
                call.saw(chunk)
                if getattr(chunk, "turn", None) is not None and turn is None:
                    # The step ends as the reply arrives, before the engine sees it: the
                    # engine may add the reply and end the turn before this generator runs
                    # again, and the turn's end must find this step already closed.
                    turn = chunk.turn
                    call.finished(turn)
                yield chunk
        except GeneratorExit:
            raise
        except BaseException as exc:
            if turn is None:
                call.failed(exc)
            raise


# -- one recorded call ----------------------------------------------------------------------------

_tools_hash_cache: dict[int, tuple[Any, Optional[str]]] = {}
_tools_hash_lock = threading.Lock()


def tools_hash(tools: Any) -> Optional[str]:
    """A short code of the tool definitions, to spot tool changes; cached per tool list (P-6)."""
    if not tools:
        return None
    key = id(tools)
    with _tools_hash_lock:
        hit = _tools_hash_cache.get(key)
        if hit is not None and hit[0] is tools:
            return hit[1]
    code = short_hash(json.dumps(tools, sort_keys=True, default=str))
    with _tools_hash_lock:
        if len(_tools_hash_cache) > 64:
            _tools_hash_cache.clear()
        _tools_hash_cache[key] = (tools, code)
    return code


def _provider_of(model: str) -> str:
    return (model or "").split(":", 1)[0] if ":" in (model or "") else "openai"


def _provider_title(provider: str) -> Optional[str]:
    try:
        from ..providers.registry import get_descriptor

        d = get_descriptor(provider)
        return getattr(d, "title", None) if d is not None else None
    except Exception:  # noqa: BLE001
        return None


def _context_window(model: str, *, catalogue_only: bool = False) -> Optional[int]:
    """The window the engine compacts against (`TurnEngine._compaction_config`): the
    catalogue, else the user's saved size, else an Ollama model's `num_ctx`. A local
    server's window is left out: finding it may mean a request to that server. With
    `catalogue_only` (a call's start) nothing is read from disk, so the request is not held."""
    try:
        from ..providers.matrix import model_context_windows

        window = model_context_windows().get(model)
        if catalogue_only:
            return int(window) if window else None
        if not window:
            from .. import model_config

            window = model_config.context_size_for(model)
        if not window:
            from ..providers.ollama_context import context_window_for

            window = context_window_for(model)
        return int(window) if window else None
    except Exception:  # noqa: BLE001
        return None


def root_cause(exc: BaseException) -> BaseException:
    seen = set()
    while id(exc) not in seen and (exc.__cause__ is not None or exc.__context__ is not None):
        seen.add(id(exc))
        exc = exc.__cause__ or exc.__context__  # type: ignore[assignment]
    return exc


def classify_error(exc: BaseException) -> str:
    """`error.type` from the failure's root cause (§5.2, recording-format §5)."""
    try:
        from ..compaction import is_context_overflow

        if is_context_overflow(exc):
            return "context_length"
    except Exception:  # noqa: BLE001
        pass
    root = root_cause(exc)
    text = f"{type(exc).__name__} {exc} {type(root).__name__} {root}".lower()
    status = getattr(exc, "status_code", None) or getattr(root, "status_code", None)
    if status in (529, 503) or "overload" in text:
        return "overloaded"
    if status == 429 or "rate limit" in text or "ratelimit" in text:
        return "rate_limited"
    if status in (401, 403) or "api key" in text or "authentication" in text:
        return "auth"
    if "timeout" in text or "timed out" in text:
        return "timeout"
    if "cancel" in text:
        return "cancelled"
    if any(k in text for k in ("connect", "dns", "ssl", "tls", "proxy", "name resolution")):
        return "network"
    return "provider_error"


def usage_attrs(usage: Any) -> dict[str, Any]:
    """The step's `usage` from the provider's `TokenUsage` (§5.2, §6.2). A count the provider
    did not send is `null`, never 0; a call with no counts is `{"reported": false}`."""
    if usage is None:
        return {"reported": False}
    return {
        "reported": True,
        "input_fresh": usage.input,
        "cache_read": usage.cache_read if getattr(usage, "cache_read_reported", True) else None,
        "cache_write": usage.cache_write if getattr(usage, "cache_write_reported", True) else None,
        "output": usage.output,
        "reasoning": getattr(usage, "reasoning", None),
    }


def _max_output(settings: Optional[dict[str, Any]]) -> Optional[int]:
    """The output limit the request asks for (format 1.2): the "cut off" issue says it."""
    for key in ("max_tokens", "max_output_tokens", "max_completion_tokens"):
        value = (settings or {}).get(key)
        if isinstance(value, int) and not isinstance(value, bool) and value > 0:
            return value
    return None


class _ModelCall:
    """The state of one recorded model call, from start to end."""

    def __init__(self, tags: Tags, sink: TurnRecording, step_id: str, streamed: bool) -> None:
        self.tags, self.sink, self.step_id, self.streamed = tags, sink, step_id, streamed
        self.http = capture.HttpCapture()
        self.t0 = time.perf_counter()
        self.ttft_ms: Optional[int] = None

    @classmethod
    def begin(cls, tags: Tags, *, model: str, tools: Any, streamed: bool,
              settings: Optional[dict[str, Any]] = None) -> Optional["_ModelCall"]:
        sink = tags.sink
        if sink is None or not sink.enabled:
            return None
        try:
            provider = _provider_of(model)
            step_id = sink.start_step(
                kind="model", name="model", turn_id=tags.turn, shadow=tags.shadow,
                parent_step_id=tags.parent,
                call_id=tags.call_id if tags.purpose == "auto_approver" else None,
                attrs={
                    "purpose": tags.purpose, "provider": provider,
                    "provider_title": _provider_title(provider), "model": model,
                    "stream": streamed, "context_window": _context_window(model, catalogue_only=True),
                    "tools_offered": len(tools or []), "tools_hash": tools_hash(tools),
                    "max_output": _max_output(settings),
                },
            )
        except Exception:  # noqa: BLE001 - recording never breaks a call
            log.exception("recording: could not start a model step")
            return None
        if step_id is None:
            return None
        call = cls(tags, sink, step_id, streamed)
        call.model = model
        return call

    def _window(self, attrs: dict[str, Any]) -> None:
        """A window the catalogue lacks (the user's saved size, Ollama's num_ctx), looked up
        once the call is over so recording never delays the request."""
        model = getattr(self, "model", "")
        if model and not _context_window(model, catalogue_only=True):
            window = _context_window(model)
            if window:
                attrs["context_window"] = window

    def saw(self, chunk: Any) -> None:
        # The first streamed piece of the reply: text or thinking (§7.2).
        if self.ttft_ms is None and (getattr(chunk, "text_delta", None) or getattr(chunk, "reasoning_delta", None)):
            self.ttft_ms = int((time.perf_counter() - self.t0) * 1000)

    def _exchange(self) -> tuple[dict[str, Any], dict[str, Any]]:
        """(attrs, content) from the HTTP capture: provider_meta, provider_facts, the exchange."""
        attrs: dict[str, Any] = {}
        content: dict[str, Any] = {}
        entries = [e for e in self.http.entries() if e.get("method") == "POST"]
        if not entries:
            return attrs, content
        recorder = self.sink.recorder
        content["exchange"] = [capture.store_exchange(e, recorder.put_piece) for e in entries]
        meta = [capture.provider_meta(e) for e in entries]
        attrs["provider_meta"] = meta
        reply = [m for m in meta if not m.get("auxiliary")] or meta
        facts = capture.provider_facts(reply[-1])
        attrs["provider_facts"] = facts
        if facts.get("served_by") is not None:
            attrs["served_by"] = facts["served_by"]
        return attrs, content

    def finished(self, turn: Any) -> None:
        try:
            attrs: dict[str, Any] = {
                "finish_reason": getattr(turn, "finish_reason", None),
                "ttft_ms": self.ttft_ms if self.streamed else None,
                "usage": usage_attrs(getattr(turn, "usage", None)),
            }
            if getattr(turn, "served_by", None):
                attrs["served_by"] = turn.served_by
            more, content = self._exchange()
            attrs.update(more)
            _fill_from_facts(attrs)
            self._window(attrs)
            self.sink.end_step(
                self.step_id, status="ok", attrs=attrs, content=content,
                wait_for_reply_message=self.tags.purpose == "main",
            )
            if self.tags.purpose == "main":
                self.sink.note_main_usage(attrs["usage"])
        except Exception:  # noqa: BLE001
            log.exception("recording: could not end a model step")

    def failed(self, exc: BaseException) -> None:
        try:
            if _is_stop(exc):
                return  # Stop: the turn's end closes the step as interrupted
            more, content = self._exchange()
            attrs: dict[str, Any] = {"finish_reason": None, "ttft_ms": self.ttft_ms if self.streamed else None,
                                     "usage": {"reported": False}, **more}
            self._window(attrs)
            text = str(root_cause(exc)).strip()[:200] or None
            self.sink.end_step(self.step_id, status="error", attrs=attrs, content=content,
                               error={"type": classify_error(exc), "text": text})
        except Exception:  # noqa: BLE001
            log.exception("recording: could not record a failed model step")


def _reply_usage(entries: list[dict[str, Any]]) -> Any:
    """The `usage` object of the last JSON reply, as `TokenUsage`; None if there is none."""
    from ..providers.openai_responses import _usage_from

    for e in reversed(entries):
        try:
            body = json.loads(e.get("body") or "")
        except ValueError:
            continue
        usage = body.get("usage") if isinstance(body, dict) else None
        if isinstance(usage, dict):
            return _usage_from(_ns(usage))
    return None


def _ns(value: Any) -> Any:
    if isinstance(value, dict):
        return SimpleNamespace(**{k: _ns(v) for k, v in value.items()})
    return value


def _is_stop(exc: BaseException) -> bool:
    return isinstance(exc, (GeneratorExit, KeyboardInterrupt))


def _fill_from_facts(attrs: dict[str, Any]) -> None:
    """A count the provider sent is recorded even where the provider code dropped it
    (V-13, V-15), and impossible numbers are flagged, never corrected (V-19)."""
    use, facts = attrs.get("usage") or {}, attrs.get("provider_facts") or {}
    if not use.get("reported"):
        return
    if use.get("reasoning") is None and facts.get("reasoning") is not None:
        use["reasoning"] = facts["reasoning"]
    if use.get("cache_write") is None and facts.get("cache_write") is not None:
        use["cache_write"] = facts["cache_write"]
    flags = []
    if use.get("reasoning") is not None and use.get("output") is not None and use["reasoning"] > use["output"]:
        flags.append("reasoning_exceeds_output")
    total = facts.get("provider_total")
    parts = (use.get("input_fresh") or 0) + (use.get("cache_read") or 0) + (use.get("cache_write") or 0) + (use.get("output") or 0)
    if total is not None and total != parts:
        flags.append("provider_total_differs")
    if flags:
        use["inconsistent"] = flags


def view_for_engine(provider: Any, sink: TurnRecording) -> "ProviderView":
    """The engine's own view (§3.3): tags its creator already set are kept, so a helper's
    engine keeps `helper`; a plain provider is wrapped first."""
    if isinstance(provider, ProviderView):
        tags = provider.tags
        return provider.tagged(
            purpose="main" if tags.purpose == "unattributed" else tags.purpose,
            sink=sink if tags.sink is None or tags.sink is provider.base._sink else tags.sink,
        )
    base = provider if isinstance(provider, RecordingProvider) else RecordingProvider(provider)
    return base.tagged(purpose="main", sink=sink)


def tag(provider: Any, **tags: Any) -> Any:
    """`provider.tagged(**tags)` for a recording provider or view; any other provider as is."""
    tagged = getattr(type(provider), "tagged", None)
    return provider.tagged(**tags) if tagged is not None else provider