Skip to content
Python
6.3 KB
Raw Blame
"""Provider-agnostic model access layer.

The runtime never imports a provider SDK directly — it talks to a `ProviderClient`.
Implementations: `OpenAIResponsesProvider` (native OpenAI via `/v1/responses`),
`OpenAIProvider` (Chat Completions — the compat world), and the native
Anthropic/Gemini/Bedrock/Vertex providers, all selected by the registry/router.
"""

from __future__ import annotations

from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Optional


@dataclass
class ToolCall:
    """A single tool call requested by the model, with parsed arguments."""

    id: str
    name: str
    arguments: dict[str, Any] = field(default_factory=dict)


@dataclass
class TokenUsage:
    """Normalized token counts for one model round-trip.

    `input` counts only fresh (uncached) prompt tokens; cached prompt tokens are
    split into `cache_read`/`cache_write`. Providers that don't report a cache
    split (Ollama, most compat vendors) leave the cache fields at 0. `output`
    includes thinking tokens where the vendor bills them as output (Gemini).

    For observability (E3) a count the provider did not send must read as unknown, not
    0: `cache_read_reported` / `cache_write_reported` say whether it was sent, and
    `reasoning` (thinking tokens within `output`) is None when not sent. The int fields
    stay ints, so existing sums are unchanged.
    """

    input: int = 0
    output: int = 0
    cache_read: int = 0
    cache_write: int = 0
    # Not part of equality: two usages with the same counts are equal, as before.
    reasoning: Optional[int] = field(default=None, compare=False)
    cache_read_reported: bool = field(default=True, compare=False)
    cache_write_reported: bool = field(default=True, compare=False)

    @property
    def context_tokens(self) -> int:
        """Prompt-side total — what actually occupied the context window."""
        return self.input + self.cache_read + self.cache_write

    def as_dict(self) -> dict[str, int]:
        return {
            "input": self.input,
            "output": self.output,
            "cache_read": self.cache_read,
            "cache_write": self.cache_write,
        }


@dataclass
class AssistantTurn:
    """One assistant response: free text and/or a set of tool calls."""

    text: Optional[str] = None
    tool_calls: list[ToolCall] = field(default_factory=list)
    finish_reason: Optional[str] = None
    raw: Any = field(default=None, repr=False, compare=False)
    # The model's thinking text (DeepSeek reasoning_content, Gemini thought summaries, …).
    # Display-only: persisted on the assistant message as the `reasoning` sidecar and shown
    # in the GUI, but stripped before every provider call — never replayed as context.
    reasoning: Optional[str] = None
    # Provider-private sidecars to persist on the canonical assistant message
    # (underscore-prefixed keys, e.g. `_gemini` thought signatures). Contract: the
    # owning provider consumes its own key when converting history; every other
    # provider must strip or ignore foreign underscore keys before its wire call.
    extras: dict[str, Any] = field(default_factory=dict)
    # Token counts for this round-trip, normalized across providers. None when the
    # backend didn't report usage (some compat servers) — never guessed.
    usage: Optional[TokenUsage] = None
    # The per-reply output-token ceiling the provider actually sent on this request
    # (`max_tokens` / `max_completion_tokens` / `max_output_tokens`), after any
    # provider-side adjustment (Anthropic budget floor, OpenAI rename). None when the
    # provider left it to the server. Persisted on the assistant message as the
    # `max_output_tokens` sidecar so a run record states its ceiling (OPE-177).
    output_limit: Optional[int] = None
    # The reasoning-effort mapping used for this request (OPE-176):
    # {requested, effective, param?, note?} — see providers/effort.py. None when no
    # level was configured or the provider has no such knob. Persisted on the
    # assistant message as the `reasoning_effort` sidecar.
    effort: Optional[dict[str, Any]] = None
    # The upstream host that served this reply, when a router names it (OpenRouter's
    # top-level `provider`, e.g. "Together"). Persisted as the `served_by` sidecar so a
    # run pinned to one host can prove the pin held on every reply.
    served_by: Optional[str] = None

    @property
    def has_tool_calls(self) -> bool:
        return bool(self.tool_calls)


@dataclass(frozen=True)
class ModelCapabilities:
    """What a given model/provider can do; used for graceful degradation."""

    tools: bool = True
    vision: bool = False
    # Native PDF ingestion (OpenAI `file` part / Anthropic document / Gemini inline_data).
    # Models without it get a local fallback: text extraction or page images (pdf_support.py).
    pdf: bool = False
    parallel_tool_calls: bool = True
    streaming: bool = True


@dataclass
class StreamChunk:
    """One streamed piece: a text and/or reasoning delta, and/or (final) the full turn."""

    text_delta: Optional[str] = None
    reasoning_delta: Optional[str] = None
    turn: Optional[AssistantTurn] = None


class ProviderClient(ABC):
    """Single-shot, provider-agnostic completion interface.

    Deliberately blocking (the turn engine wraps it in `asyncio.to_thread`) and
    deliberately without a `max_turns` loop — the runtime owns the agent loop.
    """

    @abstractmethod
    def complete(
        self,
        *,
        model: str,
        messages: list[dict[str, Any]],
        tools: Optional[list[dict[str, Any]]] = None,
        **settings: Any,
    ) -> AssistantTurn:
        """Return one assistant turn for the given messages/tools."""

    @abstractmethod
    def capabilities(self, model: str) -> ModelCapabilities:
        """Return capability flags for the given model."""

    def stream(
        self,
        *,
        model: str,
        messages: list[dict[str, Any]],
        tools: Optional[list[dict[str, Any]]] = None,
        **settings: Any,
    ):
        """Yield StreamChunks. Default: no token streaming — one final chunk with the
        full turn. Providers that support streaming (OpenAIProvider) override this."""
        yield StreamChunk(
            turn=self.complete(model=model, messages=messages, tools=tools, **settings)
        )