Skip to content

Personal AI, On Personal Devices

Python
14.1 KB
Raw Blame
"""Shared base for OpenAI-compatible ``/v1/`` engines."""

from __future__ import annotations

import json
import logging
from collections.abc import AsyncIterator, Sequence
from typing import Any, Dict, List

import httpx

from openjarvis.core.types import Message
from openjarvis.engine._base import (
    EngineConnectionError,
    EngineContextLengthError,
    InferenceEngine,
    estimate_prompt_tokens,
    messages_to_dicts,
)
from openjarvis.engine._http_async import (
    STREAM_TRANSPORT_ERRORS,
    AsyncHTTPEngineMixin,
)
from openjarvis.engine._stubs import StreamChunk
from openjarvis.intelligence.model_catalog import resolve_model_id_for_engine

logger = logging.getLogger(__name__)


class _OpenAICompatibleEngine(AsyncHTTPEngineMixin, InferenceEngine):
    """Base for engines that serve the OpenAI ``/v1/chat/completions`` API."""

    # vLLM/SGLang report context-window overflows in 400 bodies; the shared
    # ``_raise_stream_http_error`` types those as ``EngineContextLengthError``.
    _stream_400_signals_context_length = True

    engine_id: str = ""
    _default_host: str = "http://localhost:8000"
    _api_prefix: str = "/v1"

    def __init__(
        self,
        host: str | None = None,
        *,
        api_key: str | None = None,
        timeout: float = 600.0,
    ) -> None:
        import os

        # Sanitize the engine id for env-var lookup ("openai-compat" ->
        # "OPENAI_COMPAT_..."); shells cannot set hyphenated variable names.
        env_prefix = self.engine_id.upper().replace("-", "_")
        self._host = (
            host or os.environ.get(f"{env_prefix}_HOST") or self._default_host
        ).rstrip("/")
        # Bearer auth for endpoints started with e.g. ``vllm serve --api-key``.
        # Setting it on the client covers generate/stream/stream_full/
        # list_models/health alike; ``None`` keeps requests header-free.
        self._api_key = api_key or os.environ.get(f"{env_prefix}_API_KEY") or None
        headers = (
            {"Authorization": f"Bearer {self._api_key}"} if self._api_key else None
        )
        # Used by the shared async streaming plumbing (AsyncHTTPEngineMixin) so
        # the bounded request timeout is applied to streaming reads, not just
        # the synchronous methods (a wedged token read fails at ``timeout``
        # rather than hanging the caller for the httpx default).
        self._timeout = timeout
        self._headers = headers
        # Injection seam for tests: an ``httpx.MockTransport`` swapped in here lets
        # the async stream path be exercised with a mocked transport and no real
        # server. ``None`` in production so httpx uses its default networking.
        self._async_transport: httpx.AsyncBaseTransport | None = None
        self._client = httpx.Client(
            base_url=self._host, timeout=timeout, headers=headers
        )

    # -- InferenceEngine interface ------------------------------------------

    def _resolve_model_id(self, model: str) -> str:
        return resolve_model_id_for_engine(model, self.engine_id)

    def generate(
        self,
        messages: Sequence[Message],
        *,
        model: str,
        temperature: float = 0.7,
        max_tokens: int = 1024,
        **kwargs: Any,
    ) -> Dict[str, Any]:
        payload: Dict[str, Any] = {
            "model": self._resolve_model_id(model),
            "messages": messages_to_dicts(messages),
            "temperature": temperature,
            "max_tokens": max_tokens,
            "stream": False,
            **kwargs,
        }
        # Default to tool_choice=auto when tools are provided
        if "tools" in payload and "tool_choice" not in payload:
            payload["tool_choice"] = "auto"
        try:
            url = f"{self._api_prefix}/chat/completions"
            resp = self._client.post(url, json=payload)
            if resp.status_code == 400 and "tools" in payload:
                payload.pop("tools", None)
                payload.pop("tool_choice", None)
                resp = self._client.post(url, json=payload)
            resp.raise_for_status()
        except (httpx.ConnectError, httpx.TimeoutException) as exc:
            raise EngineConnectionError(
                f"{self.engine_id} engine not reachable at {self._host}"
            ) from exc
        except httpx.HTTPStatusError as exc:
            error_detail = exc.response.text.strip()
            if exc.response.status_code == 404:
                detail_suffix = (
                    f" Response body: {error_detail}" if error_detail else ""
                )
                raise EngineConnectionError(
                    f"{self.engine_id} engine at {self._host} returned 404 for "
                    f"{self._api_prefix}/chat/completions. Make sure this port "
                    "is running an OpenAI-compatible chat server, not another "
                    f"local web service.{detail_suffix}"
                ) from exc
            detail_suffix = f": {error_detail}" if error_detail else ""
            raise EngineConnectionError(
                f"{self.engine_id} engine at {self._host} returned HTTP "
                f"{exc.response.status_code}{detail_suffix}"
            ) from exc
        data = resp.json()
        choices = data.get("choices", [])
        if not choices:
            return {
                "content": "",
                "usage": data.get("usage", {}),
                "model": data.get("model", model),
                "finish_reason": "error",
            }
        choice = choices[0]
        usage = data.get("usage", {})
        # Ensure prompt_tokens reflects the full prompt size (including
        # system prompt and all conversation history).
        # OpenAI-compat APIs (vLLM, SGLang) report full counts — KV
        # caching is transparent, so evaluated == full.
        reported_prompt = usage.get("prompt_tokens", 0)
        estimated_prompt = estimate_prompt_tokens(messages)
        prompt_tokens = max(reported_prompt, estimated_prompt)
        completion_tokens = usage.get("completion_tokens", 0)
        result: Dict[str, Any] = {
            "content": choice["message"].get("content") or "",
            "usage": {
                "prompt_tokens": prompt_tokens,
                "prompt_tokens_evaluated": reported_prompt or prompt_tokens,
                "completion_tokens": completion_tokens,
                "total_tokens": prompt_tokens + completion_tokens,
            },
            "model": data.get("model", model),
            "finish_reason": choice.get("finish_reason", "stop"),
        }
        # Extract tool calls if present
        raw_tool_calls = choice["message"].get("tool_calls", [])
        if raw_tool_calls:
            result["tool_calls"] = [
                {
                    "id": tc.get("id", ""),
                    "name": tc.get("function", {}).get("name", ""),
                    "arguments": tc.get("function", {}).get("arguments", "{}"),
                }
                for tc in raw_tool_calls
            ]
        return result

    async def stream(
        self,
        messages: Sequence[Message],
        *,
        model: str,
        temperature: float = 0.7,
        max_tokens: int = 1024,
        **kwargs: Any,
    ) -> AsyncIterator[str]:
        payload: Dict[str, Any] = {
            "model": self._resolve_model_id(model),
            "messages": messages_to_dicts(messages),
            "temperature": temperature,
            "max_tokens": max_tokens,
            "stream": True,
            **kwargs,
        }
        # Default to tool_choice=auto when tools are provided
        if "tools" in payload and "tool_choice" not in payload:
            payload["tool_choice"] = "auto"
        url = f"{self._api_prefix}/chat/completions"
        try:
            # ASYNC streaming: ``httpx.AsyncClient`` + ``aiter_lines`` never
            # blocks the event loop between tokens (the previous SYNC
            # ``httpx.Client`` + ``iter_lines`` inside this ``async def`` blocked
            # the single uvicorn worker on every inter-token wait, serializing all
            # concurrent chats and letting one wedged read freeze the whole API).
            # The shared client keeps pooled connections across turns.
            client = self._get_async_client()
            async with client.stream("POST", url, json=payload) as resp:
                # ``not is_success`` covers 3xx as well as 4xx/5xx. With
                # ``follow_redirects`` off (the default) an unexpected redirect
                # would otherwise fall through to ``aiter_lines`` and surface as
                # a silent EMPTY stream instead of a clean engine error.
                if not resp.is_success:
                    # Load the (short) error body before touching ``.text``:
                    # a streaming response is otherwise unread.
                    await resp.aread()
                    self._raise_stream_http_error(resp.status_code, resp.text)
                async for line in resp.aiter_lines():
                    if not line.startswith("data:"):
                        continue
                    data_str = line[len("data:") :].strip()
                    if data_str == "[DONE]":
                        break
                    try:
                        chunk = json.loads(data_str)
                    except json.JSONDecodeError:
                        continue
                    delta = chunk.get("choices", [{}])[0].get("delta", {})
                    content = delta.get("content")
                    if content:
                        yield content
        except STREAM_TRANSPORT_ERRORS as exc:
            # A wedged upstream read trips ``timeout`` (ReadTimeout) and is mapped
            # here, so the request fails cleanly at the configured bound instead
            # of hanging indefinitely (see STREAM_TRANSPORT_ERRORS for why the
            # set is exactly this narrow).
            raise EngineConnectionError(
                f"{self.engine_id} engine not reachable at {self._host}"
            ) from exc

    async def stream_full(
        self,
        messages: Sequence[Message],
        *,
        model: str,
        temperature: float = 0.7,
        max_tokens: int = 1024,
        **kwargs: Any,
    ) -> AsyncIterator["StreamChunk"]:
        """Yield StreamChunks with content, tool_calls, and finish_reason."""
        msg_dicts = messages_to_dicts(messages)
        payload: Dict[str, Any] = {
            "model": self._resolve_model_id(model),
            "messages": msg_dicts,
            "temperature": temperature,
            "max_tokens": max_tokens,
            "stream": True,
            **kwargs,
        }
        if "tools" in payload and "tool_choice" not in payload:
            payload["tool_choice"] = "auto"
        url = f"{self._api_prefix}/chat/completions"
        try:
            # ASYNC streaming (see ``stream``): non-blocking shared client so
            # rich streaming never stalls the event loop and honours ``timeout``.
            client = self._get_async_client()
            async with client.stream("POST", url, json=payload) as resp:
                # ``not is_success`` covers 3xx as well as 4xx/5xx. With
                # ``follow_redirects`` off (the default) an unexpected redirect
                # would otherwise fall through to ``aiter_lines`` and surface as
                # a silent EMPTY stream instead of a clean engine error.
                if not resp.is_success:
                    await resp.aread()
                    self._raise_stream_http_error(resp.status_code, resp.text)
                async for line in resp.aiter_lines():
                    if not line.startswith("data:"):
                        continue
                    data_str = line[len("data:") :].strip()
                    if data_str == "[DONE]":
                        break
                    try:
                        chunk = json.loads(data_str)
                    except json.JSONDecodeError:
                        continue
                    choice = chunk.get("choices", [{}])[0]
                    delta = choice.get("delta", {})
                    finish = choice.get("finish_reason")
                    content = delta.get("content")
                    tool_calls = delta.get("tool_calls")
                    usage = chunk.get("usage")

                    if content or tool_calls or finish or usage:
                        yield StreamChunk(
                            content=content,
                            tool_calls=tool_calls,
                            finish_reason=finish,
                            usage=usage,
                        )
        except STREAM_TRANSPORT_ERRORS as exc:
            # See ``stream``: transport failures (incl. a mid-stream server
            # disconnect) map to a clean error; the set is kept narrow so
            # cancellation still propagates.
            raise EngineConnectionError(
                f"{self.engine_id} engine not reachable at {self._host}"
            ) from exc

    def list_models(self) -> List[str]:
        try:
            resp = self._client.get(f"{self._api_prefix}/models")
            resp.raise_for_status()
        except (
            httpx.ConnectError,
            httpx.TimeoutException,
            httpx.HTTPStatusError,
        ) as exc:
            logger.warning(
                "Failed to list models from %s at %s: %s",
                self.engine_id,
                self._host,
                exc,
            )
            return []
        data = resp.json()
        return [m["id"] for m in data.get("data", [])]

    def health(self) -> bool:
        try:
            resp = self._client.get(f"{self._api_prefix}/models", timeout=2.0)
            return resp.status_code == 200
        except Exception as exc:
            logger.debug(
                "%s health check failed at %s: %s",
                self.engine_id,
                self._host,
                exc,
            )
            return False

    def close(self) -> None:
        self._client.close()
        self._close_async_client()


# ``EngineContextLengthError`` moved to ``openjarvis.engine._base``; re-exported
# here for callers/tests that import it from this module.
__all__ = ["_OpenAICompatibleEngine", "EngineContextLengthError"]