Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

11.7 KB
Raw Blame
"""Shared, bounded HTTP transport for the ODS host agent."""

from __future__ import annotations

import asyncio
import errno
import json
import threading
import time
from typing import Any

import httpx

from config import AGENT_URL, ODS_AGENT_KEY


class AgentClientError(RuntimeError):
    """Base error for host-agent transport failures."""


class AgentHTTPError(AgentClientError):
    """The host agent returned a non-success HTTP status."""

    def __init__(self, status_code: int, detail: str, response_text: str = "") -> None:
        super().__init__(detail)
        self.status_code = status_code
        self.detail = detail
        self.response_text = response_text


def agent_reports_busy(error: AgentHTTPError) -> bool:
    """True for the host's 409 refusal made before acting: another model or
    Portal change holds the lifecycle, so nothing was changed."""
    if error.status_code != 409:
        return False
    try:
        body = json.loads(error.response_text)
    except (TypeError, ValueError):
        return False
    return isinstance(body, dict) and "model-lifecycle-busy" in (body.get("error"), body.get("code"))


class AgentUnavailable(AgentClientError):
    """The host agent could not be reached."""


class AgentTimeout(AgentUnavailable):
    """The host-agent request exceeded an explicit timeout."""


class AgentProtocolError(AgentClientError):
    """The host agent returned a response that violated its JSON contract."""


# The host agent closes idle keep-alive sockets after 30 s
# (ThreadedHTTPServer.request_socket_timeout). Expire pooled connections well
# before that: with equal timeouts the pool can reuse a socket the agent is
# closing, and behind Docker Desktop's proxy that close may never reach the
# container, so the request hangs until its read timeout.
_KEEPALIVE_EXPIRY_SECONDS = 10.0
_SYNC_LIMITS = httpx.Limits(
    max_connections=8,
    max_keepalive_connections=4,
    keepalive_expiry=_KEEPALIVE_EXPIRY_SECONDS,
)
_ASYNC_LIMITS = httpx.Limits(
    max_connections=2,
    max_keepalive_connections=2,
    keepalive_expiry=_KEEPALIVE_EXPIRY_SECONDS,
)
_sync_client: httpx.Client | None = None
_async_client: httpx.AsyncClient | None = None
_sync_client_lock = threading.Lock()
_async_client_lock = threading.Lock()

# Docker Desktop can briefly withdraw the synthetic host.docker.internal route
# while native model operations are changing container state.  A connect error
# is safe to retry even for POST because no connection was established and no
# request bytes reached the host agent.  Read/write/protocol failures remain
# non-retryable for POST so model activations and extension operations cannot be
# duplicated after the agent may have accepted them.
_CONNECT_RETRY_DELAYS_SECONDS = (0.25, 1.0, 3.0, 6.0)
_TRANSIENT_ROUTE_ERRNOS = {
    errno.ENETUNREACH,
    errno.EHOSTUNREACH,
    10051,  # WSAENETUNREACH
    10065,  # WSAEHOSTUNREACH
}
_TRANSIENT_ROUTE_MESSAGES = (
    "network is unreachable",
    "no route to host",
)


def _is_transient_route_connect_error(exc: BaseException) -> bool:
    """Return True only for a pre-connect host-route withdrawal.

    Connection refused, DNS errors, and generic mocked ConnectErrors still
    fail immediately.  The bounded retry is reserved for Docker Desktop's
    observed ENETUNREACH/EHOSTUNREACH route flap.
    """
    current: BaseException | None = exc
    seen: set[int] = set()
    while current is not None and id(current) not in seen:
        seen.add(id(current))
        if getattr(current, "errno", None) in _TRANSIENT_ROUTE_ERRNOS:
            return True
        message = str(current).casefold()
        if any(token in message for token in _TRANSIENT_ROUTE_MESSAGES):
            return True
        current = current.__cause__ or current.__context__
    return False


def _headers() -> dict[str, str]:
    return {"Authorization": f"Bearer {ODS_AGENT_KEY}"}


def _timeout(read_seconds: float) -> httpx.Timeout:
    return httpx.Timeout(
        connect=5.0,
        read=max(0.1, float(read_seconds)),
        write=30.0,
        pool=5.0,
    )


def _get_sync_client() -> httpx.Client:
    global _sync_client

    client = _sync_client
    if client is not None and not client.is_closed:
        return client
    with _sync_client_lock:
        client = _sync_client
        if client is None or client.is_closed:
            _sync_client = httpx.Client(
                base_url=AGENT_URL,
                headers=_headers(),
                limits=_SYNC_LIMITS,
                timeout=_timeout(5.0),
                trust_env=False,
            )
        return _sync_client


async def _get_async_client() -> httpx.AsyncClient:
    global _async_client

    loop = asyncio.get_running_loop()
    client = _async_client
    owner_loop = getattr(client, "_ods_owner_loop", None) if client is not None else None
    if (
        client is not None
        and not client.is_closed
        and (owner_loop is None or owner_loop is loop)
    ):
        # Tests and explicit adapters may inject a client. Adopt it into the
        # current loop the first time rather than treating it as stale.
        if owner_loop is None:
            setattr(client, "_ods_owner_loop", loop)
        return client
    with _async_client_lock:
        client = _async_client
        owner_loop = getattr(client, "_ods_owner_loop", None) if client is not None else None
        if (
            client is None
            or client.is_closed
            or owner_loop is not None and owner_loop is not loop
        ):
            _async_client = httpx.AsyncClient(
                base_url=AGENT_URL,
                headers=_headers(),
                limits=_ASYNC_LIMITS,
                timeout=_timeout(5.0),
                trust_env=False,
            )
            setattr(_async_client, "_ods_owner_loop", loop)
        return _async_client


def _error_detail(response: httpx.Response) -> tuple[str, str]:
    text = response.text
    detail: Any = None
    try:
        payload = response.json()
        if isinstance(payload, dict):
            detail = payload.get("error") or payload.get("detail")
    except ValueError:
        pass
    if isinstance(detail, (dict, list)):
        detail = json.dumps(detail, separators=(",", ":"))
    if not isinstance(detail, str) or not detail.strip():
        detail = f"Host agent returned HTTP {response.status_code}"
    return detail, text


def _raise_for_status(response: httpx.Response) -> None:
    if response.is_success:
        return
    detail, text = _error_detail(response)
    raise AgentHTTPError(response.status_code, detail, text)


def _decode_json(response: httpx.Response) -> dict[str, Any]:
    try:
        payload = response.json()
    except ValueError as exc:
        raise AgentProtocolError("Host agent returned invalid JSON") from exc
    if not isinstance(payload, dict):
        raise AgentProtocolError("Host agent returned non-object JSON")
    return payload


def _sync_request(
    method: str,
    path: str,
    *,
    payload: Any = None,
    params: dict[str, Any] | None = None,
    timeout: float,
) -> httpx.Response:
    method = method.upper()
    stale_connection_retries = 1 if method == "GET" else 0
    connect_retry_index = 0
    while True:
        try:
            return _get_sync_client().request(
                method,
                path,
                json=payload if payload is not None else None,
                params=params,
                timeout=_timeout(timeout),
            )
        except httpx.TimeoutException as exc:
            raise AgentTimeout(f"Host agent {method} {path} timed out") from exc
        except httpx.ConnectError as exc:
            if (
                _is_transient_route_connect_error(exc)
                and connect_retry_index < len(_CONNECT_RETRY_DELAYS_SECONDS)
            ):
                time.sleep(_CONNECT_RETRY_DELAYS_SECONDS[connect_retry_index])
                connect_retry_index += 1
                continue
            raise AgentUnavailable(f"Host agent {method} {path} is unreachable: {exc}") from exc
        except (httpx.NetworkError, httpx.RemoteProtocolError) as exc:
            if stale_connection_retries:
                stale_connection_retries -= 1
                continue
            raise AgentUnavailable(f"Host agent {method} {path} is unreachable: {exc}") from exc
        except httpx.RequestError as exc:
            raise AgentUnavailable(f"Host agent {method} {path} is unreachable: {exc}") from exc


async def _async_request(
    method: str,
    path: str,
    *,
    payload: Any = None,
    params: dict[str, Any] | None = None,
    timeout: float,
) -> httpx.Response:
    method = method.upper()
    stale_connection_retries = 1 if method == "GET" else 0
    connect_retry_index = 0
    while True:
        try:
            client = await _get_async_client()
            return await client.request(
                method,
                path,
                json=payload if payload is not None else None,
                params=params,
                timeout=_timeout(timeout),
            )
        except httpx.TimeoutException as exc:
            raise AgentTimeout(f"Host agent {method} {path} timed out") from exc
        except httpx.ConnectError as exc:
            if (
                _is_transient_route_connect_error(exc)
                and connect_retry_index < len(_CONNECT_RETRY_DELAYS_SECONDS)
            ):
                await asyncio.sleep(_CONNECT_RETRY_DELAYS_SECONDS[connect_retry_index])
                connect_retry_index += 1
                continue
            raise AgentUnavailable(f"Host agent {method} {path} is unreachable: {exc}") from exc
        except (httpx.NetworkError, httpx.RemoteProtocolError) as exc:
            if stale_connection_retries:
                stale_connection_retries -= 1
                continue
            raise AgentUnavailable(f"Host agent {method} {path} is unreachable: {exc}") from exc
        except httpx.RequestError as exc:
            raise AgentUnavailable(f"Host agent {method} {path} is unreachable: {exc}") from exc


def request_json(
    method: str,
    path: str,
    *,
    payload: Any = None,
    params: dict[str, Any] | None = None,
    timeout: float = 5.0,
) -> dict[str, Any]:
    response = _sync_request(method, path, payload=payload, params=params, timeout=timeout)
    _raise_for_status(response)
    return _decode_json(response)


def request_text(
    method: str,
    path: str,
    *,
    payload: Any = None,
    params: dict[str, Any] | None = None,
    timeout: float = 5.0,
) -> str:
    response = _sync_request(method, path, payload=payload, params=params, timeout=timeout)
    _raise_for_status(response)
    return response.text


async def async_request_json(
    method: str,
    path: str,
    *,
    payload: Any = None,
    params: dict[str, Any] | None = None,
    timeout: float = 5.0,
) -> dict[str, Any]:
    response = await _async_request(
        method, path, payload=payload, params=params, timeout=timeout
    )
    _raise_for_status(response)
    return _decode_json(response)


async def async_request_text(
    method: str,
    path: str,
    *,
    payload: Any = None,
    params: dict[str, Any] | None = None,
    timeout: float = 5.0,
) -> str:
    response = await _async_request(
        method, path, payload=payload, params=params, timeout=timeout
    )
    _raise_for_status(response)
    return response.text


async def shutdown_clients() -> None:
    """Close both process-wide pools during FastAPI shutdown."""
    global _sync_client, _async_client

    with _sync_client_lock:
        sync_client, _sync_client = _sync_client, None
    with _async_client_lock:
        async_client, _async_client = _async_client, None
    if sync_client is not None and not sync_client.is_closed:
        await asyncio.to_thread(sync_client.close)
    if async_client is not None and not async_client.is_closed:
        await async_client.aclose()