Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

5.4 KB
Raw Blame
"""Aggregate the per-serve control API (``server/control_api.py``: ``/health``,
``/v1/stats``, ``/v1/admin/prepare-stop``) up to the daemon. The per-serve ``/health`` answers
"how is the model doing?" and dies with the serve; the daemon re-exposes it under
``/engine/health`` alongside
its own reachability, so a client has one control endpoint that OUTLIVES any single serve.

Blocking ``urllib`` (stdlib — no new dependency) run from the daemon's dedicated proxy executor
(kept off the lifecycle executor so a slow/loading serve can never starve
``/engine/stop``). Single-flight + short TTL cache: N concurrent pollers cost one upstream probe.
Serve docs are snake_case; they are transformed to the daemon's camelCase contract."""

from __future__ import annotations

import json
import re
import threading
import time
import urllib.error
import urllib.request
from typing import Any, Callable

from .accounting import AccountingPrepareError, PrepareStopUnavailable

_SNAKE_RE = re.compile(r"_([a-z0-9])")


def _camel_key(key: str) -> str:
    return _SNAKE_RE.sub(lambda m: m.group(1).upper(), key)


def to_camel(obj: Any) -> Any:
    """Recursively camelCase dict keys (uptime_s→uptimeS, done_bytes→doneBytes, …) so the daemon
    emits one casing convention everywhere."""
    if isinstance(obj, dict):
        return {_camel_key(k): to_camel(v) for k, v in obj.items()}
    if isinstance(obj, list):
        return [to_camel(v) for v in obj]
    return obj


class ServeProbe:
    def __init__(
        self,
        *,
        host: str = "127.0.0.1",
        ttl_s: float = 0.25,
        timeout_s: float = 1.5,
        prepare_timeout_s: float = 15.0,
        now: Callable[[], float] = time.monotonic,
        opener: Callable[[str, float], dict] | None = None,
        prepare_opener: Callable[[str, float], dict] | None = None,
        api_key: Callable[[], str | None] = lambda: None,
    ) -> None:
        self._host = host
        self._ttl = ttl_s
        self._timeout = timeout_s
        self._prepare_timeout = prepare_timeout_s
        self._now = now
        self._opener = opener or self._urlopen
        self._prepare_opener = prepare_opener or self._urlopen_prepare
        self._api_key = api_key
        self._lock = threading.Lock()
        self._cache: dict[tuple[str, int], tuple[float, dict]] = {}

    def health(self, port: int) -> dict:
        return self._cached("health", "/health", port)

    def stats(self, port: int) -> dict:
        return self._cached("stats", "/v1/stats", port)

    def fresh_stats(self, port: int) -> dict:
        """Fetch uncached stats for a legacy stop receipt."""
        return self._fetch("/v1/stats", port)

    def prepare_stop(self, port: int) -> dict:
        """Quiesce the engine and return its sealed final accounting snapshot."""
        url = f"http://{self._host}:{port}/v1/admin/prepare-stop"
        try:
            doc = self._prepare_opener(url, self._prepare_timeout)
        except urllib.error.HTTPError as exc:
            if exc.code in (404, 405):
                raise PrepareStopUnavailable("legacy-engine") from exc
            raise AccountingPrepareError(f"prepare-stop returned HTTP {exc.code}") from exc
        except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
            raise AccountingPrepareError(f"prepare-stop request failed: {exc}") from exc
        if not isinstance(doc, dict):
            raise AccountingPrepareError("prepare-stop returned a non-object response")
        return doc

    def _cached(self, kind: str, path: str, port: int) -> dict:
        key = (kind, port)
        # Hold the lock across the fetch so concurrent pollers collapse to a single upstream call
        # (true single-flight); the short timeout + TTL keep the critical section cheap.
        with self._lock:
            hit = self._cache.get(key)
            if hit is not None and (self._now() - hit[0]) < self._ttl:
                return hit[1]
            val = self._fetch(path, port)
            self._cache[key] = (self._now(), val)
            return val

    def _fetch(self, path: str, port: int) -> dict:
        url = f"http://{self._host}:{port}{path}"
        try:
            doc = self._opener(url, self._timeout)
        except urllib.error.HTTPError as exc:
            return {"reachable": True, "status": "error", "httpStatus": exc.code}
        except (urllib.error.URLError, TimeoutError, OSError, ValueError):
            return {"reachable": False, "status": "unreachable"}
        result = to_camel(doc) if isinstance(doc, dict) else {"value": doc}
        result["reachable"] = True
        return result

    def _headers(self) -> dict[str, str]:
        headers = {"Accept": "application/json"}
        if key := self._api_key():
            headers["Authorization"] = f"Bearer {key}"
        return headers

    def _urlopen(self, url: str, timeout: float) -> dict:
        req = urllib.request.Request(url, headers=self._headers(), method="GET")
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read()
        return json.loads(raw.decode("utf-8"))

    def _urlopen_prepare(self, url: str, timeout: float) -> dict:
        req = urllib.request.Request(
            url,
            data=b"{}",
            headers={**self._headers(), "Content-Type": "application/json"},
            method="POST",
        )
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read()
        return json.loads(raw.decode("utf-8"))