Skip to content

Personal AI, On Personal Devices

Python
9.2 KB
Raw Blame
"""Engine discovery — probe running engines and aggregate available models."""

from __future__ import annotations

import logging
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, List, Tuple

from openjarvis.core.config import JarvisConfig
from openjarvis.core.registry import EngineRegistry
from openjarvis.engine._base import InferenceEngine

logger = logging.getLogger(__name__)

# Map registry keys to config host attribute (None = no host arg)
_HOST_MAP: Dict[str, str | None] = {
    "ollama": "ollama_host",
    "vllm": "vllm_host",
    "llamacpp": "llamacpp_host",
    "sglang": "sglang_host",
    "mlx": "mlx_host",
    "lmstudio": "lmstudio_host",
    "exo": "exo_host",
    "nexa": "nexa_host",
    "uzu": "uzu_host",
    "apple_fm": "apple_fm_host",
    "lemonade": "lemonade_host",
    "cloud": None,
    "litellm": None,
    "gemma_cpp": None,
    # In-process: drives the Apple FM SDK directly, so there is no host.
    "afm": None,
}


def _make_engine(key: str, config: JarvisConfig) -> InferenceEngine:
    """Instantiate a registered engine with the appropriate config host."""
    cls = EngineRegistry.get(key)

    # LiteLLM cannot enumerate every model supported by every provider.  Its
    # list_models() contract therefore advertises the configured default
    # model, which must be supplied when discovery constructs the engine.
    if key == "litellm":
        return cls(default_model=config.intelligence.default_model or None)

    # gemma_cpp: pass config fields instead of host
    if key == "gemma_cpp":
        cfg = config.engine.gemma_cpp
        return cls(
            model_path=cfg.model_path or None,
            tokenizer_path=cfg.tokenizer_path or None,
            model_type=cfg.model_type or None,
            num_threads=cfg.num_threads,
        )

    # afm: in-process engine, configured by behaviour rather than a host
    if key == "afm":
        cfg = config.engine.afm
        return cls(
            instructions=cfg.instructions,
            use_case=cfg.use_case,
            guardrails=cfg.guardrails,
            sampling=cfg.sampling,
        )

    host_attr = _HOST_MAP.get(key)
    if host_attr is not None:
        host = getattr(config.engine, host_attr, None)
        if host:
            return cls(host=host)
    return cls()


def _maybe_register_mining_sidecar_engine() -> None:
    """If a mining sidecar exists with a ``vllm_endpoint``, register a derived
    vLLM engine class pointing at it.  Idempotent.  Quiet on error.

    The trigger is the *shape* of the sidecar (presence of ``vllm_endpoint``),
    not the value of its ``provider`` field — this leaves room for future
    non-engine-replacing providers (e.g., a hypothetical cpu-pearl) whose
    sidecars don't include ``vllm_endpoint``.
    """
    try:
        from openjarvis.mining import Sidecar
        from openjarvis.mining._constants import SIDECAR_PATH
    except ImportError:
        return

    if EngineRegistry.contains("vllm-pearl-mining"):
        return  # idempotent

    payload = Sidecar.read(SIDECAR_PATH)
    if payload is None:
        return

    endpoint = payload.get("vllm_endpoint")
    model = payload.get("model")
    if not endpoint or not model:
        return  # data-driven gate: no vllm_endpoint → don't register

    from openjarvis.engine._openai_compat import _OpenAICompatibleEngine

    # Strip a trailing "/v1" path segment so _default_host is the bare
    # base URL and _api_prefix="/v1" combines correctly in request paths.
    api_prefix = "/v1"
    base_url = endpoint.rstrip("/")
    if base_url.endswith(api_prefix):
        base_url = base_url[: -len(api_prefix)]

    _cls = type(
        "VllmPearlMiningEngine",
        (_OpenAICompatibleEngine,),
        {
            "engine_id": "vllm-pearl-mining",
            "_default_host": base_url,
            "_api_prefix": api_prefix,
        },
    )
    EngineRegistry.register_value("vllm-pearl-mining", _cls)


def discover_engines(config: JarvisConfig) -> List[Tuple[str, InferenceEngine]]:
    """Probe registered engines and return ``[(key, instance)]`` for healthy ones.

    Results are sorted with the config default engine first.
    """
    _maybe_register_mining_sidecar_engine()

    # Probe engines concurrently: each health() does a blocking network
    # check with its own timeout, so a serial loop costs the SUM of all
    # probe timeouts (dead localhost ports especially). Running them in
    # threads collapses that to roughly the slowest single probe. The
    # healthy.sort() below normalizes order, so completion order is
    # irrelevant and the result is identical to the serial version (#263).
    keys = list(EngineRegistry.keys())

    def _probe(key: str) -> Tuple[str, InferenceEngine] | None:
        try:
            engine = _make_engine(key, config)
            if engine.health():
                return (key, engine)
        except Exception as exc:
            logger.debug("Engine %r failed during discovery: %s", key, exc)
        return None

    healthy: List[Tuple[str, InferenceEngine]] = []
    if keys:
        with ThreadPoolExecutor(max_workers=len(keys)) as pool:
            for result in pool.map(_probe, keys):
                if result is not None:
                    healthy.append(result)

    default_key = config.engine.default

    def sort_key(item: Tuple[str, Any]) -> Tuple[int, str]:
        return (0 if item[0] == default_key else 1, item[0])

    healthy.sort(key=sort_key)
    return healthy


def discover_models(
    engines: List[Tuple[str, InferenceEngine]],
) -> Dict[str, List[str]]:
    """Call ``list_models()`` on each engine and return a dict."""
    result: Dict[str, List[str]] = {}
    for key, engine in engines:
        try:
            result[key] = engine.list_models()
        except Exception as exc:
            logger.debug("Failed to list models for engine %r: %s", key, exc)
            result[key] = []
    return result


def get_engine(
    config: JarvisConfig,
    engine_key: str | None = None,
    model: str | None = None,
) -> Tuple[str, InferenceEngine] | None:
    """Get a specific engine by key, or the default with a named fallback.

    An explicit ``engine_key`` is authoritative and is never silently
    substituted. Default-engine fallback prefers the same local/cloud class
    and logs the selected replacement.

    When *model* is given, an engine is selected only if it can actually
    serve that model (``engine.can_serve(model)``). This stops the cloud
    fallback from being chosen — when the local engine is down — for a model
    whose provider client is missing, which otherwise surfaces as a confusing
    "OpenAI client not available" instead of a helpful "start your local
    engine" message (see #532). When *model* is ``None`` selection stays
    model-agnostic (unchanged behaviour).

    Returns ``(key, engine_instance)`` or ``None`` if no engine is available.
    """

    def _usable(engine: InferenceEngine) -> bool:
        return engine.health() and (model is None or engine.can_serve(model))

    if engine_key:
        if not EngineRegistry.contains(engine_key):
            logger.warning("Requested engine %r is not registered", engine_key)
            return None
        try:
            engine = _make_engine(engine_key, config)
            if _usable(engine):
                return (engine_key, engine)
        except Exception as exc:
            logger.debug("Engine %r health check failed: %s", engine_key, exc)
        logger.warning(
            "Requested engine %r is unavailable%s; no substitute was selected",
            engine_key,
            f" for model {model!r}" if model else "",
        )
        return None

    default_key = config.engine.default
    default_is_cloud: bool | None = None
    if default_key and EngineRegistry.contains(default_key):
        default_cls = EngineRegistry.get(default_key)
        default_is_cloud = bool(getattr(default_cls, "is_cloud", False))
        try:
            engine = _make_engine(default_key, config)
            default_is_cloud = bool(getattr(engine, "is_cloud", default_is_cloud))
            if _usable(engine):
                return (default_key, engine)
        except Exception as exc:
            logger.debug("Engine %r health check failed: %s", default_key, exc)

    candidates = [
        (key, engine)
        for key, engine in discover_engines(config)
        if key != default_key and (model is None or engine.can_serve(model))
    ]
    if not candidates:
        return None

    chosen: Tuple[str, InferenceEngine] | None = None
    if default_is_cloud is not None:
        chosen = next(
            (
                candidate
                for candidate in candidates
                if bool(getattr(candidate[1], "is_cloud", False)) == default_is_cloud
            ),
            None,
        )
    chosen = chosen or candidates[0]
    chosen_is_cloud = bool(getattr(chosen[1], "is_cloud", False))
    boundary = (
        " across the local/cloud boundary"
        if (default_is_cloud is not None and chosen_is_cloud != default_is_cloud)
        else ""
    )
    logger.warning(
        "Default engine %r is unavailable; using %r%s",
        default_key,
        chosen[0],
        boundary,
    )
    return chosen


__all__ = ["discover_engines", "discover_models", "get_engine"]