Personal AI, On Personal Devices
Python
feat: cloud model support — OpenAI, Anthropic, Google, OpenRouter in dropdown
Jon Saad-Falcon0dac51c649cf
diff --git a/src/openjarvis/cli/serve.py b/src/openjarvis/cli/serve.pyindex 1a9e21d0..9cc81705 100644--- a/src/openjarvis/cli/serve.py+++ b/src/openjarvis/cli/serve.py@@ -92,6 +92,29 @@ def serve(engine_name, engine = resolved+ # If cloud API keys are set, wrap with MultiEngine so both local+ # and cloud models appear in the model list and can be used.+ import os+ _has_cloud = (+ os.environ.get("OPENAI_API_KEY")+ or os.environ.get("ANTHROPIC_API_KEY")+ or os.environ.get("GEMINI_API_KEY")+ or os.environ.get("GOOGLE_API_KEY")+ or os.environ.get("OPENROUTER_API_KEY")+ )+ if _has_cloud and engine_name != "cloud":+ try:+ from openjarvis.engine.cloud import CloudEngine+ from openjarvis.engine.multi import MultiEngine++ cloud = CloudEngine()+ if cloud.health():+ engine = MultiEngine([(engine_name, engine), ("cloud", cloud)])+ engine_name = "multi"+ console.print(" Cloud: [cyan]enabled[/cyan] (API keys detected)")+ except Exception as exc:+ logger.debug("Cloud engine init failed: %s", exc)+# Wrap engine with InstrumentedEngine for telemetry recordingtry:from openjarvis.telemetry.instrumented_engine import InstrumentedEnginediff --git a/src/openjarvis/engine/cloud.py b/src/openjarvis/engine/cloud.pyindex 22f94fe4..cacf651f 100644--- a/src/openjarvis/engine/cloud.py+++ b/src/openjarvis/engine/cloud.py@@ -63,13 +63,29 @@ _GOOGLE_MODELS = ["gemini-3-flash-preview",]+# OpenRouter models — prefixed with "openrouter/" so they can be identified+_OPENROUTER_POPULAR = [+ "openrouter/auto",+ "openrouter/openai/gpt-4o",+ "openrouter/anthropic/claude-sonnet-4",+ "openrouter/google/gemini-2.5-pro",+ "openrouter/meta-llama/llama-3.3-70b-instruct",+ "openrouter/mistralai/mistral-large",+ "openrouter/deepseek/deepseek-r1",+ "openrouter/qwen/qwen3-235b-a22b",+]+++def _is_openrouter_model(model: str) -> bool:+ return model.startswith("openrouter/")+def _is_anthropic_model(model: str) -> bool:- return "claude" in model.lower()+ return "claude" in model.lower() and not _is_openrouter_model(model)def _is_google_model(model: str) -> bool:- return "gemini" in model.lower()+ return "gemini" in model.lower() and not _is_openrouter_model(model)def _is_openai_reasoning_model(model: str) -> bool:@@ -137,6 +153,7 @@ class CloudEngine(InferenceEngine):self._openai_client: Any = Noneself._anthropic_client: Any = Noneself._google_client: Any = None+ self._openrouter_client: Any = Noneself._init_clients()def _init_clients(self) -> None:@@ -162,6 +179,16 @@ class CloudEngine(InferenceEngine):self._google_client = genai.Client(api_key=gemini_key)except ImportError:pass+ openrouter_key = os.environ.get("OPENROUTER_API_KEY")+ if openrouter_key:+ try:+ import openai+ self._openrouter_client = openai.OpenAI(+ base_url="https://openrouter.ai/api/v1",+ api_key=openrouter_key,+ )+ except ImportError:+ passdef _generate_openai(self,@@ -533,6 +560,47 @@ class CloudEngine(InferenceEngine):return result+ def _generate_openrouter(+ self,+ messages: Sequence[Message],+ *,+ model: str,+ temperature: float,+ max_tokens: int,+ **kwargs: Any,+ ) -> Dict[str, Any]:+ if self._openrouter_client is None:+ raise EngineConnectionError(+ "OpenRouter client not available — set OPENROUTER_API_KEY"+ )+ # Strip the "openrouter/" prefix to get the actual model ID+ actual_model = model.removeprefix("openrouter/")+ kwargs.pop("response_format", None)+ create_kwargs: Dict[str, Any] = {+ "model": actual_model,+ "messages": messages_to_dicts(messages),+ "max_tokens": max_tokens,+ "temperature": temperature,+ }+ t0 = time.monotonic()+ resp = self._openrouter_client.chat.completions.create(**create_kwargs)+ elapsed = time.monotonic() - t0+ choice = resp.choices[0]+ usage = resp.usage+ prompt_tokens = usage.prompt_tokens if usage else 0+ completion_tokens = usage.completion_tokens if usage else 0+ return {+ "content": choice.message.content or "",+ "usage": {+ "prompt_tokens": prompt_tokens,+ "completion_tokens": completion_tokens,+ "total_tokens": (usage.total_tokens if usage else 0),+ },+ "model": resp.model,+ "finish_reason": choice.finish_reason or "stop",+ "ttft": elapsed,+ }+def generate(self,messages: Sequence[Message],@@ -548,6 +616,8 @@ class CloudEngine(InferenceEngine):max_tokens=max_tokens,**kwargs,)+ if _is_openrouter_model(model):+ return self._generate_openrouter(messages, **kw)if _is_anthropic_model(model):return self._generate_anthropic(messages, **kw)if _is_google_model(model):@@ -569,7 +639,12 @@ class CloudEngine(InferenceEngine):max_tokens=max_tokens,**kwargs,)- if _is_anthropic_model(model):+ if _is_openrouter_model(model):+ async for token in self._stream_openrouter(+ messages, **kw+ ):+ yield token+ elif _is_anthropic_model(model):async for token in self._stream_anthropic(messages, **kw):@@ -679,6 +754,31 @@ class CloudEngine(InferenceEngine):if chunk.text:yield chunk.text+ async def _stream_openrouter(+ self,+ messages: Sequence[Message],+ *,+ model: str,+ temperature: float,+ max_tokens: int,+ **kwargs: Any,+ ) -> AsyncIterator[str]:+ if self._openrouter_client is None:+ raise EngineConnectionError("OpenRouter client not available")+ actual_model = model.removeprefix("openrouter/")+ create_kwargs: Dict[str, Any] = {+ "model": actual_model,+ "messages": messages_to_dicts(messages),+ "max_tokens": max_tokens,+ "temperature": temperature,+ "stream": True,+ }+ resp = self._openrouter_client.chat.completions.create(**create_kwargs)+ for chunk in resp:+ delta = chunk.choices[0].delta if chunk.choices else None+ if delta and delta.content:+ yield delta.content+def list_models(self) -> List[str]:models: List[str] = []if self._openai_client is not None:@@ -687,6 +787,8 @@ class CloudEngine(InferenceEngine):models.extend(_ANTHROPIC_MODELS)if self._google_client is not None:models.extend(_GOOGLE_MODELS)+ if self._openrouter_client is not None:+ models.extend(_OPENROUTER_POPULAR)return modelsdef health(self) -> bool:@@ -694,6 +796,7 @@ class CloudEngine(InferenceEngine):self._openai_client is not Noneor self._anthropic_client is not Noneor self._google_client is not None+ or self._openrouter_client is not None)def close(self) -> None:@@ -707,6 +810,10 @@ class CloudEngine(InferenceEngine):self._anthropic_client = Noneif self._google_client is not None:self._google_client = None+ if self._openrouter_client is not None:+ if hasattr(self._openrouter_client, "close"):+ self._openrouter_client.close()+ self._openrouter_client = None__all__ = ["CloudEngine", "PRICING", "estimate_cost"]diff --git a/src/openjarvis/engine/multi.py b/src/openjarvis/engine/multi.pynew file mode 100644index 00000000..810a811b--- /dev/null+++ b/src/openjarvis/engine/multi.py@@ -0,0 +1,92 @@+"""Multi-engine wrapper — routes requests to the right backend by model name."""++from __future__ import annotations++import logging+from collections.abc import AsyncIterator, Sequence+from typing import Any, Dict, List++from openjarvis.engine._base import InferenceEngine++logger = logging.getLogger(__name__)+++class MultiEngine(InferenceEngine):+ """Wraps multiple engines and routes by model name.++ Models from each engine are discovered via ``list_models()``.+ When ``generate()`` or ``stream()`` is called, the model name+ is looked up to find which engine owns it.+ """++ engine_id = "multi"++ def __init__(self, engines: list[tuple[str, InferenceEngine]]) -> None:+ self._engines = engines+ self._model_map: Dict[str, InferenceEngine] = {}+ self._refresh_map()++ def _refresh_map(self) -> None:+ self._model_map.clear()+ for _key, engine in self._engines:+ try:+ for model_id in engine.list_models():+ self._model_map[model_id] = engine+ except Exception as exc:+ logger.debug("Failed to list models for %s: %s", _key, exc)++ def _engine_for(self, model: str) -> InferenceEngine:+ """Find the engine that owns a model, refreshing the map once if needed."""+ engine = self._model_map.get(model)+ if engine is not None:+ return engine+ # Refresh and retry (a new model may have been pulled)+ self._refresh_map()+ engine = self._model_map.get(model)+ if engine is not None:+ return engine+ # Fall back to the first engine (usually Ollama)+ return self._engines[0][1]++ def generate(+ self,+ messages: Sequence[Any],+ *,+ model: str,+ temperature: float = 0.7,+ max_tokens: int = 1024,+ **kwargs: Any,+ ) -> Dict[str, Any]:+ return self._engine_for(model).generate(+ messages, model=model, temperature=temperature,+ max_tokens=max_tokens, **kwargs,+ )++ async def stream(+ self,+ messages: Sequence[Any],+ *,+ model: str,+ temperature: float = 0.7,+ max_tokens: int = 1024,+ **kwargs: Any,+ ) -> AsyncIterator[str]:+ async for token in self._engine_for(model).stream(+ messages, model=model, temperature=temperature,+ max_tokens=max_tokens, **kwargs,+ ):+ yield token++ def list_models(self) -> List[str]:+ self._refresh_map()+ return list(self._model_map.keys())++ def health(self) -> bool:+ return any(engine.health() for _key, engine in self._engines)++ def close(self) -> None:+ for _key, engine in self._engines:+ engine.close()+++__all__ = ["MultiEngine"]