Skip to content

Personal AI, On Personal Devices

Python

feat: cloud model support — OpenAI, Anthropic, Google, OpenRouter in dropdown

Jon Saad-Falcon0dac51c649cf
diff --git a/src/openjarvis/cli/serve.py b/src/openjarvis/cli/serve.py
index 1a9e21d0..9cc81705 100644
--- a/src/openjarvis/cli/serve.py
+++ b/src/openjarvis/cli/serve.py
@@ -92,6 +92,29 @@ def serve(
engine_name, engine = resolved
+ # If cloud API keys are set, wrap with MultiEngine so both local
+ # and cloud models appear in the model list and can be used.
+ import os
+ _has_cloud = (
+ os.environ.get("OPENAI_API_KEY")
+ or os.environ.get("ANTHROPIC_API_KEY")
+ or os.environ.get("GEMINI_API_KEY")
+ or os.environ.get("GOOGLE_API_KEY")
+ or os.environ.get("OPENROUTER_API_KEY")
+ )
+ if _has_cloud and engine_name != "cloud":
+ try:
+ from openjarvis.engine.cloud import CloudEngine
+ from openjarvis.engine.multi import MultiEngine
+
+ cloud = CloudEngine()
+ if cloud.health():
+ engine = MultiEngine([(engine_name, engine), ("cloud", cloud)])
+ engine_name = "multi"
+ console.print(" Cloud: [cyan]enabled[/cyan] (API keys detected)")
+ except Exception as exc:
+ logger.debug("Cloud engine init failed: %s", exc)
+
# Wrap engine with InstrumentedEngine for telemetry recording
try:
from openjarvis.telemetry.instrumented_engine import InstrumentedEngine
diff --git a/src/openjarvis/engine/cloud.py b/src/openjarvis/engine/cloud.py
index 22f94fe4..cacf651f 100644
--- a/src/openjarvis/engine/cloud.py
+++ b/src/openjarvis/engine/cloud.py
@@ -63,13 +63,29 @@ _GOOGLE_MODELS = [
"gemini-3-flash-preview",
]
+# OpenRouter models — prefixed with "openrouter/" so they can be identified
+_OPENROUTER_POPULAR = [
+ "openrouter/auto",
+ "openrouter/openai/gpt-4o",
+ "openrouter/anthropic/claude-sonnet-4",
+ "openrouter/google/gemini-2.5-pro",
+ "openrouter/meta-llama/llama-3.3-70b-instruct",
+ "openrouter/mistralai/mistral-large",
+ "openrouter/deepseek/deepseek-r1",
+ "openrouter/qwen/qwen3-235b-a22b",
+]
+
+
+def _is_openrouter_model(model: str) -> bool:
+ return model.startswith("openrouter/")
+
def _is_anthropic_model(model: str) -> bool:
- return "claude" in model.lower()
+ return "claude" in model.lower() and not _is_openrouter_model(model)
def _is_google_model(model: str) -> bool:
- return "gemini" in model.lower()
+ return "gemini" in model.lower() and not _is_openrouter_model(model)
def _is_openai_reasoning_model(model: str) -> bool:
@@ -137,6 +153,7 @@ class CloudEngine(InferenceEngine):
self._openai_client: Any = None
self._anthropic_client: Any = None
self._google_client: Any = None
+ self._openrouter_client: Any = None
self._init_clients()
def _init_clients(self) -> None:
@@ -162,6 +179,16 @@ class CloudEngine(InferenceEngine):
self._google_client = genai.Client(api_key=gemini_key)
except ImportError:
pass
+ openrouter_key = os.environ.get("OPENROUTER_API_KEY")
+ if openrouter_key:
+ try:
+ import openai
+ self._openrouter_client = openai.OpenAI(
+ base_url="https://openrouter.ai/api/v1",
+ api_key=openrouter_key,
+ )
+ except ImportError:
+ pass
def _generate_openai(
self,
@@ -533,6 +560,47 @@ class CloudEngine(InferenceEngine):
return result
+ def _generate_openrouter(
+ self,
+ messages: Sequence[Message],
+ *,
+ model: str,
+ temperature: float,
+ max_tokens: int,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ if self._openrouter_client is None:
+ raise EngineConnectionError(
+ "OpenRouter client not available — set OPENROUTER_API_KEY"
+ )
+ # Strip the "openrouter/" prefix to get the actual model ID
+ actual_model = model.removeprefix("openrouter/")
+ kwargs.pop("response_format", None)
+ create_kwargs: Dict[str, Any] = {
+ "model": actual_model,
+ "messages": messages_to_dicts(messages),
+ "max_tokens": max_tokens,
+ "temperature": temperature,
+ }
+ t0 = time.monotonic()
+ resp = self._openrouter_client.chat.completions.create(**create_kwargs)
+ elapsed = time.monotonic() - t0
+ choice = resp.choices[0]
+ usage = resp.usage
+ prompt_tokens = usage.prompt_tokens if usage else 0
+ completion_tokens = usage.completion_tokens if usage else 0
+ return {
+ "content": choice.message.content or "",
+ "usage": {
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": (usage.total_tokens if usage else 0),
+ },
+ "model": resp.model,
+ "finish_reason": choice.finish_reason or "stop",
+ "ttft": elapsed,
+ }
+
def generate(
self,
messages: Sequence[Message],
@@ -548,6 +616,8 @@ class CloudEngine(InferenceEngine):
max_tokens=max_tokens,
**kwargs,
)
+ if _is_openrouter_model(model):
+ return self._generate_openrouter(messages, **kw)
if _is_anthropic_model(model):
return self._generate_anthropic(messages, **kw)
if _is_google_model(model):
@@ -569,7 +639,12 @@ class CloudEngine(InferenceEngine):
max_tokens=max_tokens,
**kwargs,
)
- if _is_anthropic_model(model):
+ if _is_openrouter_model(model):
+ async for token in self._stream_openrouter(
+ messages, **kw
+ ):
+ yield token
+ elif _is_anthropic_model(model):
async for token in self._stream_anthropic(
messages, **kw
):
@@ -679,6 +754,31 @@ class CloudEngine(InferenceEngine):
if chunk.text:
yield chunk.text
+ async def _stream_openrouter(
+ self,
+ messages: Sequence[Message],
+ *,
+ model: str,
+ temperature: float,
+ max_tokens: int,
+ **kwargs: Any,
+ ) -> AsyncIterator[str]:
+ if self._openrouter_client is None:
+ raise EngineConnectionError("OpenRouter client not available")
+ actual_model = model.removeprefix("openrouter/")
+ create_kwargs: Dict[str, Any] = {
+ "model": actual_model,
+ "messages": messages_to_dicts(messages),
+ "max_tokens": max_tokens,
+ "temperature": temperature,
+ "stream": True,
+ }
+ resp = self._openrouter_client.chat.completions.create(**create_kwargs)
+ for chunk in resp:
+ delta = chunk.choices[0].delta if chunk.choices else None
+ if delta and delta.content:
+ yield delta.content
+
def list_models(self) -> List[str]:
models: List[str] = []
if self._openai_client is not None:
@@ -687,6 +787,8 @@ class CloudEngine(InferenceEngine):
models.extend(_ANTHROPIC_MODELS)
if self._google_client is not None:
models.extend(_GOOGLE_MODELS)
+ if self._openrouter_client is not None:
+ models.extend(_OPENROUTER_POPULAR)
return models
def health(self) -> bool:
@@ -694,6 +796,7 @@ class CloudEngine(InferenceEngine):
self._openai_client is not None
or self._anthropic_client is not None
or self._google_client is not None
+ or self._openrouter_client is not None
)
def close(self) -> None:
@@ -707,6 +810,10 @@ class CloudEngine(InferenceEngine):
self._anthropic_client = None
if self._google_client is not None:
self._google_client = None
+ if self._openrouter_client is not None:
+ if hasattr(self._openrouter_client, "close"):
+ self._openrouter_client.close()
+ self._openrouter_client = None
__all__ = ["CloudEngine", "PRICING", "estimate_cost"]
diff --git a/src/openjarvis/engine/multi.py b/src/openjarvis/engine/multi.py
new file mode 100644
index 00000000..810a811b
--- /dev/null
+++ b/src/openjarvis/engine/multi.py
@@ -0,0 +1,92 @@
+"""Multi-engine wrapper — routes requests to the right backend by model name."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import AsyncIterator, Sequence
+from typing import Any, Dict, List
+
+from openjarvis.engine._base import InferenceEngine
+
+logger = logging.getLogger(__name__)
+
+
+class MultiEngine(InferenceEngine):
+ """Wraps multiple engines and routes by model name.
+
+ Models from each engine are discovered via ``list_models()``.
+ When ``generate()`` or ``stream()`` is called, the model name
+ is looked up to find which engine owns it.
+ """
+
+ engine_id = "multi"
+
+ def __init__(self, engines: list[tuple[str, InferenceEngine]]) -> None:
+ self._engines = engines
+ self._model_map: Dict[str, InferenceEngine] = {}
+ self._refresh_map()
+
+ def _refresh_map(self) -> None:
+ self._model_map.clear()
+ for _key, engine in self._engines:
+ try:
+ for model_id in engine.list_models():
+ self._model_map[model_id] = engine
+ except Exception as exc:
+ logger.debug("Failed to list models for %s: %s", _key, exc)
+
+ def _engine_for(self, model: str) -> InferenceEngine:
+ """Find the engine that owns a model, refreshing the map once if needed."""
+ engine = self._model_map.get(model)
+ if engine is not None:
+ return engine
+ # Refresh and retry (a new model may have been pulled)
+ self._refresh_map()
+ engine = self._model_map.get(model)
+ if engine is not None:
+ return engine
+ # Fall back to the first engine (usually Ollama)
+ return self._engines[0][1]
+
+ def generate(
+ self,
+ messages: Sequence[Any],
+ *,
+ model: str,
+ temperature: float = 0.7,
+ max_tokens: int = 1024,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ return self._engine_for(model).generate(
+ messages, model=model, temperature=temperature,
+ max_tokens=max_tokens, **kwargs,
+ )
+
+ async def stream(
+ self,
+ messages: Sequence[Any],
+ *,
+ model: str,
+ temperature: float = 0.7,
+ max_tokens: int = 1024,
+ **kwargs: Any,
+ ) -> AsyncIterator[str]:
+ async for token in self._engine_for(model).stream(
+ messages, model=model, temperature=temperature,
+ max_tokens=max_tokens, **kwargs,
+ ):
+ yield token
+
+ def list_models(self) -> List[str]:
+ self._refresh_map()
+ return list(self._model_map.keys())
+
+ def health(self) -> bool:
+ return any(engine.health() for _key, engine in self._engines)
+
+ def close(self) -> None:
+ for _key, engine in self._engines:
+ engine.close()
+
+
+__all__ = ["MultiEngine"]