Skip to content

Personal AI, On Personal Devices

Python

make cloud models on desktop app work

mrTSBf8c37fe85ee4
diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs
index 0f2d6c1c..6fa1a44a 100644
--- a/desktop/src-tauri/src/lib.rs
+++ b/desktop/src-tauri/src/lib.rs
@@ -649,7 +649,12 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
s.detail = "Installing dependencies...".into();
}
let _ = tokio::process::Command::new(&uv_bin)
- .args(["sync", "--extra", "server"])
+ .args([
+ "sync",
+ "--extra", "server",
+ "--extra", "inference-cloud",
+ "--extra", "inference-google",
+ ])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.current_dir(root)
@@ -1124,6 +1129,15 @@ async fn save_cloud_key(key_name: String, key_value: String) -> Result<(), Strin
let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600));
}
+ // Tell the running server to hot-reload its cloud engine so the user
+ // doesn't need to restart the app after entering an API key.
+ let reload_url = format!("http://127.0.0.1:{}/v1/cloud/reload", JARVIS_PORT);
+ let _ = reqwest::Client::new()
+ .post(&reload_url)
+ .timeout(std::time::Duration::from_secs(10))
+ .send()
+ .await;
+
Ok(())
}
diff --git a/frontend/src/components/CommandPalette.tsx b/frontend/src/components/CommandPalette.tsx
index 2d110290..743fe823 100644
--- a/frontend/src/components/CommandPalette.tsx
+++ b/frontend/src/components/CommandPalette.tsx
@@ -223,8 +223,11 @@ export function CommandPalette() {
useAppStore.getState().addLogEntry({
timestamp: Date.now(), level: 'info', category: 'model',
- message: `${provider.name} API key ${value ? 'saved' : 'removed'}. Restart the app for cloud models to appear.`,
+ message: `${provider.name} API key ${value ? 'saved' : 'removed'}. Refreshing model list…`,
});
+
+ // Refresh the model list so cloud models appear immediately.
+ await refreshModels();
};
const handleKeyDown = (e: React.KeyboardEvent) => {
diff --git a/src/openjarvis/cli/serve.py b/src/openjarvis/cli/serve.py
index a236e6b5..066d6ecc 100644
--- a/src/openjarvis/cli/serve.py
+++ b/src/openjarvis/cli/serve.py
@@ -120,10 +120,15 @@ def serve(
from openjarvis.engine.multi import MultiEngine
cloud = CloudEngine()
+ engine = MultiEngine([(engine_name, engine), ("cloud", cloud)])
+ engine_name = "multi"
if cloud.health():
- engine = MultiEngine([(engine_name, engine), ("cloud", cloud)])
- engine_name = "multi"
console.print(" Cloud: [cyan]enabled[/cyan] (API keys detected)")
+ else:
+ console.print(
+ " Cloud: [yellow]keys set but packages missing[/yellow] "
+ "(run: uv sync --extra inference-cloud --extra inference-google)"
+ )
except Exception as exc:
logger.debug("Cloud engine init failed: %s", exc)
diff --git a/src/openjarvis/server/cloud_router.py b/src/openjarvis/server/cloud_router.py
new file mode 100644
index 00000000..c90ccf59
--- /dev/null
+++ b/src/openjarvis/server/cloud_router.py
@@ -0,0 +1,324 @@
+"""Direct cloud API router — bypasses the engine system entirely.
+
+Reads API keys from ~/.openjarvis/cloud-keys.env at request time so
+it works even when the server was started without cloud keys in its
+environment. Uses httpx directly so no cloud SDK packages are required.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import AsyncIterator
+from pathlib import Path
+from typing import Any, Sequence
+
+import httpx
+
+from openjarvis.core.types import Message
+
+# ---------------------------------------------------------------------------
+# Key / provider detection
+# ---------------------------------------------------------------------------
+
+_CLOUD_ENV_FILE = Path.home() / ".openjarvis" / "cloud-keys.env"
+
+_OPENAI_PREFIXES = ("gpt-", "o1-", "o3-", "o4-", "chatgpt-")
+_ANTHROPIC_PREFIXES = ("claude-",)
+_GOOGLE_PREFIXES = ("gemini-",)
+_MINIMAX_PREFIXES = ("MiniMax-",)
+
+
+def _load_keys() -> dict[str, str]:
+ """Read cloud-keys.env from disk every call so live updates are picked up."""
+ keys: dict[str, str] = {}
+ # File first, then fall back to process environment
+ if _CLOUD_ENV_FILE.exists():
+ for raw in _CLOUD_ENV_FILE.read_text().splitlines():
+ line = raw.strip()
+ if line and not line.startswith("#") and "=" in line:
+ k, v = line.split("=", 1)
+ keys[k.strip()] = v.strip()
+ # Process env can override (e.g. during testing)
+ for name in (
+ "OPENAI_API_KEY",
+ "ANTHROPIC_API_KEY",
+ "GEMINI_API_KEY",
+ "GOOGLE_API_KEY",
+ "OPENROUTER_API_KEY",
+ "MINIMAX_API_KEY",
+ ):
+ val = os.environ.get(name)
+ if val:
+ keys[name] = val
+ return keys
+
+
+def get_provider(model: str) -> str | None:
+ """Return the provider for a model name, or None if it's a local model."""
+ if any(model.startswith(p) for p in _OPENAI_PREFIXES):
+ return "openai"
+ if any(model.startswith(p) for p in _ANTHROPIC_PREFIXES):
+ return "anthropic"
+ if any(model.startswith(p) for p in _GOOGLE_PREFIXES):
+ return "google"
+ if any(model.startswith(p) for p in _MINIMAX_PREFIXES):
+ return "minimax"
+ if "/" in model: # openrouter format: "meta-llama/llama-3-8b"
+ return "openrouter"
+ return None
+
+
+def is_cloud_model(model: str) -> bool:
+ return get_provider(model) is not None
+
+
+# ---------------------------------------------------------------------------
+# Message conversion
+# ---------------------------------------------------------------------------
+
+def _to_openai_msgs(messages: Sequence[Message]) -> list[dict[str, Any]]:
+ out = []
+ for m in messages:
+ role = m.role.value if hasattr(m.role, "value") else str(m.role)
+ out.append({"role": role, "content": m.content or ""})
+ return out
+
+
+def _to_anthropic_msgs(
+ messages: Sequence[Message],
+) -> tuple[str, list[dict[str, Any]]]:
+ """Return (system_text, chat_messages) in Anthropic format."""
+ system_text = ""
+ chat: list[dict[str, Any]] = []
+ for m in messages:
+ role = m.role.value if hasattr(m.role, "value") else str(m.role)
+ if role == "system":
+ system_text = m.content or ""
+ else:
+ # Anthropic only allows "user" and "assistant"
+ ar = "user" if role != "assistant" else "assistant"
+ chat.append({"role": ar, "content": m.content or ""})
+ return system_text, chat
+
+
+def _to_google_contents(messages: Sequence[Message]) -> list[dict[str, Any]]:
+ """Convert to Google Gemini content format."""
+ contents = []
+ for m in messages:
+ role = m.role.value if hasattr(m.role, "value") else str(m.role)
+ if role == "system":
+ # Gemini doesn't have a system role in the contents array;
+ # prepend as a user message.
+ contents.append({"role": "user", "parts": [{"text": m.content or ""}]})
+ contents.append({"role": "model", "parts": [{"text": "Understood."}]})
+ elif role == "assistant":
+ contents.append({"role": "model", "parts": [{"text": m.content or ""}]})
+ else:
+ contents.append({"role": "user", "parts": [{"text": m.content or ""}]})
+ return contents
+
+
+# ---------------------------------------------------------------------------
+# Streaming generators
+# ---------------------------------------------------------------------------
+
+async def _stream_openai(
+ model: str,
+ messages: Sequence[Message],
+ temperature: float,
+ max_tokens: int,
+ base_url: str = "https://api.openai.com/v1",
+ api_key_name: str = "OPENAI_API_KEY",
+) -> AsyncIterator[str]:
+ keys = _load_keys()
+ api_key = keys.get(api_key_name, "")
+ if not api_key:
+ raise ValueError(f"{api_key_name} not set — add it in the Cloud Models tab")
+
+ payload = {
+ "model": model,
+ "messages": _to_openai_msgs(messages),
+ "temperature": temperature,
+ "max_tokens": max_tokens,
+ "stream": True,
+ }
+
+ async with httpx.AsyncClient(timeout=180) as client:
+ async with client.stream(
+ "POST",
+ f"{base_url}/chat/completions",
+ json=payload,
+ headers={
+ "Authorization": f"Bearer {api_key}",
+ "Content-Type": "application/json",
+ },
+ ) as resp:
+ resp.raise_for_status()
+ async for line in resp.aiter_lines():
+ if not line.startswith("data: "):
+ continue
+ data = line[6:].strip()
+ if data == "[DONE]":
+ break
+ try:
+ chunk = json.loads(data)
+ delta = chunk["choices"][0]["delta"].get("content") or ""
+ if delta:
+ yield delta
+ except Exception:
+ pass
+
+
+async def _stream_anthropic(
+ model: str,
+ messages: Sequence[Message],
+ temperature: float,
+ max_tokens: int,
+) -> AsyncIterator[str]:
+ keys = _load_keys()
+ api_key = keys.get("ANTHROPIC_API_KEY", "")
+ if not api_key:
+ raise ValueError("ANTHROPIC_API_KEY not set — add it in the Cloud Models tab")
+
+ system_text, chat_msgs = _to_anthropic_msgs(messages)
+ payload: dict[str, Any] = {
+ "model": model,
+ "messages": chat_msgs,
+ "max_tokens": max_tokens,
+ "temperature": temperature,
+ "stream": True,
+ }
+ if system_text:
+ payload["system"] = system_text
+
+ async with httpx.AsyncClient(timeout=180) as client:
+ async with client.stream(
+ "POST",
+ "https://api.anthropic.com/v1/messages",
+ json=payload,
+ headers={
+ "x-api-key": api_key,
+ "anthropic-version": "2023-06-01",
+ "Content-Type": "application/json",
+ },
+ ) as resp:
+ resp.raise_for_status()
+ async for line in resp.aiter_lines():
+ if not line.startswith("data: "):
+ continue
+ data = line[6:].strip()
+ try:
+ event = json.loads(data)
+ if event.get("type") == "content_block_delta":
+ text = event.get("delta", {}).get("text", "")
+ if text:
+ yield text
+ except Exception:
+ pass
+
+
+async def _stream_google(
+ model: str,
+ messages: Sequence[Message],
+ temperature: float,
+ max_tokens: int,
+) -> AsyncIterator[str]:
+ keys = _load_keys()
+ api_key = keys.get("GEMINI_API_KEY") or keys.get("GOOGLE_API_KEY", "")
+ if not api_key:
+ raise ValueError("GEMINI_API_KEY not set — add it in the Cloud Models tab")
+
+ contents = _to_google_contents(messages)
+ payload: dict[str, Any] = {
+ "contents": contents,
+ "generationConfig": {
+ "temperature": temperature,
+ "maxOutputTokens": max_tokens,
+ },
+ }
+
+ url = (
+ f"https://generativelanguage.googleapis.com/v1beta/models/"
+ f"{model}:streamGenerateContent?alt=sse&key={api_key}"
+ )
+
+ async with httpx.AsyncClient(timeout=180) as client:
+ async with client.stream("POST", url, json=payload) as resp:
+ resp.raise_for_status()
+ async for line in resp.aiter_lines():
+ if not line.startswith("data: "):
+ continue
+ data = line[6:].strip()
+ try:
+ chunk = json.loads(data)
+ parts = (
+ chunk.get("candidates", [{}])[0]
+ .get("content", {})
+ .get("parts", [])
+ )
+ for part in parts:
+ text = part.get("text", "")
+ if text:
+ yield text
+ except Exception:
+ pass
+
+
+# ---------------------------------------------------------------------------
+# Public entry point
+# ---------------------------------------------------------------------------
+
+async def stream_cloud(
+ model: str,
+ messages: Sequence[Message],
+ temperature: float = 0.7,
+ max_tokens: int = 1024,
+) -> AsyncIterator[str]:
+ """Stream tokens from a cloud provider for the given model."""
+ provider = get_provider(model)
+
+ if provider == "openai":
+ async for token in _stream_openai(model, messages, temperature, max_tokens):
+ yield token
+
+ elif provider == "anthropic":
+ async for token in _stream_anthropic(model, messages, temperature, max_tokens):
+ yield token
+
+ elif provider == "google":
+ async for token in _stream_google(model, messages, temperature, max_tokens):
+ yield token
+
+ elif provider == "openrouter":
+ keys = _load_keys()
+ api_key = keys.get("OPENROUTER_API_KEY", "")
+ if not api_key:
+ raise ValueError("OPENROUTER_API_KEY not set — add it in the Cloud Models tab")
+ async for token in _stream_openai(
+ model,
+ messages,
+ temperature,
+ max_tokens,
+ base_url="https://openrouter.ai/api/v1",
+ api_key_name="OPENROUTER_API_KEY",
+ ):
+ yield token
+
+ elif provider == "minimax":
+ keys = _load_keys()
+ api_key = keys.get("MINIMAX_API_KEY", "")
+ if not api_key:
+ raise ValueError("MINIMAX_API_KEY not set — add it in the Cloud Models tab")
+ async for token in _stream_openai(
+ model,
+ messages,
+ temperature,
+ max_tokens,
+ base_url="https://api.minimax.io/v1",
+ api_key_name="MINIMAX_API_KEY",
+ ):
+ yield token
+
+ else:
+ raise ValueError(f"Unknown cloud provider for model: {model!r}")
diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py
index a615d7d0..8084bac2 100644
--- a/src/openjarvis/server/routes.py
+++ b/src/openjarvis/server/routes.py
@@ -290,9 +290,15 @@ async def _handle_stream(
complexity_info=None,
):
"""Stream response using SSE format."""
+ from openjarvis.server.cloud_router import is_cloud_model, stream_cloud
+
messages = _to_messages(req.messages)
chunk_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
+ # Choose token source: cloud router bypasses the engine system entirely
+ # so cloud models work regardless of how the server was started.
+ use_cloud = is_cloud_model(model)
+
async def generate():
# Send role chunk first
first_chunk = ChatCompletionChunk(
@@ -308,12 +314,17 @@ async def _handle_stream(
try:
# Stream content
- async for token in engine.stream(
- messages,
- model=model,
- temperature=req.temperature,
- max_tokens=req.max_tokens,
- ):
+ token_iter = (
+ stream_cloud(model, messages, req.temperature, req.max_tokens)
+ if use_cloud
+ else engine.stream(
+ messages,
+ model=model,
+ temperature=req.temperature,
+ max_tokens=req.max_tokens,
+ )
+ )
+ async for token in token_iter:
chunk = ChatCompletionChunk(
id=chunk_id,
model=model,
@@ -392,9 +403,32 @@ async def _handle_stream(
@router.get("/v1/models")
async def list_models(request: Request) -> ModelListResponse:
- """List available models from the engine."""
+ """List available models from the engine plus any configured cloud models."""
+ from openjarvis.server.cloud_router import _load_keys
+ from openjarvis.engine.cloud import (
+ _OPENAI_MODELS,
+ _ANTHROPIC_MODELS,
+ _GOOGLE_MODELS,
+ _OPENROUTER_POPULAR,
+ _MINIMAX_MODELS,
+ )
+
engine = request.app.state.engine
- model_ids = engine.list_models()
+ model_ids: list[str] = list(engine.list_models())
+
+ # Append cloud models for any provider whose key is present on disk.
+ keys = _load_keys()
+ if keys.get("OPENAI_API_KEY"):
+ model_ids.extend(m for m in _OPENAI_MODELS if m not in model_ids)
+ if keys.get("ANTHROPIC_API_KEY"):
+ model_ids.extend(m for m in _ANTHROPIC_MODELS if m not in model_ids)
+ if keys.get("GEMINI_API_KEY") or keys.get("GOOGLE_API_KEY"):
+ model_ids.extend(m for m in _GOOGLE_MODELS if m not in model_ids)
+ if keys.get("OPENROUTER_API_KEY"):
+ model_ids.extend(m for m in _OPENROUTER_POPULAR if m not in model_ids)
+ if keys.get("MINIMAX_API_KEY"):
+ model_ids.extend(m for m in _MINIMAX_MODELS if m not in model_ids)
+
return ModelListResponse(
data=[ModelObject(id=mid) for mid in model_ids],
)
@@ -472,6 +506,60 @@ async def delete_model(model_name: str, request: Request):
return {"status": "deleted", "model": model_name}
[email protected]("/v1/cloud/reload")
+async def reload_cloud_engine(request: Request):
+ """Hot-reload cloud API keys and (re-)initialize the cloud engine.
+
+ Called by the desktop app immediately after the user saves a cloud API
+ key so that cloud models become available without a full app restart.
+ """
+ import os
+ from pathlib import Path
+
+ # Re-read ~/.openjarvis/cloud-keys.env and update the running process env.
+ keys_path = Path.home() / ".openjarvis" / "cloud-keys.env"
+ if keys_path.exists():
+ for raw_line in keys_path.read_text().splitlines():
+ line = raw_line.strip()
+ if line and not line.startswith("#") and "=" in line:
+ k, v = line.split("=", 1)
+ os.environ[k.strip()] = v.strip()
+
+ # Try to build a fresh CloudEngine.
+ try:
+ from openjarvis.engine.cloud import CloudEngine
+ from openjarvis.engine.multi import MultiEngine
+
+ cloud = CloudEngine()
+ if not cloud.health():
+ return {"status": "no_cloud", "message": "No cloud models available (check API keys)"}
+ except Exception as exc:
+ return {"status": "error", "message": str(exc)}
+
+ # Locate the innermost engine, working through InstrumentedEngine layers.
+ outer = request.app.state.engine
+ inner = getattr(outer, "_inner", outer)
+
+ if isinstance(inner, MultiEngine):
+ # Replace or insert the cloud entry in the existing MultiEngine.
+ new_engines = [(k, e) for k, e in inner._engines if k != "cloud"]
+ new_engines.append(("cloud", cloud))
+ inner._engines = new_engines
+ inner._refresh_map()
+ else:
+ # Wrap the existing engine (which may be security-wrapped) with a new
+ # MultiEngine that includes the cloud engine.
+ engine_name = getattr(request.app.state, "engine_name", "local")
+ new_multi = MultiEngine([(engine_name, inner), ("cloud", cloud)])
+ if hasattr(outer, "_inner"):
+ outer._inner = new_multi
+ else:
+ request.app.state.engine = new_multi
+ request.app.state.engine_name = "multi"
+
+ return {"status": "ok", "message": "Cloud engine reloaded"}
+
+
@router.get("/v1/savings")
async def savings(request: Request):
"""Return savings summary compared to cloud providers.