Skip to content

Personal AI, On Personal Devices

Python

fix local

mrTSB2feaa10d97e2
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index 55621bd6..9be233d3 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -46,6 +46,9 @@
"typescript": "~5.7.0",
"vite": "^6.0.0",
"vite-plugin-pwa": "^1.2.0"
+ },
+ "engines": {
+ "node": ">=20"
}
},
"node_modules/@apideck/better-ajv-errors": {
diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsx
index c395e71c..ed63f845 100644
--- a/frontend/src/components/Chat/InputArea.tsx
+++ b/frontend/src/components/Chat/InputArea.tsx
@@ -243,9 +243,10 @@ export function InputArea() {
accumulatedContent = 'No response was generated. Please try again.';
}
const totalMs = Date.now() - startTime;
- const serverInfo = useAppStore.getState().serverInfo;
+ const _CLOUD_PREFIXES = ['gpt-', 'o1-', 'o3-', 'o4-', 'claude-', 'gemini-', 'openrouter/', 'MiniMax-', 'chatgpt-'];
+ const engineLabel = _CLOUD_PREFIXES.some(p => selectedModel.startsWith(p)) ? 'cloud' : 'ollama';
const telemetry: MessageTelemetry = {
- engine: serverInfo?.engine,
+ engine: engineLabel,
model_id: selectedModel,
total_ms: totalMs,
ttft_ms: ttftMs,
diff --git a/src/openjarvis/server/cloud_router.py b/src/openjarvis/server/cloud_router.py
index c90ccf59..1e8530d6 100644
--- a/src/openjarvis/server/cloud_router.py
+++ b/src/openjarvis/server/cloud_router.py
@@ -265,6 +265,64 @@ async def _stream_google(
pass
+# ---------------------------------------------------------------------------
+# Local (Ollama) direct streaming — bypasses engine routing entirely
+# ---------------------------------------------------------------------------
+
+def _ollama_host() -> str:
+ return os.environ.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/")
+
+
+async def stream_local(
+ model: str,
+ messages: Sequence[Message],
+ temperature: float = 0.7,
+ max_tokens: int = 1024,
+) -> AsyncIterator[str]:
+ """Stream tokens directly from Ollama, bypassing the engine system."""
+ payload = {
+ "model": model,
+ "messages": _to_openai_msgs(messages),
+ "stream": True,
+ # Disable extended thinking (Qwen3.5 etc.) — when enabled all tokens
+ # go into the 'thinking' field and 'content' stays empty.
+ "think": False,
+ "options": {
+ "temperature": temperature,
+ "num_predict": max_tokens,
+ },
+ }
+ host = _ollama_host()
+ async with httpx.AsyncClient(timeout=300) as client:
+ async with client.stream("POST", f"{host}/api/chat", json=payload) as resp:
+ resp.raise_for_status()
+ async for line in resp.aiter_lines():
+ if not line:
+ continue
+ try:
+ data = json.loads(line)
+ token = data.get("message", {}).get("content", "")
+ if token:
+ yield token
+ if data.get("done"):
+ break
+ except Exception:
+ pass
+
+
+async def list_local_models() -> list[str]:
+ """Return Ollama model names directly from the Ollama API."""
+ host = _ollama_host()
+ try:
+ async with httpx.AsyncClient(timeout=10) as client:
+ resp = await client.get(f"{host}/api/tags")
+ resp.raise_for_status()
+ data = resp.json()
+ return [m["name"] for m in data.get("models", [])]
+ except Exception:
+ return []
+
+
# ---------------------------------------------------------------------------
# Public entry point
# ---------------------------------------------------------------------------
diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py
index 8084bac2..133e991e 100644
--- a/src/openjarvis/server/routes.py
+++ b/src/openjarvis/server/routes.py
@@ -290,13 +290,13 @@ async def _handle_stream(
complexity_info=None,
):
"""Stream response using SSE format."""
- from openjarvis.server.cloud_router import is_cloud_model, stream_cloud
+ from openjarvis.server.cloud_router import is_cloud_model, stream_cloud, stream_local
messages = _to_messages(req.messages)
chunk_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
- # Choose token source: cloud router bypasses the engine system entirely
- # so cloud models work regardless of how the server was started.
+ # Route directly to the right backend — bypasses engine routing entirely
+ # so broken MultiEngine state can never misdirect requests.
use_cloud = is_cloud_model(model)
async def generate():
@@ -313,17 +313,12 @@ async def _handle_stream(
yield f"data: {first_chunk.model_dump_json()}\n\n"
try:
- # Stream content
- token_iter = (
- stream_cloud(model, messages, req.temperature, req.max_tokens)
- if use_cloud
- else engine.stream(
- messages,
- model=model,
- temperature=req.temperature,
- max_tokens=req.max_tokens,
- )
- )
+ # Cloud models → direct cloud API; local models → direct Ollama.
+ # Both paths bypass the engine/MultiEngine routing system.
+ if use_cloud:
+ token_iter = stream_cloud(model, messages, req.temperature, req.max_tokens)
+ else:
+ token_iter = stream_local(model, messages, req.temperature, req.max_tokens)
async for token in token_iter:
chunk = ChatCompletionChunk(
id=chunk_id,
@@ -376,17 +371,11 @@ async def _handle_stream(
)
finish_dict = _json.loads(finish_data.model_dump_json())
- # Pull usage from the engine if it tracked it during streaming
- raw_engine = engine
- # Unwrap InstrumentedEngine if present
- if hasattr(raw_engine, "_inner"):
- raw_engine = raw_engine._inner
- # Unwrap MultiEngine if present
- if hasattr(raw_engine, "_engine_for"):
- raw_engine = raw_engine._engine_for(model)
- stream_usage = getattr(raw_engine, "_last_stream_usage", None)
- if isinstance(stream_usage, dict) and stream_usage.get("total_tokens", 0) > 0:
- finish_dict["usage"] = stream_usage
+ # Tag the finish chunk with the correct engine label.
+ # We use the routing decision (use_cloud) directly rather than
+ # unwrapping the engine chain, which can be in a broken state.
+ finish_dict.setdefault("telemetry", {})
+ finish_dict["telemetry"]["engine"] = "cloud" if use_cloud else "ollama"
if complexity_info is not None:
finish_dict["complexity"] = complexity_info.model_dump()
@@ -403,31 +392,21 @@ async def _handle_stream(
@router.get("/v1/models")
async def list_models(request: Request) -> ModelListResponse:
- """List available models from the engine plus any configured cloud models."""
- from openjarvis.server.cloud_router import _load_keys
- from openjarvis.engine.cloud import (
- _OPENAI_MODELS,
- _ANTHROPIC_MODELS,
- _GOOGLE_MODELS,
- _OPENROUTER_POPULAR,
- _MINIMAX_MODELS,
- )
+ """List locally installed models (Ollama).
- engine = request.app.state.engine
- model_ids: list[str] = list(engine.list_models())
-
- # Append cloud models for any provider whose key is present on disk.
- keys = _load_keys()
- if keys.get("OPENAI_API_KEY"):
- model_ids.extend(m for m in _OPENAI_MODELS if m not in model_ids)
- if keys.get("ANTHROPIC_API_KEY"):
- model_ids.extend(m for m in _ANTHROPIC_MODELS if m not in model_ids)
- if keys.get("GEMINI_API_KEY") or keys.get("GOOGLE_API_KEY"):
- model_ids.extend(m for m in _GOOGLE_MODELS if m not in model_ids)
- if keys.get("OPENROUTER_API_KEY"):
- model_ids.extend(m for m in _OPENROUTER_POPULAR if m not in model_ids)
- if keys.get("MINIMAX_API_KEY"):
- model_ids.extend(m for m in _MINIMAX_MODELS if m not in model_ids)
+ Cloud models are not included here — they live in the Cloud Models tab
+ of the UI and are selected there, not from this endpoint.
+ """
+ from openjarvis.server.cloud_router import list_local_models
+
+ model_ids = await list_local_models()
+
+ # Fall back to engine.list_models() if Ollama direct call fails
+ if not model_ids:
+ engine = request.app.state.engine
+ all_ids = engine.list_models()
+ from openjarvis.server.cloud_router import is_cloud_model
+ model_ids = [m for m in all_ids if not is_cloud_model(m)]
return ModelListResponse(
data=[ModelObject(id=mid) for mid in model_ids],