Personal AI, On Personal Devices
Python
fix local
mrTSB2feaa10d97e2
diff --git a/frontend/package-lock.json b/frontend/package-lock.jsonindex 55621bd6..9be233d3 100644--- a/frontend/package-lock.json+++ b/frontend/package-lock.json@@ -46,6 +46,9 @@"typescript": "~5.7.0","vite": "^6.0.0","vite-plugin-pwa": "^1.2.0"+ },+ "engines": {+ "node": ">=20"}},"node_modules/@apideck/better-ajv-errors": {diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsxindex c395e71c..ed63f845 100644--- a/frontend/src/components/Chat/InputArea.tsx+++ b/frontend/src/components/Chat/InputArea.tsx@@ -243,9 +243,10 @@ export function InputArea() {accumulatedContent = 'No response was generated. Please try again.';}const totalMs = Date.now() - startTime;- const serverInfo = useAppStore.getState().serverInfo;+ const _CLOUD_PREFIXES = ['gpt-', 'o1-', 'o3-', 'o4-', 'claude-', 'gemini-', 'openrouter/', 'MiniMax-', 'chatgpt-'];+ const engineLabel = _CLOUD_PREFIXES.some(p => selectedModel.startsWith(p)) ? 'cloud' : 'ollama';const telemetry: MessageTelemetry = {- engine: serverInfo?.engine,+ engine: engineLabel,model_id: selectedModel,total_ms: totalMs,ttft_ms: ttftMs,diff --git a/src/openjarvis/server/cloud_router.py b/src/openjarvis/server/cloud_router.pyindex c90ccf59..1e8530d6 100644--- a/src/openjarvis/server/cloud_router.py+++ b/src/openjarvis/server/cloud_router.py@@ -265,6 +265,64 @@ async def _stream_google(pass+# ---------------------------------------------------------------------------+# Local (Ollama) direct streaming — bypasses engine routing entirely+# ---------------------------------------------------------------------------++def _ollama_host() -> str:+ return os.environ.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/")+++async def stream_local(+ model: str,+ messages: Sequence[Message],+ temperature: float = 0.7,+ max_tokens: int = 1024,+) -> AsyncIterator[str]:+ """Stream tokens directly from Ollama, bypassing the engine system."""+ payload = {+ "model": model,+ "messages": _to_openai_msgs(messages),+ "stream": True,+ # Disable extended thinking (Qwen3.5 etc.) — when enabled all tokens+ # go into the 'thinking' field and 'content' stays empty.+ "think": False,+ "options": {+ "temperature": temperature,+ "num_predict": max_tokens,+ },+ }+ host = _ollama_host()+ async with httpx.AsyncClient(timeout=300) as client:+ async with client.stream("POST", f"{host}/api/chat", json=payload) as resp:+ resp.raise_for_status()+ async for line in resp.aiter_lines():+ if not line:+ continue+ try:+ data = json.loads(line)+ token = data.get("message", {}).get("content", "")+ if token:+ yield token+ if data.get("done"):+ break+ except Exception:+ pass+++async def list_local_models() -> list[str]:+ """Return Ollama model names directly from the Ollama API."""+ host = _ollama_host()+ try:+ async with httpx.AsyncClient(timeout=10) as client:+ resp = await client.get(f"{host}/api/tags")+ resp.raise_for_status()+ data = resp.json()+ return [m["name"] for m in data.get("models", [])]+ except Exception:+ return []++# ---------------------------------------------------------------------------# Public entry point# ---------------------------------------------------------------------------diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.pyindex 8084bac2..133e991e 100644--- a/src/openjarvis/server/routes.py+++ b/src/openjarvis/server/routes.py@@ -290,13 +290,13 @@ async def _handle_stream(complexity_info=None,):"""Stream response using SSE format."""- from openjarvis.server.cloud_router import is_cloud_model, stream_cloud+ from openjarvis.server.cloud_router import is_cloud_model, stream_cloud, stream_localmessages = _to_messages(req.messages)chunk_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"- # Choose token source: cloud router bypasses the engine system entirely- # so cloud models work regardless of how the server was started.+ # Route directly to the right backend — bypasses engine routing entirely+ # so broken MultiEngine state can never misdirect requests.use_cloud = is_cloud_model(model)async def generate():@@ -313,17 +313,12 @@ async def _handle_stream(yield f"data: {first_chunk.model_dump_json()}\n\n"try:- # Stream content- token_iter = (- stream_cloud(model, messages, req.temperature, req.max_tokens)- if use_cloud- else engine.stream(- messages,- model=model,- temperature=req.temperature,- max_tokens=req.max_tokens,- )- )+ # Cloud models → direct cloud API; local models → direct Ollama.+ # Both paths bypass the engine/MultiEngine routing system.+ if use_cloud:+ token_iter = stream_cloud(model, messages, req.temperature, req.max_tokens)+ else:+ token_iter = stream_local(model, messages, req.temperature, req.max_tokens)async for token in token_iter:chunk = ChatCompletionChunk(id=chunk_id,@@ -376,17 +371,11 @@ async def _handle_stream()finish_dict = _json.loads(finish_data.model_dump_json())- # Pull usage from the engine if it tracked it during streaming- raw_engine = engine- # Unwrap InstrumentedEngine if present- if hasattr(raw_engine, "_inner"):- raw_engine = raw_engine._inner- # Unwrap MultiEngine if present- if hasattr(raw_engine, "_engine_for"):- raw_engine = raw_engine._engine_for(model)- stream_usage = getattr(raw_engine, "_last_stream_usage", None)- if isinstance(stream_usage, dict) and stream_usage.get("total_tokens", 0) > 0:- finish_dict["usage"] = stream_usage+ # Tag the finish chunk with the correct engine label.+ # We use the routing decision (use_cloud) directly rather than+ # unwrapping the engine chain, which can be in a broken state.+ finish_dict.setdefault("telemetry", {})+ finish_dict["telemetry"]["engine"] = "cloud" if use_cloud else "ollama"if complexity_info is not None:finish_dict["complexity"] = complexity_info.model_dump()@@ -403,31 +392,21 @@ async def _handle_stream(@router.get("/v1/models")async def list_models(request: Request) -> ModelListResponse:- """List available models from the engine plus any configured cloud models."""- from openjarvis.server.cloud_router import _load_keys- from openjarvis.engine.cloud import (- _OPENAI_MODELS,- _ANTHROPIC_MODELS,- _GOOGLE_MODELS,- _OPENROUTER_POPULAR,- _MINIMAX_MODELS,- )+ """List locally installed models (Ollama).- engine = request.app.state.engine- model_ids: list[str] = list(engine.list_models())-- # Append cloud models for any provider whose key is present on disk.- keys = _load_keys()- if keys.get("OPENAI_API_KEY"):- model_ids.extend(m for m in _OPENAI_MODELS if m not in model_ids)- if keys.get("ANTHROPIC_API_KEY"):- model_ids.extend(m for m in _ANTHROPIC_MODELS if m not in model_ids)- if keys.get("GEMINI_API_KEY") or keys.get("GOOGLE_API_KEY"):- model_ids.extend(m for m in _GOOGLE_MODELS if m not in model_ids)- if keys.get("OPENROUTER_API_KEY"):- model_ids.extend(m for m in _OPENROUTER_POPULAR if m not in model_ids)- if keys.get("MINIMAX_API_KEY"):- model_ids.extend(m for m in _MINIMAX_MODELS if m not in model_ids)+ Cloud models are not included here — they live in the Cloud Models tab+ of the UI and are selected there, not from this endpoint.+ """+ from openjarvis.server.cloud_router import list_local_models++ model_ids = await list_local_models()++ # Fall back to engine.list_models() if Ollama direct call fails+ if not model_ids:+ engine = request.app.state.engine+ all_ids = engine.list_models()+ from openjarvis.server.cloud_router import is_cloud_model+ model_ids = [m for m in all_ids if not is_cloud_model(m)]return ModelListResponse(data=[ModelObject(id=mid) for mid in model_ids],