Personal AI, On Personal Devices
Python
fix(server): offload blocking async route work (#886)
Elliot Slusky8165c2a2e97d
diff --git a/src/openjarvis/server/api_routes.py b/src/openjarvis/server/api_routes.pyindex d84981ca..0ae099c2 100644--- a/src/openjarvis/server/api_routes.py+++ b/src/openjarvis/server/api_routes.py@@ -184,7 +184,7 @@ def _get_memory_backend(request: Request):@memory_router.post("/store")-async def memory_store(req: MemoryStoreRequest, request: Request):+def memory_store(req: MemoryStoreRequest, request: Request):"""Store content in memory."""backend = _get_memory_backend(request)if backend is None:@@ -199,7 +199,7 @@ async def memory_store(req: MemoryStoreRequest, request: Request):@memory_router.post("/search")-async def memory_search(req: MemorySearchRequest, request: Request):+def memory_search(req: MemorySearchRequest, request: Request):"""Search memory for relevant content."""backend = _get_memory_backend(request)if backend is None:@@ -220,7 +220,7 @@ async def memory_search(req: MemorySearchRequest, request: Request):@memory_router.get("/stats")-async def memory_stats(request: Request):+def memory_stats(request: Request):"""Get memory backend statistics."""backend = _get_memory_backend(request)if backend is None:@@ -284,7 +284,7 @@ async def memory_config(request: Request):@memory_router.post("/index")-async def memory_index(req: MemoryIndexRequest, request: Request):+def memory_index(req: MemoryIndexRequest, request: Request):"""Index files from a path into memory."""try:import os@@ -653,6 +653,40 @@ async def prometheus_metrics(request: Request):# ---- WebSocket streaming routes ----websocket_router = APIRouter(tags=["websocket"])+_SYNC_STREAM_END = object()+++async def _next_sync_stream(iterator: Any) -> Any:+ """Fetch one sync-stream item without blocking or racing cancellation."""+ next_task = asyncio.create_task(asyncio.to_thread(next, iterator, _SYNC_STREAM_END))+ try:+ return await asyncio.shield(next_task)+ except asyncio.CancelledError:+ # ``to_thread`` cannot stop a running ``next()``. Wait for it before+ # allowing the iterator to be closed so cancellation cannot race a+ # generator that is still executing in the worker thread.+ try:+ await next_task+ except Exception:+ pass+ raise+++async def _iterate_sync_stream(iterator: Any):+ """Adapt a blocking iterator to an async generator, closing it safely."""+ try:+ while True:+ item = await _next_sync_stream(iterator)+ if item is _SYNC_STREAM_END:+ return+ yield item+ finally:+ close = getattr(iterator, "close", None)+ if callable(close):+ try:+ await asyncio.to_thread(close)+ except Exception as exc:+ logger.warning("Failed to close synchronous engine stream: %s", exc)def _record_ws_trace(@@ -759,9 +793,9 @@ async def websocket_chat_stream(websocket: WebSocket):{"type": "chunk", "content": token},)else:- # Sync generator — iterate in a thread to avoid- # blocking the event loop- for token in gen:+ # Each ``next()`` can perform a blocking upstream+ # read, so offload iteration one item at a time.+ async for token in _iterate_sync_stream(iter(gen)):full_content += tokenawait websocket.send_json({"type": "chunk", "content": token},diff --git a/src/openjarvis/server/digest_routes.py b/src/openjarvis/server/digest_routes.pyindex 04e27ef8..16f3126a 100644--- a/src/openjarvis/server/digest_routes.py+++ b/src/openjarvis/server/digest_routes.py@@ -2,6 +2,7 @@from __future__ import annotations+import asynciofrom typing import Optionalfrom fastapi import APIRouter, HTTPException@@ -24,6 +25,14 @@ class ScheduleUpdate(BaseModel):cron: Optional[str] = None+def _generate_digest_sync() -> str:+ """Generate a digest with the whole Jarvis lifecycle on one worker."""+ from openjarvis.sdk import Jarvis++ with Jarvis() as jarvis:+ return jarvis.ask("Generate my morning digest", agent="morning_digest")++def create_digest_router(*, db_path: str = "") -> APIRouter:"""Create a digest API router with the given store path."""router = APIRouter(prefix="/api/digest", tags=["digest"])@@ -65,10 +74,7 @@ def create_digest_router(*, db_path: str = "") -> APIRouter:async def generate_digest():"""Force re-generation of the digest."""try:- from openjarvis.sdk import Jarvis-- with Jarvis() as j:- result = j.ask("Generate my morning digest", agent="morning_digest")+ result = await asyncio.to_thread(_generate_digest_sync)return {"status": "ok", "text": result}except Exception as exc:raise HTTPException(status_code=500, detail=str(exc))diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.pyindex 16b584c2..7ad27d4b 100644--- a/src/openjarvis/server/routes.py+++ b/src/openjarvis/server/routes.py@@ -1454,7 +1454,7 @@ async def security_scan():from openjarvis.cli.scan_cmd import PrivacyScannerscanner = PrivacyScanner()- results = scanner.run_all()+ results = await asyncio.to_thread(scanner.run_all)return {"has_warnings": any(r.status == "warn" for r in results),"has_failures": any(r.status == "fail" for r in results),diff --git a/tests/server/test_api_routes.py b/tests/server/test_api_routes.pyindex 1a7d8ed8..f4e06eac 100644--- a/tests/server/test_api_routes.py+++ b/tests/server/test_api_routes.py@@ -1,5 +1,6 @@"""Tests for extended API routes."""+import inspectfrom types import SimpleNamespaceimport pytest@@ -8,7 +9,13 @@ fastapi = pytest.importorskip("fastapi")from fastapi import FastAPI # noqa: E402from fastapi.testclient import TestClient # noqa: E402-from openjarvis.server.api_routes import include_all_routes # noqa: E402+from openjarvis.server.api_routes import ( # noqa: E402+ include_all_routes,+ memory_index,+ memory_search,+ memory_stats,+ memory_store,+)def _make_app():@@ -45,6 +52,11 @@ class TestMemoryRoutes:# that cannot be built is tolerated the same way a 500 is._BACKEND_OPTIONAL = (200, 500, 503)+ def test_blocking_memory_handlers_use_starlette_threadpool(self):+ """Sync endpoints are automatically dispatched to worker threads."""+ for handler in (memory_store, memory_search, memory_stats, memory_index):+ assert not inspect.iscoroutinefunction(handler)+def test_search(self):client = TestClient(_make_app())resp = client.post("/v1/memory/search", json={"query": "test"})diff --git a/tests/server/test_digest_routes.py b/tests/server/test_digest_routes.pyindex 51a9516a..01d3a705 100644--- a/tests/server/test_digest_routes.py+++ b/tests/server/test_digest_routes.py@@ -2,6 +2,8 @@from __future__ import annotations+import asyncio+import threadingfrom datetime import datetime, timezoneimport pytest@@ -102,3 +104,45 @@ def test_get_history(store, tmp_path):data = resp.json()assert len(data) == 1assert data[0]["voice_used"] == "jarvis"+++def test_generate_runs_entire_jarvis_lifecycle_on_one_worker(tmp_path, monkeypatch):+ """Construction, ask, and cleanup all stay off the event-loop thread."""+ from openjarvis.server import digest_routes++ calls: list[tuple[str, int]] = []++ class FakeJarvis:+ def __init__(self):+ calls.append(("init", threading.get_ident()))++ def __enter__(self):+ calls.append(("enter", threading.get_ident()))+ return self++ def ask(self, prompt, *, agent):+ calls.append(("ask", threading.get_ident()))+ assert prompt == "Generate my morning digest"+ assert agent == "morning_digest"+ return "digest"++ def __exit__(self, exc_type, exc, tb):+ calls.append(("exit", threading.get_ident()))++ monkeypatch.setattr("openjarvis.sdk.Jarvis", FakeJarvis)+ router = digest_routes.create_digest_router(db_path=str(tmp_path / "digest.db"))+ endpoint = next(+ route.endpoint for route in router.routes if route.path.endswith("/generate")+ )++ async def exercise():+ loop_thread = threading.get_ident()+ response = await endpoint()+ return loop_thread, response++ loop_thread, response = asyncio.run(exercise())++ assert response == {"status": "ok", "text": "digest"}+ assert [name for name, _thread in calls] == ["init", "enter", "ask", "exit"]+ assert len({thread for _name, thread in calls}) == 1+ assert calls[0][1] != loop_threaddiff --git a/tests/server/test_nonblocking_routes.py b/tests/server/test_nonblocking_routes.pynew file mode 100644index 00000000..90c93405--- /dev/null+++ b/tests/server/test_nonblocking_routes.py@@ -0,0 +1,83 @@+"""Regressions for blocking work exposed through async server routes."""++from __future__ import annotations++import asyncio+import threading+from types import SimpleNamespace+from unittest.mock import patch++import pytest++pytest.importorskip("fastapi", reason="openjarvis[server] not installed")++from openjarvis.cli.scan_cmd import PrivacyScanner # noqa: E402+from openjarvis.server.api_routes import _iterate_sync_stream # noqa: E402+from openjarvis.server.routes import security_scan # noqa: E402+++def test_security_scan_runs_on_worker_thread() -> None:+ calls: list[int] = []++ def run_all(_self):+ calls.append(threading.get_ident())+ return [+ SimpleNamespace(+ name="test",+ status="ok",+ message="safe",+ platform="all",+ )+ ]++ async def exercise():+ loop_thread = threading.get_ident()+ result = await security_scan()+ return loop_thread, result++ with patch.object(PrivacyScanner, "run_all", run_all):+ loop_thread, result = asyncio.run(exercise())++ assert calls and calls[0] != loop_thread+ assert result["findings"][0]["name"] == "test"+++def test_sync_stream_cancellation_waits_for_next_before_close() -> None:+ started = threading.Event()+ release = threading.Event()+ closed = threading.Event()+ executing = threading.Event()++ class BlockingIterator:+ def __iter__(self):+ return self++ def __next__(self):+ executing.set()+ started.set()+ release.wait(timeout=2)+ executing.clear()+ return "late token"++ def close(self):+ assert not executing.is_set()+ closed.set()++ async def exercise() -> None:+ stream = _iterate_sync_stream(BlockingIterator())+ pending = asyncio.create_task(anext(stream))+ while not started.is_set():+ await asyncio.sleep(0)++ pending.cancel()+ timer = threading.Timer(0.05, release.set)+ timer.start()+ try:+ with pytest.raises(asyncio.CancelledError):+ await pending+ finally:+ timer.cancel()++ assert closed.is_set()++ asyncio.run(exercise())diff --git a/tests/server/test_websocket.py b/tests/server/test_websocket.pyindex 74dfa2c9..3bc48b33 100644--- a/tests/server/test_websocket.py+++ b/tests/server/test_websocket.py@@ -3,6 +3,7 @@from __future__ import annotationsimport json+import threadingfrom unittest.mock import MagicMockimport pytest@@ -94,6 +95,45 @@ class TestWebSocketStreaming:assert done is not Noneassert done["content"] == "Hello world"+ def test_sync_stream_next_and_close_run_off_event_loop(self):+ stream_threads: list[int] = []+ next_threads: list[int] = []+ close_threads: list[int] = []++ class BlockingIterator:+ def __init__(self):+ self._tokens = iter(["one", "two"])++ def __iter__(self):+ return self++ def __next__(self):+ next_threads.append(threading.get_ident())+ return next(self._tokens)++ def close(self):+ close_threads.append(threading.get_ident())++ class SyncEngine:+ engine_id = "sync"++ def stream(self, messages, *, model="test-model", **kwargs):+ stream_threads.append(threading.get_ident())+ return BlockingIterator()++ app = _make_app(engine=SyncEngine())+ client = TestClient(app)+ with client.websocket_connect("/v1/chat/stream") as ws:+ ws.send_text(json.dumps({"message": "Hi"}))+ while ws.receive_json()["type"] != "done":+ pass++ assert stream_threads+ assert next_threads+ assert close_threads+ assert all(thread != stream_threads[0] for thread in next_threads)+ assert close_threads[0] != stream_threads[0]+def test_missing_message_field(self):"""Sending JSON without a 'message' field should return an error."""app = _make_app()