Skip to content

Personal AI, On Personal Devices

Python

fix(server): offload blocking async route work (#886)

Elliot Slusky8165c2a2e97d
diff --git a/src/openjarvis/server/api_routes.py b/src/openjarvis/server/api_routes.py
index d84981ca..0ae099c2 100644
--- a/src/openjarvis/server/api_routes.py
+++ b/src/openjarvis/server/api_routes.py
@@ -184,7 +184,7 @@ def _get_memory_backend(request: Request):
@memory_router.post("/store")
-async def memory_store(req: MemoryStoreRequest, request: Request):
+def memory_store(req: MemoryStoreRequest, request: Request):
"""Store content in memory."""
backend = _get_memory_backend(request)
if backend is None:
@@ -199,7 +199,7 @@ async def memory_store(req: MemoryStoreRequest, request: Request):
@memory_router.post("/search")
-async def memory_search(req: MemorySearchRequest, request: Request):
+def memory_search(req: MemorySearchRequest, request: Request):
"""Search memory for relevant content."""
backend = _get_memory_backend(request)
if backend is None:
@@ -220,7 +220,7 @@ async def memory_search(req: MemorySearchRequest, request: Request):
@memory_router.get("/stats")
-async def memory_stats(request: Request):
+def memory_stats(request: Request):
"""Get memory backend statistics."""
backend = _get_memory_backend(request)
if backend is None:
@@ -284,7 +284,7 @@ async def memory_config(request: Request):
@memory_router.post("/index")
-async def memory_index(req: MemoryIndexRequest, request: Request):
+def memory_index(req: MemoryIndexRequest, request: Request):
"""Index files from a path into memory."""
try:
import os
@@ -653,6 +653,40 @@ async def prometheus_metrics(request: Request):
# ---- WebSocket streaming routes ----
websocket_router = APIRouter(tags=["websocket"])
+_SYNC_STREAM_END = object()
+
+
+async def _next_sync_stream(iterator: Any) -> Any:
+ """Fetch one sync-stream item without blocking or racing cancellation."""
+ next_task = asyncio.create_task(asyncio.to_thread(next, iterator, _SYNC_STREAM_END))
+ try:
+ return await asyncio.shield(next_task)
+ except asyncio.CancelledError:
+ # ``to_thread`` cannot stop a running ``next()``. Wait for it before
+ # allowing the iterator to be closed so cancellation cannot race a
+ # generator that is still executing in the worker thread.
+ try:
+ await next_task
+ except Exception:
+ pass
+ raise
+
+
+async def _iterate_sync_stream(iterator: Any):
+ """Adapt a blocking iterator to an async generator, closing it safely."""
+ try:
+ while True:
+ item = await _next_sync_stream(iterator)
+ if item is _SYNC_STREAM_END:
+ return
+ yield item
+ finally:
+ close = getattr(iterator, "close", None)
+ if callable(close):
+ try:
+ await asyncio.to_thread(close)
+ except Exception as exc:
+ logger.warning("Failed to close synchronous engine stream: %s", exc)
def _record_ws_trace(
@@ -759,9 +793,9 @@ async def websocket_chat_stream(websocket: WebSocket):
{"type": "chunk", "content": token},
)
else:
- # Sync generator — iterate in a thread to avoid
- # blocking the event loop
- for token in gen:
+ # Each ``next()`` can perform a blocking upstream
+ # read, so offload iteration one item at a time.
+ async for token in _iterate_sync_stream(iter(gen)):
full_content += token
await websocket.send_json(
{"type": "chunk", "content": token},
diff --git a/src/openjarvis/server/digest_routes.py b/src/openjarvis/server/digest_routes.py
index 04e27ef8..16f3126a 100644
--- a/src/openjarvis/server/digest_routes.py
+++ b/src/openjarvis/server/digest_routes.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import asyncio
from typing import Optional
from fastapi import APIRouter, HTTPException
@@ -24,6 +25,14 @@ class ScheduleUpdate(BaseModel):
cron: Optional[str] = None
+def _generate_digest_sync() -> str:
+ """Generate a digest with the whole Jarvis lifecycle on one worker."""
+ from openjarvis.sdk import Jarvis
+
+ with Jarvis() as jarvis:
+ return jarvis.ask("Generate my morning digest", agent="morning_digest")
+
+
def create_digest_router(*, db_path: str = "") -> APIRouter:
"""Create a digest API router with the given store path."""
router = APIRouter(prefix="/api/digest", tags=["digest"])
@@ -65,10 +74,7 @@ def create_digest_router(*, db_path: str = "") -> APIRouter:
async def generate_digest():
"""Force re-generation of the digest."""
try:
- from openjarvis.sdk import Jarvis
-
- with Jarvis() as j:
- result = j.ask("Generate my morning digest", agent="morning_digest")
+ result = await asyncio.to_thread(_generate_digest_sync)
return {"status": "ok", "text": result}
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc))
diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py
index 16b584c2..7ad27d4b 100644
--- a/src/openjarvis/server/routes.py
+++ b/src/openjarvis/server/routes.py
@@ -1454,7 +1454,7 @@ async def security_scan():
from openjarvis.cli.scan_cmd import PrivacyScanner
scanner = PrivacyScanner()
- results = scanner.run_all()
+ results = await asyncio.to_thread(scanner.run_all)
return {
"has_warnings": any(r.status == "warn" for r in results),
"has_failures": any(r.status == "fail" for r in results),
diff --git a/tests/server/test_api_routes.py b/tests/server/test_api_routes.py
index 1a7d8ed8..f4e06eac 100644
--- a/tests/server/test_api_routes.py
+++ b/tests/server/test_api_routes.py
@@ -1,5 +1,6 @@
"""Tests for extended API routes."""
+import inspect
from types import SimpleNamespace
import pytest
@@ -8,7 +9,13 @@ fastapi = pytest.importorskip("fastapi")
from fastapi import FastAPI # noqa: E402
from fastapi.testclient import TestClient # noqa: E402
-from openjarvis.server.api_routes import include_all_routes # noqa: E402
+from openjarvis.server.api_routes import ( # noqa: E402
+ include_all_routes,
+ memory_index,
+ memory_search,
+ memory_stats,
+ memory_store,
+)
def _make_app():
@@ -45,6 +52,11 @@ class TestMemoryRoutes:
# that cannot be built is tolerated the same way a 500 is.
_BACKEND_OPTIONAL = (200, 500, 503)
+ def test_blocking_memory_handlers_use_starlette_threadpool(self):
+ """Sync endpoints are automatically dispatched to worker threads."""
+ for handler in (memory_store, memory_search, memory_stats, memory_index):
+ assert not inspect.iscoroutinefunction(handler)
+
def test_search(self):
client = TestClient(_make_app())
resp = client.post("/v1/memory/search", json={"query": "test"})
diff --git a/tests/server/test_digest_routes.py b/tests/server/test_digest_routes.py
index 51a9516a..01d3a705 100644
--- a/tests/server/test_digest_routes.py
+++ b/tests/server/test_digest_routes.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import asyncio
+import threading
from datetime import datetime, timezone
import pytest
@@ -102,3 +104,45 @@ def test_get_history(store, tmp_path):
data = resp.json()
assert len(data) == 1
assert data[0]["voice_used"] == "jarvis"
+
+
+def test_generate_runs_entire_jarvis_lifecycle_on_one_worker(tmp_path, monkeypatch):
+ """Construction, ask, and cleanup all stay off the event-loop thread."""
+ from openjarvis.server import digest_routes
+
+ calls: list[tuple[str, int]] = []
+
+ class FakeJarvis:
+ def __init__(self):
+ calls.append(("init", threading.get_ident()))
+
+ def __enter__(self):
+ calls.append(("enter", threading.get_ident()))
+ return self
+
+ def ask(self, prompt, *, agent):
+ calls.append(("ask", threading.get_ident()))
+ assert prompt == "Generate my morning digest"
+ assert agent == "morning_digest"
+ return "digest"
+
+ def __exit__(self, exc_type, exc, tb):
+ calls.append(("exit", threading.get_ident()))
+
+ monkeypatch.setattr("openjarvis.sdk.Jarvis", FakeJarvis)
+ router = digest_routes.create_digest_router(db_path=str(tmp_path / "digest.db"))
+ endpoint = next(
+ route.endpoint for route in router.routes if route.path.endswith("/generate")
+ )
+
+ async def exercise():
+ loop_thread = threading.get_ident()
+ response = await endpoint()
+ return loop_thread, response
+
+ loop_thread, response = asyncio.run(exercise())
+
+ assert response == {"status": "ok", "text": "digest"}
+ assert [name for name, _thread in calls] == ["init", "enter", "ask", "exit"]
+ assert len({thread for _name, thread in calls}) == 1
+ assert calls[0][1] != loop_thread
diff --git a/tests/server/test_nonblocking_routes.py b/tests/server/test_nonblocking_routes.py
new file mode 100644
index 00000000..90c93405
--- /dev/null
+++ b/tests/server/test_nonblocking_routes.py
@@ -0,0 +1,83 @@
+"""Regressions for blocking work exposed through async server routes."""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+pytest.importorskip("fastapi", reason="openjarvis[server] not installed")
+
+from openjarvis.cli.scan_cmd import PrivacyScanner # noqa: E402
+from openjarvis.server.api_routes import _iterate_sync_stream # noqa: E402
+from openjarvis.server.routes import security_scan # noqa: E402
+
+
+def test_security_scan_runs_on_worker_thread() -> None:
+ calls: list[int] = []
+
+ def run_all(_self):
+ calls.append(threading.get_ident())
+ return [
+ SimpleNamespace(
+ name="test",
+ status="ok",
+ message="safe",
+ platform="all",
+ )
+ ]
+
+ async def exercise():
+ loop_thread = threading.get_ident()
+ result = await security_scan()
+ return loop_thread, result
+
+ with patch.object(PrivacyScanner, "run_all", run_all):
+ loop_thread, result = asyncio.run(exercise())
+
+ assert calls and calls[0] != loop_thread
+ assert result["findings"][0]["name"] == "test"
+
+
+def test_sync_stream_cancellation_waits_for_next_before_close() -> None:
+ started = threading.Event()
+ release = threading.Event()
+ closed = threading.Event()
+ executing = threading.Event()
+
+ class BlockingIterator:
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ executing.set()
+ started.set()
+ release.wait(timeout=2)
+ executing.clear()
+ return "late token"
+
+ def close(self):
+ assert not executing.is_set()
+ closed.set()
+
+ async def exercise() -> None:
+ stream = _iterate_sync_stream(BlockingIterator())
+ pending = asyncio.create_task(anext(stream))
+ while not started.is_set():
+ await asyncio.sleep(0)
+
+ pending.cancel()
+ timer = threading.Timer(0.05, release.set)
+ timer.start()
+ try:
+ with pytest.raises(asyncio.CancelledError):
+ await pending
+ finally:
+ timer.cancel()
+
+ assert closed.is_set()
+
+ asyncio.run(exercise())
diff --git a/tests/server/test_websocket.py b/tests/server/test_websocket.py
index 74dfa2c9..3bc48b33 100644
--- a/tests/server/test_websocket.py
+++ b/tests/server/test_websocket.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
+import threading
from unittest.mock import MagicMock
import pytest
@@ -94,6 +95,45 @@ class TestWebSocketStreaming:
assert done is not None
assert done["content"] == "Hello world"
+ def test_sync_stream_next_and_close_run_off_event_loop(self):
+ stream_threads: list[int] = []
+ next_threads: list[int] = []
+ close_threads: list[int] = []
+
+ class BlockingIterator:
+ def __init__(self):
+ self._tokens = iter(["one", "two"])
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ next_threads.append(threading.get_ident())
+ return next(self._tokens)
+
+ def close(self):
+ close_threads.append(threading.get_ident())
+
+ class SyncEngine:
+ engine_id = "sync"
+
+ def stream(self, messages, *, model="test-model", **kwargs):
+ stream_threads.append(threading.get_ident())
+ return BlockingIterator()
+
+ app = _make_app(engine=SyncEngine())
+ client = TestClient(app)
+ with client.websocket_connect("/v1/chat/stream") as ws:
+ ws.send_text(json.dumps({"message": "Hi"}))
+ while ws.receive_json()["type"] != "done":
+ pass
+
+ assert stream_threads
+ assert next_threads
+ assert close_threads
+ assert all(thread != stream_threads[0] for thread in next_threads)
+ assert close_threads[0] != stream_threads[0]
+
def test_missing_message_field(self):
"""Sending JSON without a 'message' field should return an error."""
app = _make_app()