Personal AI, On Personal Devices
Python
fix(agents): serialize ticks and preserve lifecycle ownership (#939)
Haotian Zheng56671aff7de6
diff --git a/src/openjarvis/agents/manager.py b/src/openjarvis/agents/manager.pyindex 50ec03d7..e247bdaf 100644--- a/src/openjarvis/agents/manager.py+++ b/src/openjarvis/agents/manager.py@@ -9,8 +9,10 @@ from __future__ import annotationsimport jsonimport loggingimport sqlite3+import threadingimport timeimport uuid+from functools import wrapsfrom typing import Any, Dict, List, Optionalfrom uuid import uuid4@@ -95,6 +97,17 @@ CREATE TABLE IF NOT EXISTS agent_learning_log (_SUMMARY_MAX = 16000+def _db_locked(func):+ """Serialize access to the connection shared by manager worker threads."""++ @wraps(func)+ def wrapped(self, *args, **kwargs):+ with self._db_lock:+ return func(self, *args, **kwargs)++ return wrapped++class AgentManager:"""Persistent agent lifecycle manager with SQLite backing."""@@ -107,6 +120,7 @@ class AgentManager:def __init__(self, db_path: str, *, clear_stale_running: bool = False) -> None:self._db_path = str(db_path)+ self._db_lock = threading.RLock()self._conn = sqlite3.connect(self._db_path, check_same_thread=False)self._conn.row_factory = sqlite3.Rowself._conn.execute("PRAGMA journal_mode=WAL")@@ -149,6 +163,7 @@ class AgentManager:if clear_stale_running:self._clear_stale_running_state()+ @_db_lockeddef _clear_stale_running_state(self) -> None:"""Reset any agent stuck in ``status='running'`` on startup.@@ -175,11 +190,13 @@ class AgentManager:cur.rowcount,)+ @_db_lockeddef close(self) -> None:self._conn.close()# ── Agent CRUD ────────────────────────────────────────────────+ @_db_lockeddef create_agent(self,name: str,@@ -200,6 +217,7 @@ class AgentManager:self._conn.commit()return self.get_agent(agent_id) # type: ignore[return-value]+ @_db_lockeddef list_agents(self, include_archived: bool = False) -> List[Dict[str, Any]]:query = "SELECT * FROM managed_agents"if not include_archived:@@ -208,12 +226,14 @@ class AgentManager:rows = self._conn.execute(query).fetchall()return [self._row_to_agent(r) for r in rows]+ @_db_lockeddef get_agent(self, agent_id: str) -> Optional[Dict[str, Any]]:row = self._conn.execute("SELECT * FROM managed_agents WHERE id = ?", (agent_id,)).fetchone()return self._row_to_agent(row) if row else None+ @_db_lockeddef update_agent(self, agent_id: str, **kwargs: Any) -> Dict[str, Any]:sets: List[str] = []vals: List[Any] = []@@ -261,15 +281,19 @@ class AgentManager:self._conn.commit()return self.get_agent(agent_id) # type: ignore[return-value]+ @_db_lockeddef delete_agent(self, agent_id: str) -> None:self._set_status(agent_id, "archived")+ @_db_lockeddef pause_agent(self, agent_id: str) -> None:self._set_status(agent_id, "paused")+ @_db_lockeddef resume_agent(self, agent_id: str) -> None:self._set_status(agent_id, "idle")+ @_db_lockeddef _set_status(self, agent_id: str, status: str) -> None:self._conn.execute("UPDATE managed_agents SET status = ?, updated_at = ? WHERE id = ?",@@ -279,6 +303,7 @@ class AgentManager:# ── Tick concurrency guard ────────────────────────────────────+ @_db_lockeddef start_tick(self, agent_id: str) -> None:"""Mark agent as running. Raises ValueError if already running.@@ -286,23 +311,60 @@ class AgentManager:without any update is treated as a zombie left by a dead worker andovertaken rather than refused — otherwise a crash with no serveraround to sweep it would wedge the agent forever.++ The state transition is a single conditional SQLite update. The+ read-before-write approach is racy when two workers start together:+ both can observe ``idle`` and then mark the row ``running``. The+ conditional update lets SQLite choose exactly one winner, including+ when workers use separate connections or processes."""- agent = self.get_agent(agent_id)- if agent and agent["status"] == "running":- age = time.time() - (agent.get("updated_at") or 0)- if age < self._STALE_TICK_SECONDS:- raise ValueError(f"Agent {agent_id} is already executing a tick")- logger.warning(- "Agent %s: overtaking stale tick lock (running, idle for %.0fs)",- agent_id,- age,+ while True:+ now = time.time()+ stale_before = now - self._STALE_TICK_SECONDS+ agent = self.get_agent(agent_id)+ if agent is None:+ return++ was_stale = (+ agent["status"] == "running"+ and (agent.get("updated_at") or 0) <= stale_before)- self._set_status(agent_id, "running")-+ cursor = self._conn.execute(+ "UPDATE managed_agents SET status = 'running', "+ "updated_at = ? WHERE id = ? AND "+ "(status != 'running' OR updated_at <= ?)",+ (now, agent_id, stale_before),+ )+ self._conn.commit()+ if cursor.rowcount:+ if was_stale:+ age = time.time() - (agent.get("updated_at") or 0)+ logger.warning(+ "Agent %s: overtaking stale tick lock "+ "(running, idle for %.0fs)",+ agent_id,+ age,+ )+ return++ # Another connection acquired the lock after our read. If that+ # lock is still fresh, this caller loses; if it ended between the+ # failed update and this read, retry so the caller never proceeds+ # without owning the tick lock.+ current = self.get_agent(agent_id)+ if current is None:+ return+ if current["status"] == "running":+ age = time.time() - (current.get("updated_at") or 0)+ if age < self._STALE_TICK_SECONDS:+ raise ValueError(f"Agent {agent_id} is already executing a tick")++ @_db_lockeddef end_tick(self, agent_id: str) -> None:self._conn.execute("UPDATE managed_agents SET status = 'idle', "- "current_activity = '', updated_at = ? WHERE id = ?",+ "current_activity = '', updated_at = ? "+ "WHERE id = ? AND status = 'running'",(time.time(), agent_id),)self._conn.commit()@@ -311,6 +373,7 @@ class AgentManager:_CHECKPOINT_RETENTION = 5+ @_db_lockeddef save_checkpoint(self,agent_id: str,@@ -348,6 +411,7 @@ class AgentManager:"created_at": now,}+ @_db_lockeddef list_checkpoints(self, agent_id: str) -> list:rows = self._conn.execute("SELECT * FROM agent_checkpoints"@@ -356,6 +420,7 @@ class AgentManager:).fetchall()return [self._row_to_checkpoint(r) for r in rows]+ @_db_lockeddef get_latest_checkpoint(self, agent_id: str) -> Optional[Dict[str, Any]]:row = self._conn.execute("SELECT * FROM agent_checkpoints"@@ -364,6 +429,7 @@ class AgentManager:).fetchone()return self._row_to_checkpoint(row) if row else None+ @_db_lockeddef recover_agent(self, agent_id: str) -> Optional[Dict[str, Any]]:checkpoint = self.get_latest_checkpoint(agent_id)# Always reset to idle — clearing the error state is the primary purpose@@ -383,6 +449,7 @@ class AgentManager:# ── Summary memory ────────────────────────────────────────────+ @_db_lockeddef update_summary_memory(self, agent_id: str, summary: str) -> None:truncated = summary[:_SUMMARY_MAX]self._conn.execute(@@ -393,6 +460,7 @@ class AgentManager:# ── Task CRUD ─────────────────────────────────────────────────+ @_db_lockeddef create_task(self, agent_id: str, description: str, status: str = "pending") -> Dict[str, Any]:@@ -406,6 +474,7 @@ class AgentManager:self._conn.commit()return self._get_task(task_id) # type: ignore[return-value]+ @_db_lockeddef list_tasks(self, agent_id: str, status: Optional[str] = None) -> List[Dict[str, Any]]:@@ -418,6 +487,7 @@ class AgentManager:rows = self._conn.execute(query, params).fetchall()return [self._row_to_task(r) for r in rows]+ @_db_lockeddef update_task(self, task_id: str, **kwargs: Any) -> Dict[str, Any]:sets: List[str] = []vals: List[Any] = []@@ -440,10 +510,12 @@ class AgentManager:self._conn.commit()return self._get_task(task_id) # type: ignore[return-value]+ @_db_lockeddef delete_task(self, task_id: str) -> None:self._conn.execute("DELETE FROM agent_tasks WHERE id = ?", (task_id,))self._conn.commit()+ @_db_lockeddef _get_task(self, task_id: str) -> Optional[Dict[str, Any]]:row = self._conn.execute("SELECT * FROM agent_tasks WHERE id = ?", (task_id,)@@ -452,6 +524,7 @@ class AgentManager:# ── Channel bindings ──────────────────────────────────────────+ @_db_lockeddef bind_channel(self,agent_id: str,@@ -471,22 +544,26 @@ class AgentManager:self._conn.commit()return self._get_binding(binding_id) # type: ignore[return-value]+ @_db_lockeddef list_channel_bindings(self, agent_id: str) -> List[Dict[str, Any]]:rows = self._conn.execute("SELECT * FROM channel_bindings WHERE agent_id = ?", (agent_id,)).fetchall()return [self._row_to_binding(r) for r in rows]+ @_db_lockeddef unbind_channel(self, binding_id: str) -> None:self._conn.execute("DELETE FROM channel_bindings WHERE id = ?", (binding_id,))self._conn.commit()+ @_db_lockeddef _get_binding(self, binding_id: str) -> Optional[Dict[str, Any]]:row = self._conn.execute("SELECT * FROM channel_bindings WHERE id = ?", (binding_id,)).fetchone()return self._row_to_binding(row) if row else None+ @_db_lockeddef find_binding_for_channel(self, channel_type: str, channel_id: str) -> Optional[Dict[str, Any]]:@@ -568,6 +645,7 @@ class AgentManager:# ── Message queue ─────────────────────────────────────────────+ @_db_lockeddef send_message(self, agent_id: str, content: str, mode: str = "queued") -> dict:msg_id = uuid4().hex[:16]now = time.time()@@ -588,6 +666,7 @@ class AgentManager:"created_at": now,}+ @_db_lockeddef store_agent_response(self,agent_id: str,@@ -623,6 +702,7 @@ class AgentManager:"tool_calls": tool_calls or None,}+ @_db_lockeddef list_messages(self, agent_id: str, limit: int = 50) -> list[dict]:rows = self._conn.execute("SELECT * FROM agent_messages"@@ -631,6 +711,7 @@ class AgentManager:).fetchall()return [self._row_to_message(r) for r in rows]+ @_db_lockeddef get_pending_messages(self, agent_id: str) -> list[dict]:rows = self._conn.execute("SELECT * FROM agent_messages"@@ -640,6 +721,7 @@ class AgentManager:).fetchall()return [self._row_to_message(r) for r in rows]+ @_db_lockeddef mark_message_delivered(self, message_id: str) -> None:self._conn.execute("UPDATE agent_messages SET status = 'delivered' WHERE id = ?",@@ -647,6 +729,7 @@ class AgentManager:)self._conn.commit()+ @_db_lockeddef add_agent_response(self, agent_id: str, content: str) -> dict:msg_id = uuid4().hex[:16]now = time.time()@@ -692,6 +775,7 @@ class AgentManager:# ── Learning log ──────────────────────────────────────────+ @_db_lockeddef add_learning_log(self,agent_id: str,@@ -717,6 +801,7 @@ class AgentManager:"created_at": now,}+ @_db_lockeddef list_learning_log(self, agent_id: str, limit: int = 50) -> list[dict]:rows = self._conn.execute("SELECT * FROM agent_learning_log"diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.pyindex 4e501ba3..deb62e14 100644--- a/src/openjarvis/server/agent_manager_routes.py+++ b/src/openjarvis/server/agent_manager_routes.py@@ -5,7 +5,7 @@ from __future__ import annotationsimport loggingimport re as _reimport threading-from typing import Any, Dict, List, Optional, Tuple+from typing import Any, Callable, Dict, List, Optional, Tuplefrom openjarvis.agents.manager import AgentManagerfrom openjarvis.agents.tool_resolver import (@@ -852,12 +852,15 @@ async def _stream_managed_agent(engine: Any,bus: Any,app_state: Any = None,+ on_complete: Optional[Callable[[], None]] = None,) -> StreamingResponse:"""Run a managed agent with real LLM token streaming via SSE.Uses ``engine.stream_full()`` to yield tokens as they arrive from theLLM. Supports multi-turn tool-calling: when the model emits tool_calls,- they are executed and the results fed back for the next turn.+ they are executed and the results fed back for the next turn. When+ ``on_complete`` is provided, it runs after either streaming branch has+ finished its response-local cleanup."""import jsonimport uuid@@ -867,6 +870,21 @@ async def _stream_managed_agent(from openjarvis.core.types import Message, Roleagent_id = agent_record["id"]++ completion_lock = threading.Lock()+ completion_called = False++ def _complete_stream() -> None:+ nonlocal completion_called+ with completion_lock:+ if completion_called or on_complete is None:+ return+ completion_called = True+ try:+ on_complete()+ except Exception:+ logger.warning("Managed agent stream cleanup failed", exc_info=True)+config = agent_record.get("config", {})# Resolve the model: prefer the agent's own config, then the server's# resolved model (app.state.model — what the engine was booted with),@@ -950,6 +968,16 @@ async def _stream_managed_agent(if agent_type == "deep_research":dr_tools = resolved_toolkit.instancesif dr_tools:+ worker_owns_tick = threading.Event()++ def _complete_research_response() -> None:+ # A disconnected/timed-out response cannot release a tick+ # while its synchronous research worker is still executing.+ if not worker_owns_tick.is_set():+ try:+ resolved_toolkit.close()+ finally:+ _complete_stream()async def generate_deep_research():"""Run DeepResearchAgent in thread, stream progress + result."""@@ -1091,14 +1119,22 @@ async def _stream_managed_agent(})+ def _run_with_tick_cleanup():+ try:+ _run_agent()+ finally:+ _complete_stream()++ worker_owns_tick.set()try:_start_managed_worker(app_state,- _run_agent,+ _run_with_tick_cleanup,name=f"managed-agent-deep-research-{agent_id}",)except Exception as exc:- resolved_toolkit.close()+ worker_owns_tick.clear()+ _complete_research_response()progress_q.put({"type": "error",@@ -1231,13 +1267,21 @@ async def _stream_managed_agent()break+ async def guarded_deep_research():+ try:+ async for chunk in generate_deep_research():+ yield chunk+ finally:+ _complete_research_response()+return StreamingResponse(- generate_deep_research(),+ guarded_deep_research(),media_type="text/event-stream",headers={"Cache-Control": "no-cache","Connection": "keep-alive",},+ background=BackgroundTask(_complete_research_response),)# The canonical resolver exposes the same live instances as OpenAI specs@@ -1306,7 +1350,10 @@ async def _stream_managed_agent(try:_persist_final()finally:- resolved_toolkit.close()+ try:+ resolved_toolkit.close()+ finally:+ _complete_stream()async def generate():"""Async generator yielding SSE-formatted chunks with real token streaming."""@@ -1991,8 +2038,29 @@ def create_agent_manager_router():manager.update_agent(agent_id, status="idle")+ stream_tick_acquired = False+ stream_engine = None+ if req.stream:+ stream_engine = getattr(request.app.state, "engine", None)+ if stream_engine is not None:+ # Unlike the queued/immediate paths, streaming runs directly+ # in the request, so it must acquire the same tick guard here.+ try:+ manager.start_tick(agent_id)+ stream_tick_acquired = True+ except ValueError:+ raise HTTPException(+ status_code=409,+ detail="Agent is already running",+ )+# Store user message in DB (always, regardless of stream mode)- msg = manager.send_message(agent_id, req.content, mode=req.mode)+ try:+ msg = manager.send_message(agent_id, req.content, mode=req.mode)+ except Exception:+ if stream_tick_acquired:+ manager.end_tick(agent_id)+ raiseif not req.stream and req.mode != "immediate":return msg@@ -2072,7 +2140,7 @@ def create_agent_manager_router(return msg# --- Streaming mode: run agent and return SSE response ---- engine = getattr(request.app.state, "engine", None)+ engine = stream_enginebus = getattr(request.app.state, "bus", None)if engine is None:raise HTTPException(@@ -2080,15 +2148,21 @@ def create_agent_manager_router(detail="Engine not available for streaming",)- return await _stream_managed_agent(- manager=manager,- agent_record=agent_record,- user_content=req.content,- message_id=msg["id"],- engine=engine,- bus=bus,- app_state=request.app.state,- )+ try:+ return await _stream_managed_agent(+ manager=manager,+ agent_record=agent_record,+ user_content=req.content,+ message_id=msg["id"],+ engine=engine,+ bus=bus,+ app_state=request.app.state,+ on_complete=lambda: manager.end_tick(agent_id),+ )+ except Exception:+ if stream_tick_acquired:+ manager.end_tick(agent_id)+ raise# ── State inspection ─────────────────────────────────────diff --git a/tests/agents/test_manager.py b/tests/agents/test_manager.pyindex ea51d111..005fc887 100644--- a/tests/agents/test_manager.py+++ b/tests/agents/test_manager.py@@ -2,7 +2,10 @@from __future__ import annotations+import concurrent.futuresimport tempfile+import threading+import timefrom pathlib import Pathimport pytest@@ -167,6 +170,142 @@ class TestConcurrency:with pytest.raises(ValueError, match="already executing"):manager.start_tick(agent["id"])+ def test_start_tick_is_atomic_across_connections(self, tmp_path, monkeypatch):+ """Only one caller may acquire a tick when two connections race."""+ from openjarvis.agents.manager import AgentManager++ db_path = tmp_path / "agents.db"+ first = AgentManager(str(db_path))+ second = AgentManager(str(db_path))+ try:+ agent = first.create_agent(name="racing", agent_type="simple")++ # The old implementation read the idle row before calling+ # _set_status(). Holding both writers at that boundary makes the+ # check-then-update race deterministic on the unfixed code.+ writers_ready = threading.Barrier(2)+ original_set_status = AgentManager._set_status++ def hold_running_writes(self, agent_id, status):+ if status == "running":+ writers_ready.wait(timeout=5)+ return original_set_status(self, agent_id, status)++ monkeypatch.setattr(AgentManager, "_set_status", hold_running_writes)++ managers = (first, second)+ with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:+ futures = [+ pool.submit(current.start_tick, agent["id"]) for current in managers+ ]+ outcomes = []+ for future in futures:+ try:+ future.result(timeout=5)+ except ValueError as exc:+ outcomes.append(str(exc))+ else:+ outcomes.append("success")++ assert outcomes.count("success") == 1+ assert sum("already executing" in outcome for outcome in outcomes) == 1+ assert first.get_agent(agent["id"])["status"] == "running"+ finally:+ first.close()+ second.close()++ def test_start_tick_serializes_its_uncommitted_update(self, manager):+ """Shared-connection callers cannot observe or commit a partial tick."""+ agent = manager.create_agent(name="pending-tick", agent_type="simple")+ original_connection = manager._conn+ update_pending = threading.Event()+ allow_commit = threading.Event()+ reader_started = threading.Event()+ reader_finished = threading.Event()++ class PausedCommit:+ def __getattr__(self, name):+ return getattr(original_connection, name)++ def commit(self):+ update_pending.set()+ assert allow_commit.wait(5)+ return original_connection.commit()++ manager._conn = PausedCommit()++ def read_agent():+ reader_started.set()+ try:+ return manager.get_agent(agent["id"])+ finally:+ reader_finished.set()++ with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:+ tick = pool.submit(manager.start_tick, agent["id"])+ assert update_pending.wait(5)+ reader = pool.submit(read_agent)+ assert reader_started.wait(5)+ try:+ assert not reader_finished.wait(0.1), "tick leaked uncommitted state"+ finally:+ allow_commit.set()+ tick.result(timeout=5)+ assert reader.result(timeout=5)["status"] == "running"++ def test_parallel_ticks_on_distinct_agents_share_one_connection(self, manager):+ """Independent agents must not interfere through SQLite transactions."""+ agents = [+ manager.create_agent(name=f"parallel-{index}", agent_type="simple")+ for index in range(8)+ ]+ ready = threading.Barrier(len(agents))++ def tick_many(agent):+ ready.wait(timeout=5)+ for _ in range(50):+ manager.start_tick(agent["id"])+ manager.end_tick(agent["id"])++ with concurrent.futures.ThreadPoolExecutor(max_workers=len(agents)) as pool:+ futures = [pool.submit(tick_many, agent) for agent in agents]+ for future in futures:+ future.result(timeout=10)+ assert all(+ manager.get_agent(agent["id"])["status"] == "idle" for agent in agents+ )++ def test_start_tick_overtakes_stale_lock(self, manager):+ agent = manager.create_agent(name="stale", agent_type="simple")+ stale_at = time.time() - manager._STALE_TICK_SECONDS - 1+ manager._conn.execute(+ "UPDATE managed_agents SET status = 'running', updated_at = ? WHERE id = ?",+ (stale_at, agent["id"]),+ )+ manager._conn.commit()++ manager.start_tick(agent["id"])++ refreshed = manager.get_agent(agent["id"])+ assert refreshed["status"] == "running"+ assert refreshed["updated_at"] > stale_at++ @pytest.mark.parametrize(+ ("change_status", "expected_status"),+ [("pause_agent", "paused"), ("delete_agent", "archived")],+ )+ def test_end_tick_preserves_user_status_change(+ self, manager, change_status, expected_status+ ):+ """A late tick cleanup must not undo a user's pause or archive action."""+ agent = manager.create_agent(name="lifecycle", agent_type="simple")+ manager.start_tick(agent["id"])++ getattr(manager, change_status)(agent["id"])+ manager.end_tick(agent["id"])++ assert manager.get_agent(agent["id"])["status"] == expected_status+class TestCheckpoints:def test_save_checkpoint(self, manager):@@ -373,3 +512,23 @@ class TestSchemaAndThreading:t.join(timeout=5)assert len(results) == 1assert results[0]["name"] == "threaded"++ def test_concurrent_updates_are_serialized(self, manager):+ """Concurrent updates on the shared manager connection must not be lost."""+ from concurrent.futures import ThreadPoolExecutor++ agent = manager.create_agent(name="concurrent", agent_type="simple")+ workers = 8+ updates_per_worker = 20++ def update_many():+ for _ in range(updates_per_worker):+ manager.update_agent(agent["id"], total_tokens_increment=1)++ with ThreadPoolExecutor(max_workers=workers) as pool:+ futures = [pool.submit(update_many) for _ in range(workers)]+ for future in futures:+ future.result()++ updated = manager.get_agent(agent["id"])+ assert updated["total_tokens"] == workers * updates_per_workerdiff --git a/tests/server/test_agent_manager_routes.py b/tests/server/test_agent_manager_routes.pyindex 895b1651..9d3aab54 100644--- a/tests/server/test_agent_manager_routes.py+++ b/tests/server/test_agent_manager_routes.py@@ -487,6 +487,105 @@ class TestAgentManagerStreaming:assert resp.status_code == 200assert "Error:" in resp.text or "error" in resp.text.lower()assert "data: [DONE]" in resp.text+ assert manager.get_agent(agent["id"])["status"] == "idle"++ def test_streaming_requests_share_tick_guard(self, manager, tmp_path, monkeypatch):+ """A second stream for the same agent must not start another tick."""+ import asyncio++ from fastapi import FastAPI+ from fastapi.testclient import TestClient++ from openjarvis.engine._stubs import StreamChunk+ from openjarvis.server.agent_manager_routes import (+ create_agent_manager_router,+ )++ class SlowEngine:+ engine_id = "slow"+ _model = "test-model"++ def __init__(self):+ self.calls = 0+ self.lock = threading.Lock()+ self.first_started = threading.Event()+ self.second_started = threading.Event()+ self.release = threading.Event()++ async def stream_full(self, messages, *, model, **kwargs):+ with self.lock:+ self.calls += 1+ call = self.calls+ if call == 1:+ self.first_started.set()+ else:+ self.second_started.set()+ while not self.release.is_set():+ await asyncio.sleep(0.01)+ yield StreamChunk(content=f"response-{call}")+ yield StreamChunk(finish_reason="stop")++ monkeypatch.setenv("OPENJARVIS_HOME", str(tmp_path / "runtime"))+ engine = SlowEngine()+ app = FastAPI()+ app.state.engine = engine+ app.state.bus = None+ for router in create_agent_manager_router(manager):+ app.include_router(router)++ agent = manager.create_agent(name="stream-race", agent_type="simple")+ url = f"/v1/managed-agents/{agent['id']}/messages"++ def send(client):+ return client.post(url, json={"content": "hello", "stream": True})++ client1 = TestClient(app)+ client2 = TestClient(app)+ with ThreadPoolExecutor(max_workers=2) as pool:+ first = pool.submit(send, client1)+ assert engine.first_started.wait(5)+ second = pool.submit(send, client2)+ try:+ assert not engine.second_started.wait(0.5)+ second_response = second.result(timeout=5)+ assert second_response.status_code == 409+ finally:+ engine.release.set()++ first_response = first.result(timeout=10)++ assert first_response.status_code == 200+ assert engine.calls == 1+ assert manager.get_agent(agent["id"])["status"] == "idle"++ def test_stream_initialization_failure_releases_tick_guard(self, manager):+ """A stream setup error must not leave the agent marked as running."""+ from fastapi import FastAPI+ from fastapi.testclient import TestClient as TC++ from openjarvis.server.agent_manager_routes import (+ create_agent_manager_router,+ )++ app = FastAPI()+ app.state.engine = MagicMock()+ app.state.bus = None+ for router in create_agent_manager_router(manager):+ app.include_router(router)+ client = TC(app, raise_server_exceptions=False)++ agent = manager.create_agent(name="stream-init-error", agent_type="simple")+ with patch(+ "openjarvis.server.agent_manager_routes._stream_managed_agent",+ new=AsyncMock(side_effect=RuntimeError("stream setup failed")),+ ):+ resp = client.post(+ f"/v1/managed-agents/{agent['id']}/messages",+ json={"content": "fail during setup", "stream": True},+ )++ assert resp.status_code == 500+ assert manager.get_agent(agent["id"])["status"] == "idle"@pytest.mark.skipif(not HAS_FASTAPI, reason="fastapi not installed")diff --git a/tests/server/test_deep_research_tools_wiring.py b/tests/server/test_deep_research_tools_wiring.pyindex 5de649af..7c96d46c 100644--- a/tests/server/test_deep_research_tools_wiring.py+++ b/tests/server/test_deep_research_tools_wiring.py@@ -181,6 +181,7 @@ async def test_server_deep_research_merges_and_executes_all_tool_sources(manager = MagicMock()manager.list_messages.return_value = []engine = _ScriptedDeepResearchEngine()+ on_complete = MagicMock()response = await routes._stream_managed_agent(manager=manager,@@ -199,11 +200,15 @@ async def test_server_deep_research_merges_and_executes_all_tool_sources(engine=engine,bus=None,app_state=app_state,+ on_complete=on_complete,)body_parts: list[str] = []async for part in response.body_iterator:body_parts.append(part.decode() if isinstance(part, bytes) else part)+ assert response.background is not None+ await response.background()+ on_complete.assert_called_once_with()expected_names = {_ConfiguredResearchProbe.tool_id,@@ -239,3 +244,74 @@ async def test_server_deep_research_merges_and_executes_all_tool_sources(assert persisted_calls[0]["tool"] == _ConfiguredResearchProbe.tool_idassert persisted_calls[0]["result"] == "configured:sentinel"assert persisted_calls[0]["success"] is True++[email protected](not HAS_FASTAPI, reason="fastapi not installed")+async def test_disconnected_research_stream_keeps_tick_until_worker_finishes(+ tmp_path: Path,+ monkeypatch,+) -> None:+ """A closed SSE response must not admit another still-running research tick."""+ import asyncio+ import threading++ from openjarvis.agents._stubs import AgentResult+ from openjarvis.agents.manager import AgentManager+ from openjarvis.server import agent_manager_routes as routes++ started = threading.Event()+ release = threading.Event()+ completed = threading.Event()+ toolkit = SimpleNamespace(+ instances=[_MCPResearchProbe()], mcp_clients=[], close=MagicMock()+ )++ class BlockedResearchAgent:+ def __init__(self, **kwargs):+ self._executor = SimpleNamespace(execute=MagicMock())++ def run(self, query):+ started.set()+ assert release.wait(5)+ return AgentResult(content="finished")++ monkeypatch.setattr(routes, "resolve_agent_tools", lambda *args, **kwargs: toolkit)+ monkeypatch.setattr(+ "openjarvis.agents.deep_research.DeepResearchAgent",+ BlockedResearchAgent,+ )+ manager = AgentManager(str(tmp_path / "agents.db"))+ agent = manager.create_agent(name="blocked", agent_type="deep_research")+ manager.start_tick(agent["id"])+ message = manager.send_message(agent["id"], "research")++ def finish_tick():+ manager.end_tick(agent["id"])+ completed.set()++ response = await routes._stream_managed_agent(+ manager=manager,+ agent_record=agent,+ user_content="research",+ message_id=message["id"],+ engine=MagicMock(),+ bus=None,+ app_state=SimpleNamespace(config=None),+ on_complete=finish_tick,+ )+ consumer = asyncio.create_task(anext(response.body_iterator))+ try:+ assert await asyncio.to_thread(started.wait, 5)+ consumer.cancel()+ with pytest.raises(asyncio.CancelledError):+ await consumer+ await response.background()+ assert not completed.is_set()+ with pytest.raises(ValueError, match="already executing"):+ manager.start_tick(agent["id"])+ finally:+ release.set()+ assert await asyncio.to_thread(completed.wait, 5)+ assert manager.get_agent(agent["id"])["status"] == "idle"+ manager.close()