Skip to content

Personal AI, On Personal Devices

Python

fix(agents): serialize ticks and preserve lifecycle ownership (#939)

Haotian Zheng56671aff7de6
diff --git a/src/openjarvis/agents/manager.py b/src/openjarvis/agents/manager.py
index 50ec03d7..e247bdaf 100644
--- a/src/openjarvis/agents/manager.py
+++ b/src/openjarvis/agents/manager.py
@@ -9,8 +9,10 @@ from __future__ import annotations
import json
import logging
import sqlite3
+import threading
import time
import uuid
+from functools import wraps
from typing import Any, Dict, List, Optional
from uuid import uuid4
@@ -95,6 +97,17 @@ CREATE TABLE IF NOT EXISTS agent_learning_log (
_SUMMARY_MAX = 16000
+def _db_locked(func):
+ """Serialize access to the connection shared by manager worker threads."""
+
+ @wraps(func)
+ def wrapped(self, *args, **kwargs):
+ with self._db_lock:
+ return func(self, *args, **kwargs)
+
+ return wrapped
+
+
class AgentManager:
"""Persistent agent lifecycle manager with SQLite backing."""
@@ -107,6 +120,7 @@ class AgentManager:
def __init__(self, db_path: str, *, clear_stale_running: bool = False) -> None:
self._db_path = str(db_path)
+ self._db_lock = threading.RLock()
self._conn = sqlite3.connect(self._db_path, check_same_thread=False)
self._conn.row_factory = sqlite3.Row
self._conn.execute("PRAGMA journal_mode=WAL")
@@ -149,6 +163,7 @@ class AgentManager:
if clear_stale_running:
self._clear_stale_running_state()
+ @_db_locked
def _clear_stale_running_state(self) -> None:
"""Reset any agent stuck in ``status='running'`` on startup.
@@ -175,11 +190,13 @@ class AgentManager:
cur.rowcount,
)
+ @_db_locked
def close(self) -> None:
self._conn.close()
# ── Agent CRUD ────────────────────────────────────────────────
+ @_db_locked
def create_agent(
self,
name: str,
@@ -200,6 +217,7 @@ class AgentManager:
self._conn.commit()
return self.get_agent(agent_id) # type: ignore[return-value]
+ @_db_locked
def list_agents(self, include_archived: bool = False) -> List[Dict[str, Any]]:
query = "SELECT * FROM managed_agents"
if not include_archived:
@@ -208,12 +226,14 @@ class AgentManager:
rows = self._conn.execute(query).fetchall()
return [self._row_to_agent(r) for r in rows]
+ @_db_locked
def get_agent(self, agent_id: str) -> Optional[Dict[str, Any]]:
row = self._conn.execute(
"SELECT * FROM managed_agents WHERE id = ?", (agent_id,)
).fetchone()
return self._row_to_agent(row) if row else None
+ @_db_locked
def update_agent(self, agent_id: str, **kwargs: Any) -> Dict[str, Any]:
sets: List[str] = []
vals: List[Any] = []
@@ -261,15 +281,19 @@ class AgentManager:
self._conn.commit()
return self.get_agent(agent_id) # type: ignore[return-value]
+ @_db_locked
def delete_agent(self, agent_id: str) -> None:
self._set_status(agent_id, "archived")
+ @_db_locked
def pause_agent(self, agent_id: str) -> None:
self._set_status(agent_id, "paused")
+ @_db_locked
def resume_agent(self, agent_id: str) -> None:
self._set_status(agent_id, "idle")
+ @_db_locked
def _set_status(self, agent_id: str, status: str) -> None:
self._conn.execute(
"UPDATE managed_agents SET status = ?, updated_at = ? WHERE id = ?",
@@ -279,6 +303,7 @@ class AgentManager:
# ── Tick concurrency guard ────────────────────────────────────
+ @_db_locked
def start_tick(self, agent_id: str) -> None:
"""Mark agent as running. Raises ValueError if already running.
@@ -286,23 +311,60 @@ class AgentManager:
without any update is treated as a zombie left by a dead worker and
overtaken rather than refused — otherwise a crash with no server
around to sweep it would wedge the agent forever.
+
+ The state transition is a single conditional SQLite update. The
+ read-before-write approach is racy when two workers start together:
+ both can observe ``idle`` and then mark the row ``running``. The
+ conditional update lets SQLite choose exactly one winner, including
+ when workers use separate connections or processes.
"""
- agent = self.get_agent(agent_id)
- if agent and agent["status"] == "running":
- age = time.time() - (agent.get("updated_at") or 0)
- if age < self._STALE_TICK_SECONDS:
- raise ValueError(f"Agent {agent_id} is already executing a tick")
- logger.warning(
- "Agent %s: overtaking stale tick lock (running, idle for %.0fs)",
- agent_id,
- age,
+ while True:
+ now = time.time()
+ stale_before = now - self._STALE_TICK_SECONDS
+ agent = self.get_agent(agent_id)
+ if agent is None:
+ return
+
+ was_stale = (
+ agent["status"] == "running"
+ and (agent.get("updated_at") or 0) <= stale_before
)
- self._set_status(agent_id, "running")
-
+ cursor = self._conn.execute(
+ "UPDATE managed_agents SET status = 'running', "
+ "updated_at = ? WHERE id = ? AND "
+ "(status != 'running' OR updated_at <= ?)",
+ (now, agent_id, stale_before),
+ )
+ self._conn.commit()
+ if cursor.rowcount:
+ if was_stale:
+ age = time.time() - (agent.get("updated_at") or 0)
+ logger.warning(
+ "Agent %s: overtaking stale tick lock "
+ "(running, idle for %.0fs)",
+ agent_id,
+ age,
+ )
+ return
+
+ # Another connection acquired the lock after our read. If that
+ # lock is still fresh, this caller loses; if it ended between the
+ # failed update and this read, retry so the caller never proceeds
+ # without owning the tick lock.
+ current = self.get_agent(agent_id)
+ if current is None:
+ return
+ if current["status"] == "running":
+ age = time.time() - (current.get("updated_at") or 0)
+ if age < self._STALE_TICK_SECONDS:
+ raise ValueError(f"Agent {agent_id} is already executing a tick")
+
+ @_db_locked
def end_tick(self, agent_id: str) -> None:
self._conn.execute(
"UPDATE managed_agents SET status = 'idle', "
- "current_activity = '', updated_at = ? WHERE id = ?",
+ "current_activity = '', updated_at = ? "
+ "WHERE id = ? AND status = 'running'",
(time.time(), agent_id),
)
self._conn.commit()
@@ -311,6 +373,7 @@ class AgentManager:
_CHECKPOINT_RETENTION = 5
+ @_db_locked
def save_checkpoint(
self,
agent_id: str,
@@ -348,6 +411,7 @@ class AgentManager:
"created_at": now,
}
+ @_db_locked
def list_checkpoints(self, agent_id: str) -> list:
rows = self._conn.execute(
"SELECT * FROM agent_checkpoints"
@@ -356,6 +420,7 @@ class AgentManager:
).fetchall()
return [self._row_to_checkpoint(r) for r in rows]
+ @_db_locked
def get_latest_checkpoint(self, agent_id: str) -> Optional[Dict[str, Any]]:
row = self._conn.execute(
"SELECT * FROM agent_checkpoints"
@@ -364,6 +429,7 @@ class AgentManager:
).fetchone()
return self._row_to_checkpoint(row) if row else None
+ @_db_locked
def recover_agent(self, agent_id: str) -> Optional[Dict[str, Any]]:
checkpoint = self.get_latest_checkpoint(agent_id)
# Always reset to idle — clearing the error state is the primary purpose
@@ -383,6 +449,7 @@ class AgentManager:
# ── Summary memory ────────────────────────────────────────────
+ @_db_locked
def update_summary_memory(self, agent_id: str, summary: str) -> None:
truncated = summary[:_SUMMARY_MAX]
self._conn.execute(
@@ -393,6 +460,7 @@ class AgentManager:
# ── Task CRUD ─────────────────────────────────────────────────
+ @_db_locked
def create_task(
self, agent_id: str, description: str, status: str = "pending"
) -> Dict[str, Any]:
@@ -406,6 +474,7 @@ class AgentManager:
self._conn.commit()
return self._get_task(task_id) # type: ignore[return-value]
+ @_db_locked
def list_tasks(
self, agent_id: str, status: Optional[str] = None
) -> List[Dict[str, Any]]:
@@ -418,6 +487,7 @@ class AgentManager:
rows = self._conn.execute(query, params).fetchall()
return [self._row_to_task(r) for r in rows]
+ @_db_locked
def update_task(self, task_id: str, **kwargs: Any) -> Dict[str, Any]:
sets: List[str] = []
vals: List[Any] = []
@@ -440,10 +510,12 @@ class AgentManager:
self._conn.commit()
return self._get_task(task_id) # type: ignore[return-value]
+ @_db_locked
def delete_task(self, task_id: str) -> None:
self._conn.execute("DELETE FROM agent_tasks WHERE id = ?", (task_id,))
self._conn.commit()
+ @_db_locked
def _get_task(self, task_id: str) -> Optional[Dict[str, Any]]:
row = self._conn.execute(
"SELECT * FROM agent_tasks WHERE id = ?", (task_id,)
@@ -452,6 +524,7 @@ class AgentManager:
# ── Channel bindings ──────────────────────────────────────────
+ @_db_locked
def bind_channel(
self,
agent_id: str,
@@ -471,22 +544,26 @@ class AgentManager:
self._conn.commit()
return self._get_binding(binding_id) # type: ignore[return-value]
+ @_db_locked
def list_channel_bindings(self, agent_id: str) -> List[Dict[str, Any]]:
rows = self._conn.execute(
"SELECT * FROM channel_bindings WHERE agent_id = ?", (agent_id,)
).fetchall()
return [self._row_to_binding(r) for r in rows]
+ @_db_locked
def unbind_channel(self, binding_id: str) -> None:
self._conn.execute("DELETE FROM channel_bindings WHERE id = ?", (binding_id,))
self._conn.commit()
+ @_db_locked
def _get_binding(self, binding_id: str) -> Optional[Dict[str, Any]]:
row = self._conn.execute(
"SELECT * FROM channel_bindings WHERE id = ?", (binding_id,)
).fetchone()
return self._row_to_binding(row) if row else None
+ @_db_locked
def find_binding_for_channel(
self, channel_type: str, channel_id: str
) -> Optional[Dict[str, Any]]:
@@ -568,6 +645,7 @@ class AgentManager:
# ── Message queue ─────────────────────────────────────────────
+ @_db_locked
def send_message(self, agent_id: str, content: str, mode: str = "queued") -> dict:
msg_id = uuid4().hex[:16]
now = time.time()
@@ -588,6 +666,7 @@ class AgentManager:
"created_at": now,
}
+ @_db_locked
def store_agent_response(
self,
agent_id: str,
@@ -623,6 +702,7 @@ class AgentManager:
"tool_calls": tool_calls or None,
}
+ @_db_locked
def list_messages(self, agent_id: str, limit: int = 50) -> list[dict]:
rows = self._conn.execute(
"SELECT * FROM agent_messages"
@@ -631,6 +711,7 @@ class AgentManager:
).fetchall()
return [self._row_to_message(r) for r in rows]
+ @_db_locked
def get_pending_messages(self, agent_id: str) -> list[dict]:
rows = self._conn.execute(
"SELECT * FROM agent_messages"
@@ -640,6 +721,7 @@ class AgentManager:
).fetchall()
return [self._row_to_message(r) for r in rows]
+ @_db_locked
def mark_message_delivered(self, message_id: str) -> None:
self._conn.execute(
"UPDATE agent_messages SET status = 'delivered' WHERE id = ?",
@@ -647,6 +729,7 @@ class AgentManager:
)
self._conn.commit()
+ @_db_locked
def add_agent_response(self, agent_id: str, content: str) -> dict:
msg_id = uuid4().hex[:16]
now = time.time()
@@ -692,6 +775,7 @@ class AgentManager:
# ── Learning log ──────────────────────────────────────────
+ @_db_locked
def add_learning_log(
self,
agent_id: str,
@@ -717,6 +801,7 @@ class AgentManager:
"created_at": now,
}
+ @_db_locked
def list_learning_log(self, agent_id: str, limit: int = 50) -> list[dict]:
rows = self._conn.execute(
"SELECT * FROM agent_learning_log"
diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py
index 4e501ba3..deb62e14 100644
--- a/src/openjarvis/server/agent_manager_routes.py
+++ b/src/openjarvis/server/agent_manager_routes.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import logging
import re as _re
import threading
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
from openjarvis.agents.manager import AgentManager
from openjarvis.agents.tool_resolver import (
@@ -852,12 +852,15 @@ async def _stream_managed_agent(
engine: Any,
bus: Any,
app_state: Any = None,
+ on_complete: Optional[Callable[[], None]] = None,
) -> StreamingResponse:
"""Run a managed agent with real LLM token streaming via SSE.
Uses ``engine.stream_full()`` to yield tokens as they arrive from the
LLM. Supports multi-turn tool-calling: when the model emits tool_calls,
- they are executed and the results fed back for the next turn.
+ they are executed and the results fed back for the next turn. When
+ ``on_complete`` is provided, it runs after either streaming branch has
+ finished its response-local cleanup.
"""
import json
import uuid
@@ -867,6 +870,21 @@ async def _stream_managed_agent(
from openjarvis.core.types import Message, Role
agent_id = agent_record["id"]
+
+ completion_lock = threading.Lock()
+ completion_called = False
+
+ def _complete_stream() -> None:
+ nonlocal completion_called
+ with completion_lock:
+ if completion_called or on_complete is None:
+ return
+ completion_called = True
+ try:
+ on_complete()
+ except Exception:
+ logger.warning("Managed agent stream cleanup failed", exc_info=True)
+
config = agent_record.get("config", {})
# Resolve the model: prefer the agent's own config, then the server's
# resolved model (app.state.model — what the engine was booted with),
@@ -950,6 +968,16 @@ async def _stream_managed_agent(
if agent_type == "deep_research":
dr_tools = resolved_toolkit.instances
if dr_tools:
+ worker_owns_tick = threading.Event()
+
+ def _complete_research_response() -> None:
+ # A disconnected/timed-out response cannot release a tick
+ # while its synchronous research worker is still executing.
+ if not worker_owns_tick.is_set():
+ try:
+ resolved_toolkit.close()
+ finally:
+ _complete_stream()
async def generate_deep_research():
"""Run DeepResearchAgent in thread, stream progress + result."""
@@ -1091,14 +1119,22 @@ async def _stream_managed_agent(
}
)
+ def _run_with_tick_cleanup():
+ try:
+ _run_agent()
+ finally:
+ _complete_stream()
+
+ worker_owns_tick.set()
try:
_start_managed_worker(
app_state,
- _run_agent,
+ _run_with_tick_cleanup,
name=f"managed-agent-deep-research-{agent_id}",
)
except Exception as exc:
- resolved_toolkit.close()
+ worker_owns_tick.clear()
+ _complete_research_response()
progress_q.put(
{
"type": "error",
@@ -1231,13 +1267,21 @@ async def _stream_managed_agent(
)
break
+ async def guarded_deep_research():
+ try:
+ async for chunk in generate_deep_research():
+ yield chunk
+ finally:
+ _complete_research_response()
+
return StreamingResponse(
- generate_deep_research(),
+ guarded_deep_research(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
},
+ background=BackgroundTask(_complete_research_response),
)
# The canonical resolver exposes the same live instances as OpenAI specs
@@ -1306,7 +1350,10 @@ async def _stream_managed_agent(
try:
_persist_final()
finally:
- resolved_toolkit.close()
+ try:
+ resolved_toolkit.close()
+ finally:
+ _complete_stream()
async def generate():
"""Async generator yielding SSE-formatted chunks with real token streaming."""
@@ -1991,8 +2038,29 @@ def create_agent_manager_router(
):
manager.update_agent(agent_id, status="idle")
+ stream_tick_acquired = False
+ stream_engine = None
+ if req.stream:
+ stream_engine = getattr(request.app.state, "engine", None)
+ if stream_engine is not None:
+ # Unlike the queued/immediate paths, streaming runs directly
+ # in the request, so it must acquire the same tick guard here.
+ try:
+ manager.start_tick(agent_id)
+ stream_tick_acquired = True
+ except ValueError:
+ raise HTTPException(
+ status_code=409,
+ detail="Agent is already running",
+ )
+
# Store user message in DB (always, regardless of stream mode)
- msg = manager.send_message(agent_id, req.content, mode=req.mode)
+ try:
+ msg = manager.send_message(agent_id, req.content, mode=req.mode)
+ except Exception:
+ if stream_tick_acquired:
+ manager.end_tick(agent_id)
+ raise
if not req.stream and req.mode != "immediate":
return msg
@@ -2072,7 +2140,7 @@ def create_agent_manager_router(
return msg
# --- Streaming mode: run agent and return SSE response ---
- engine = getattr(request.app.state, "engine", None)
+ engine = stream_engine
bus = getattr(request.app.state, "bus", None)
if engine is None:
raise HTTPException(
@@ -2080,15 +2148,21 @@ def create_agent_manager_router(
detail="Engine not available for streaming",
)
- return await _stream_managed_agent(
- manager=manager,
- agent_record=agent_record,
- user_content=req.content,
- message_id=msg["id"],
- engine=engine,
- bus=bus,
- app_state=request.app.state,
- )
+ try:
+ return await _stream_managed_agent(
+ manager=manager,
+ agent_record=agent_record,
+ user_content=req.content,
+ message_id=msg["id"],
+ engine=engine,
+ bus=bus,
+ app_state=request.app.state,
+ on_complete=lambda: manager.end_tick(agent_id),
+ )
+ except Exception:
+ if stream_tick_acquired:
+ manager.end_tick(agent_id)
+ raise
# ── State inspection ─────────────────────────────────────
diff --git a/tests/agents/test_manager.py b/tests/agents/test_manager.py
index ea51d111..005fc887 100644
--- a/tests/agents/test_manager.py
+++ b/tests/agents/test_manager.py
@@ -2,7 +2,10 @@
from __future__ import annotations
+import concurrent.futures
import tempfile
+import threading
+import time
from pathlib import Path
import pytest
@@ -167,6 +170,142 @@ class TestConcurrency:
with pytest.raises(ValueError, match="already executing"):
manager.start_tick(agent["id"])
+ def test_start_tick_is_atomic_across_connections(self, tmp_path, monkeypatch):
+ """Only one caller may acquire a tick when two connections race."""
+ from openjarvis.agents.manager import AgentManager
+
+ db_path = tmp_path / "agents.db"
+ first = AgentManager(str(db_path))
+ second = AgentManager(str(db_path))
+ try:
+ agent = first.create_agent(name="racing", agent_type="simple")
+
+ # The old implementation read the idle row before calling
+ # _set_status(). Holding both writers at that boundary makes the
+ # check-then-update race deterministic on the unfixed code.
+ writers_ready = threading.Barrier(2)
+ original_set_status = AgentManager._set_status
+
+ def hold_running_writes(self, agent_id, status):
+ if status == "running":
+ writers_ready.wait(timeout=5)
+ return original_set_status(self, agent_id, status)
+
+ monkeypatch.setattr(AgentManager, "_set_status", hold_running_writes)
+
+ managers = (first, second)
+ with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+ futures = [
+ pool.submit(current.start_tick, agent["id"]) for current in managers
+ ]
+ outcomes = []
+ for future in futures:
+ try:
+ future.result(timeout=5)
+ except ValueError as exc:
+ outcomes.append(str(exc))
+ else:
+ outcomes.append("success")
+
+ assert outcomes.count("success") == 1
+ assert sum("already executing" in outcome for outcome in outcomes) == 1
+ assert first.get_agent(agent["id"])["status"] == "running"
+ finally:
+ first.close()
+ second.close()
+
+ def test_start_tick_serializes_its_uncommitted_update(self, manager):
+ """Shared-connection callers cannot observe or commit a partial tick."""
+ agent = manager.create_agent(name="pending-tick", agent_type="simple")
+ original_connection = manager._conn
+ update_pending = threading.Event()
+ allow_commit = threading.Event()
+ reader_started = threading.Event()
+ reader_finished = threading.Event()
+
+ class PausedCommit:
+ def __getattr__(self, name):
+ return getattr(original_connection, name)
+
+ def commit(self):
+ update_pending.set()
+ assert allow_commit.wait(5)
+ return original_connection.commit()
+
+ manager._conn = PausedCommit()
+
+ def read_agent():
+ reader_started.set()
+ try:
+ return manager.get_agent(agent["id"])
+ finally:
+ reader_finished.set()
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+ tick = pool.submit(manager.start_tick, agent["id"])
+ assert update_pending.wait(5)
+ reader = pool.submit(read_agent)
+ assert reader_started.wait(5)
+ try:
+ assert not reader_finished.wait(0.1), "tick leaked uncommitted state"
+ finally:
+ allow_commit.set()
+ tick.result(timeout=5)
+ assert reader.result(timeout=5)["status"] == "running"
+
+ def test_parallel_ticks_on_distinct_agents_share_one_connection(self, manager):
+ """Independent agents must not interfere through SQLite transactions."""
+ agents = [
+ manager.create_agent(name=f"parallel-{index}", agent_type="simple")
+ for index in range(8)
+ ]
+ ready = threading.Barrier(len(agents))
+
+ def tick_many(agent):
+ ready.wait(timeout=5)
+ for _ in range(50):
+ manager.start_tick(agent["id"])
+ manager.end_tick(agent["id"])
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=len(agents)) as pool:
+ futures = [pool.submit(tick_many, agent) for agent in agents]
+ for future in futures:
+ future.result(timeout=10)
+ assert all(
+ manager.get_agent(agent["id"])["status"] == "idle" for agent in agents
+ )
+
+ def test_start_tick_overtakes_stale_lock(self, manager):
+ agent = manager.create_agent(name="stale", agent_type="simple")
+ stale_at = time.time() - manager._STALE_TICK_SECONDS - 1
+ manager._conn.execute(
+ "UPDATE managed_agents SET status = 'running', updated_at = ? WHERE id = ?",
+ (stale_at, agent["id"]),
+ )
+ manager._conn.commit()
+
+ manager.start_tick(agent["id"])
+
+ refreshed = manager.get_agent(agent["id"])
+ assert refreshed["status"] == "running"
+ assert refreshed["updated_at"] > stale_at
+
+ @pytest.mark.parametrize(
+ ("change_status", "expected_status"),
+ [("pause_agent", "paused"), ("delete_agent", "archived")],
+ )
+ def test_end_tick_preserves_user_status_change(
+ self, manager, change_status, expected_status
+ ):
+ """A late tick cleanup must not undo a user's pause or archive action."""
+ agent = manager.create_agent(name="lifecycle", agent_type="simple")
+ manager.start_tick(agent["id"])
+
+ getattr(manager, change_status)(agent["id"])
+ manager.end_tick(agent["id"])
+
+ assert manager.get_agent(agent["id"])["status"] == expected_status
+
class TestCheckpoints:
def test_save_checkpoint(self, manager):
@@ -373,3 +512,23 @@ class TestSchemaAndThreading:
t.join(timeout=5)
assert len(results) == 1
assert results[0]["name"] == "threaded"
+
+ def test_concurrent_updates_are_serialized(self, manager):
+ """Concurrent updates on the shared manager connection must not be lost."""
+ from concurrent.futures import ThreadPoolExecutor
+
+ agent = manager.create_agent(name="concurrent", agent_type="simple")
+ workers = 8
+ updates_per_worker = 20
+
+ def update_many():
+ for _ in range(updates_per_worker):
+ manager.update_agent(agent["id"], total_tokens_increment=1)
+
+ with ThreadPoolExecutor(max_workers=workers) as pool:
+ futures = [pool.submit(update_many) for _ in range(workers)]
+ for future in futures:
+ future.result()
+
+ updated = manager.get_agent(agent["id"])
+ assert updated["total_tokens"] == workers * updates_per_worker
diff --git a/tests/server/test_agent_manager_routes.py b/tests/server/test_agent_manager_routes.py
index 895b1651..9d3aab54 100644
--- a/tests/server/test_agent_manager_routes.py
+++ b/tests/server/test_agent_manager_routes.py
@@ -487,6 +487,105 @@ class TestAgentManagerStreaming:
assert resp.status_code == 200
assert "Error:" in resp.text or "error" in resp.text.lower()
assert "data: [DONE]" in resp.text
+ assert manager.get_agent(agent["id"])["status"] == "idle"
+
+ def test_streaming_requests_share_tick_guard(self, manager, tmp_path, monkeypatch):
+ """A second stream for the same agent must not start another tick."""
+ import asyncio
+
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient
+
+ from openjarvis.engine._stubs import StreamChunk
+ from openjarvis.server.agent_manager_routes import (
+ create_agent_manager_router,
+ )
+
+ class SlowEngine:
+ engine_id = "slow"
+ _model = "test-model"
+
+ def __init__(self):
+ self.calls = 0
+ self.lock = threading.Lock()
+ self.first_started = threading.Event()
+ self.second_started = threading.Event()
+ self.release = threading.Event()
+
+ async def stream_full(self, messages, *, model, **kwargs):
+ with self.lock:
+ self.calls += 1
+ call = self.calls
+ if call == 1:
+ self.first_started.set()
+ else:
+ self.second_started.set()
+ while not self.release.is_set():
+ await asyncio.sleep(0.01)
+ yield StreamChunk(content=f"response-{call}")
+ yield StreamChunk(finish_reason="stop")
+
+ monkeypatch.setenv("OPENJARVIS_HOME", str(tmp_path / "runtime"))
+ engine = SlowEngine()
+ app = FastAPI()
+ app.state.engine = engine
+ app.state.bus = None
+ for router in create_agent_manager_router(manager):
+ app.include_router(router)
+
+ agent = manager.create_agent(name="stream-race", agent_type="simple")
+ url = f"/v1/managed-agents/{agent['id']}/messages"
+
+ def send(client):
+ return client.post(url, json={"content": "hello", "stream": True})
+
+ client1 = TestClient(app)
+ client2 = TestClient(app)
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ first = pool.submit(send, client1)
+ assert engine.first_started.wait(5)
+ second = pool.submit(send, client2)
+ try:
+ assert not engine.second_started.wait(0.5)
+ second_response = second.result(timeout=5)
+ assert second_response.status_code == 409
+ finally:
+ engine.release.set()
+
+ first_response = first.result(timeout=10)
+
+ assert first_response.status_code == 200
+ assert engine.calls == 1
+ assert manager.get_agent(agent["id"])["status"] == "idle"
+
+ def test_stream_initialization_failure_releases_tick_guard(self, manager):
+ """A stream setup error must not leave the agent marked as running."""
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient as TC
+
+ from openjarvis.server.agent_manager_routes import (
+ create_agent_manager_router,
+ )
+
+ app = FastAPI()
+ app.state.engine = MagicMock()
+ app.state.bus = None
+ for router in create_agent_manager_router(manager):
+ app.include_router(router)
+ client = TC(app, raise_server_exceptions=False)
+
+ agent = manager.create_agent(name="stream-init-error", agent_type="simple")
+ with patch(
+ "openjarvis.server.agent_manager_routes._stream_managed_agent",
+ new=AsyncMock(side_effect=RuntimeError("stream setup failed")),
+ ):
+ resp = client.post(
+ f"/v1/managed-agents/{agent['id']}/messages",
+ json={"content": "fail during setup", "stream": True},
+ )
+
+ assert resp.status_code == 500
+ assert manager.get_agent(agent["id"])["status"] == "idle"
@pytest.mark.skipif(not HAS_FASTAPI, reason="fastapi not installed")
diff --git a/tests/server/test_deep_research_tools_wiring.py b/tests/server/test_deep_research_tools_wiring.py
index 5de649af..7c96d46c 100644
--- a/tests/server/test_deep_research_tools_wiring.py
+++ b/tests/server/test_deep_research_tools_wiring.py
@@ -181,6 +181,7 @@ async def test_server_deep_research_merges_and_executes_all_tool_sources(
manager = MagicMock()
manager.list_messages.return_value = []
engine = _ScriptedDeepResearchEngine()
+ on_complete = MagicMock()
response = await routes._stream_managed_agent(
manager=manager,
@@ -199,11 +200,15 @@ async def test_server_deep_research_merges_and_executes_all_tool_sources(
engine=engine,
bus=None,
app_state=app_state,
+ on_complete=on_complete,
)
body_parts: list[str] = []
async for part in response.body_iterator:
body_parts.append(part.decode() if isinstance(part, bytes) else part)
+ assert response.background is not None
+ await response.background()
+ on_complete.assert_called_once_with()
expected_names = {
_ConfiguredResearchProbe.tool_id,
@@ -239,3 +244,74 @@ async def test_server_deep_research_merges_and_executes_all_tool_sources(
assert persisted_calls[0]["tool"] == _ConfiguredResearchProbe.tool_id
assert persisted_calls[0]["result"] == "configured:sentinel"
assert persisted_calls[0]["success"] is True
+
+
[email protected](not HAS_FASTAPI, reason="fastapi not installed")
+async def test_disconnected_research_stream_keeps_tick_until_worker_finishes(
+ tmp_path: Path,
+ monkeypatch,
+) -> None:
+ """A closed SSE response must not admit another still-running research tick."""
+ import asyncio
+ import threading
+
+ from openjarvis.agents._stubs import AgentResult
+ from openjarvis.agents.manager import AgentManager
+ from openjarvis.server import agent_manager_routes as routes
+
+ started = threading.Event()
+ release = threading.Event()
+ completed = threading.Event()
+ toolkit = SimpleNamespace(
+ instances=[_MCPResearchProbe()], mcp_clients=[], close=MagicMock()
+ )
+
+ class BlockedResearchAgent:
+ def __init__(self, **kwargs):
+ self._executor = SimpleNamespace(execute=MagicMock())
+
+ def run(self, query):
+ started.set()
+ assert release.wait(5)
+ return AgentResult(content="finished")
+
+ monkeypatch.setattr(routes, "resolve_agent_tools", lambda *args, **kwargs: toolkit)
+ monkeypatch.setattr(
+ "openjarvis.agents.deep_research.DeepResearchAgent",
+ BlockedResearchAgent,
+ )
+ manager = AgentManager(str(tmp_path / "agents.db"))
+ agent = manager.create_agent(name="blocked", agent_type="deep_research")
+ manager.start_tick(agent["id"])
+ message = manager.send_message(agent["id"], "research")
+
+ def finish_tick():
+ manager.end_tick(agent["id"])
+ completed.set()
+
+ response = await routes._stream_managed_agent(
+ manager=manager,
+ agent_record=agent,
+ user_content="research",
+ message_id=message["id"],
+ engine=MagicMock(),
+ bus=None,
+ app_state=SimpleNamespace(config=None),
+ on_complete=finish_tick,
+ )
+ consumer = asyncio.create_task(anext(response.body_iterator))
+ try:
+ assert await asyncio.to_thread(started.wait, 5)
+ consumer.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await consumer
+ await response.background()
+ assert not completed.is_set()
+ with pytest.raises(ValueError, match="already executing"):
+ manager.start_tick(agent["id"])
+ finally:
+ release.set()
+ assert await asyncio.to_thread(completed.wait, 5)
+ assert manager.get_agent(agent["id"])["status"] == "idle"
+ manager.close()