Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(society): keep archive storage off the desktop event loop (#266)
Ruben Lütkef83b924c0ef5
diff --git a/jarvis/society/conversation.py b/jarvis/society/conversation.pyindex 4ce8534617..03983b511f 100644--- a/jarvis/society/conversation.py+++ b/jarvis/society/conversation.py@@ -6,6 +6,7 @@ reads and a parameterized text-search fallback on every supported OS.from __future__ import annotations+import asyncioimport jsonimport loggingimport re@@ -30,11 +31,40 @@ def event_text(event: dict[str, Any]) -> str:class ConversationArchive:- def __init__(self, path: Path) -> None:- path.parent.mkdir(parents=True, exist_ok=True)- self._db = sqlite3.connect(path, check_same_thread=False)- self._db.row_factory = sqlite3.Row+ def __init__(self, path: Path, *, defer_open: bool = False) -> None:+ self._path = pathself._lock = threading.RLock()+ self._connection: sqlite3.Connection | None = None+ self._closed = False+ self.fts_available = False+ if not defer_open:+ self.open()++ @property+ def _db(self) -> sqlite3.Connection:+ self.open()+ assert self._connection is not None+ return self._connection++ def open(self) -> None:+ """Initialize storage once. Async owners must call this off their loop."""+ with self._lock:+ if self._closed:+ raise sqlite3.ProgrammingError("The conversation archive is closed")+ if self._connection is not None:+ return+ self._path.parent.mkdir(parents=True, exist_ok=True)+ self._connection = sqlite3.connect(self._path, check_same_thread=False)+ self._connection.row_factory = sqlite3.Row+ try:+ self._initialize_schema()+ except BaseException:+ self._connection.close()+ self._connection = None+ self.fts_available = False+ raise++ def _initialize_schema(self) -> None:self._db.executescript("""PRAGMA journal_mode=WAL;CREATE TABLE IF NOT EXISTS messages (@@ -62,7 +92,10 @@ class ConversationArchive:def close(self) -> None:with self._lock:- self._db.close()+ self._closed = True+ if self._connection is not None:+ self._connection.close()+ self._connection = Nonedef ingest(self, session: str, events: list[dict[str, Any]]) -> None:with self._lock, self._db:@@ -86,6 +119,7 @@ class ConversationArchive:return []limit = max(1, min(20, limit))with self._lock:+ self.open()if self.fts_available:expression = " OR ".join('"' + w + '"' for w in words[:32])rows = self._db.execute(@@ -210,11 +244,11 @@ async def prepare_history(archive = runtime.conversationssid = session.session_id- archive.ingest(sid, events)+ await asyncio.to_thread(archive.ingest, sid, events)window = max(1024, int(getattr(provider, "context_window", 0) or 32768))# Reserve room for the system/tool surface, current request and generated output.budget = max(1024, window * 3 - prompt_chars - len(query))- through, summary = archive.checkpoint(sid)+ through, summary = await asyncio.to_thread(archive.checkpoint, sid)remaining = [e for e in events if int(e.get("seq") or 0) > through]size = sum(len(event_text(e)) + 32 for e in remaining)if size + len(summary) > budget or len(summary) > budget // 3:@@ -290,7 +324,7 @@ async def prepare_history(raise ValueError("An empty summary cannot replace conversation context")if boundary is not None:through = boundary- archive.save_checkpoint(sid, through, summary)+ await asyncio.to_thread(archive.save_checkpoint, sid, through, summary)remaining = [e for e in events if int(e.get("seq") or 0) > through]if retained_request is not None and int(retained_request.get("seq") or 0) <= through:# Keep the last human request verbatim even if its unusually long@@ -298,7 +332,7 @@ async def prepare_history(remaining.insert(0, retained_request)history = brain_history_from_events(remaining, max_messages=None)if summary:- recalled = archive.search(sid, query, limit=4)+ recalled = await asyncio.to_thread(archive.search, sid, query, limit=4)excerpts = "\n".join(f"{h['source']}: {h['text'][:1500]}" for h in recalled if h["seq"] <= through)diff --git a/jarvis/society/conversation_tool.py b/jarvis/society/conversation_tool.pyindex 7e388466fe..9a696ece0d 100644--- a/jarvis/society/conversation_tool.py+++ b/jarvis/society/conversation_tool.py@@ -2,6 +2,7 @@from __future__ import annotations+import asynciofrom typing import Anyfrom jarvis.core.protocols import ToolResult@@ -35,14 +36,14 @@ class ConversationRecallTool:service = self._runtime.chat_service()archive = self._runtime.conversationsif service is not None:- archive.ingest(session, service.store.list_events(session))+ events = await asyncio.to_thread(service.store.list_events, session)+ await asyncio.to_thread(archive.ingest, session, events)limit = max(1, min(20, int(args.get("limit") or 5)))if str(args.get("query") or "").strip():- return ToolResult(- True, {"hits": archive.search(session, str(args["query"]), limit=limit)}- )- events = archive.read(- session, after_seq=max(0, int(args.get("after_seq") or 0)), limit=limit+ hits = await asyncio.to_thread(archive.search, session, str(args["query"]), limit=limit)+ return ToolResult(True, {"hits": hits})+ events = await asyncio.to_thread(+ archive.read, session, after_seq=max(0, int(args.get("after_seq") or 0)), limit=limit)return ToolResult(True, {"events": events, "next_after_seq": events[-1]["seq"] if events else None}diff --git a/jarvis/society/runtime.py b/jarvis/society/runtime.pyindex 29bfcdedba..df2ddf3585 100644--- a/jarvis/society/runtime.py+++ b/jarvis/society/runtime.py@@ -250,7 +250,9 @@ class SocietyRuntime:self._publish_event = event_publish#: The society's one memory service; every touch moves the figure to the Memory House.self.memory = SocietyMemory(self, on_activity=self.checkpoints.note_memory_activity)- self.conversations = ConversationArchive(self._data_dir / "society-conversations.db")+ self.conversations = ConversationArchive(+ self._data_dir / "society-conversations.db", defer_open=True+ )self._review_lock = asyncio.Lock()#: The Quest Board: the person's jobs, routed to one taker, read back off the board.self.quests = Quests(self)@@ -329,6 +331,10 @@ class SocietyRuntime:async def _start_runtime(self) -> SocietyRuntime:await self.store.open()self._require_open_owner()+ # FTS backfill can take seconds on a busy disk. It must never stall+ # HTTP, microphone controls or the first response from the desktop.+ await asyncio.to_thread(self.conversations.open)+ self._require_open_owner()self.scheduler._budget = self._get_budget() # noqa: SLF001 — the runtime owns its schedulerself.scheduler.attach()self._delivery_unsubscribe = self.store.bus.subscribe_all(self._delivery_failed)@@ -556,13 +562,13 @@ class SocietyRuntime:async with AsyncExitStack() as cleanup:cleanup.callback(clear_runtime)cleanup.push_async_callback(self.store.close)+ cleanup.push_async_callback(asyncio.to_thread, self.conversations.close)for release in (self.world_feed.detach,self.quests.detach,self.checkpoints.detach,self.bridge.detach,self.scheduler.detach,- self.conversations.close,):cleanup.callback(release)cleanup.push_async_callback(self.browser.close)@@ -606,7 +612,7 @@ class SocietyRuntime:import jsonevents = json.loads(completion.events_json)- self.conversations.ingest(session.session_id, events)+ await asyncio.to_thread(self.conversations.ingest, session.session_id, events)await self._complete_message_reply(session, completion, events)terminal = [e for e in events if e.get("kind") == "turn_finished"]if not terminal or terminal[-1].get("payload", {}).get("status") not in {@@ -631,7 +637,8 @@ class SocietyRuntime:# Reviewing them again wastes a model call and can# duplicate a standing instruction as a conflicting memory.return- if self.conversations.queue_review(+ if await asyncio.to_thread(+ self.conversations.queue_review,session.session_id,completion.turn.turn_id,events,diff --git a/tests/unit/society/test_conversation_responsiveness.py b/tests/unit/society/test_conversation_responsiveness.pynew file mode 100644index 0000000000..f7a90ea1fd--- /dev/null+++ b/tests/unit/society/test_conversation_responsiveness.py@@ -0,0 +1,100 @@+"""Busy conversation storage must not freeze the desktop's serving loop."""+from __future__ import annotations++import asyncio+import sqlite3+import threading++import pytest++from jarvis.society.conversation import ConversationArchive+from jarvis.society.runtime import SocietyRuntime+++def test_deferred_archive_does_not_touch_storage_until_used(tmp_path):+ path = tmp_path / "unused" / "archive.db"+ archive = ConversationArchive(path, defer_open=True)+ assert not path.parent.exists()+ archive.open()+ try:+ archive.ingest(+ "agent-a", [{"seq": 1, "kind": "user_message", "payload": {"text": "hello"}}]+ )+ assert archive.search("agent-a", "hello")[0]["seq"] == 1+ assert archive.search("agent-b", "hello") == []+ finally:+ archive.close()+++def test_closed_deferred_archive_cannot_create_storage(tmp_path):+ path = tmp_path / "unused" / "archive.db"+ archive = ConversationArchive(path, defer_open=True)+ archive.close()+ archive.close()+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):+ archive.open()+ assert not path.parent.exists()+++def test_failed_initialization_releases_connection_and_can_retry(tmp_path, monkeypatch):+ archive = ConversationArchive(tmp_path / "archive.db", defer_open=True)+ initialize = archive._initialize_schema+ attempts = 0++ def initialize_once():+ nonlocal attempts+ attempts += 1+ if attempts == 1:+ raise sqlite3.OperationalError("test storage unavailable")+ initialize()++ monkeypatch.setattr(archive, "_initialize_schema", initialize_once)+ with pytest.raises(sqlite3.OperationalError):+ archive.open()+ assert archive._connection is None+ archive.open()+ try:+ assert archive.read("agent-a") == []+ finally:+ archive.close()+++async def test_slow_archive_start_and_cancelled_cleanup_keep_loop_responsive(tmp_path, monkeypatch):+ loop_thread = threading.get_ident()+ entered = threading.Event()+ release = threading.Event()+ initialize = ConversationArchive._initialize_schema++ def slow_schema(self):+ assert threading.get_ident() != loop_thread+ entered.set()+ assert release.wait(10), "test must release the database initializer"+ initialize(self)++ monkeypatch.setattr(ConversationArchive, "_initialize_schema", slow_schema)+ runtime = SocietyRuntime(tmp_path)+ assert not (tmp_path / "society-conversations.db").exists()+ startup = asyncio.create_task(runtime.ensure_started())+ closing = None+ try:+ assert await asyncio.to_thread(entered.wait, 5)+ heartbeat = asyncio.Event()+ asyncio.get_running_loop().call_soon(heartbeat.set)+ await asyncio.wait_for(heartbeat.wait(), 1)+ startup.cancel()+ with pytest.raises(asyncio.CancelledError):+ await startup+ closing = asyncio.create_task(runtime.close())+ heartbeat.clear()+ asyncio.get_running_loop().call_soon(heartbeat.set)+ await asyncio.wait_for(heartbeat.wait(), 1)+ finally:+ release.set()+ await asyncio.gather(startup, return_exceptions=True)+ if closing is not None:+ await closing+ else:+ await runtime.close()+ assert runtime.conversations._connection is None+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):+ runtime.conversations.open()