Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(society): keep archive storage off the desktop event loop (#266)

Ruben Lütkef83b924c0ef5
diff --git a/jarvis/society/conversation.py b/jarvis/society/conversation.py
index 4ce8534617..03983b511f 100644
--- a/jarvis/society/conversation.py
+++ b/jarvis/society/conversation.py
@@ -6,6 +6,7 @@ reads and a parameterized text-search fallback on every supported OS.
from __future__ import annotations
+import asyncio
import json
import logging
import re
@@ -30,11 +31,40 @@ def event_text(event: dict[str, Any]) -> str:
class ConversationArchive:
- def __init__(self, path: Path) -> None:
- path.parent.mkdir(parents=True, exist_ok=True)
- self._db = sqlite3.connect(path, check_same_thread=False)
- self._db.row_factory = sqlite3.Row
+ def __init__(self, path: Path, *, defer_open: bool = False) -> None:
+ self._path = path
self._lock = threading.RLock()
+ self._connection: sqlite3.Connection | None = None
+ self._closed = False
+ self.fts_available = False
+ if not defer_open:
+ self.open()
+
+ @property
+ def _db(self) -> sqlite3.Connection:
+ self.open()
+ assert self._connection is not None
+ return self._connection
+
+ def open(self) -> None:
+ """Initialize storage once. Async owners must call this off their loop."""
+ with self._lock:
+ if self._closed:
+ raise sqlite3.ProgrammingError("The conversation archive is closed")
+ if self._connection is not None:
+ return
+ self._path.parent.mkdir(parents=True, exist_ok=True)
+ self._connection = sqlite3.connect(self._path, check_same_thread=False)
+ self._connection.row_factory = sqlite3.Row
+ try:
+ self._initialize_schema()
+ except BaseException:
+ self._connection.close()
+ self._connection = None
+ self.fts_available = False
+ raise
+
+ def _initialize_schema(self) -> None:
self._db.executescript("""
PRAGMA journal_mode=WAL;
CREATE TABLE IF NOT EXISTS messages (
@@ -62,7 +92,10 @@ class ConversationArchive:
def close(self) -> None:
with self._lock:
- self._db.close()
+ self._closed = True
+ if self._connection is not None:
+ self._connection.close()
+ self._connection = None
def ingest(self, session: str, events: list[dict[str, Any]]) -> None:
with self._lock, self._db:
@@ -86,6 +119,7 @@ class ConversationArchive:
return []
limit = max(1, min(20, limit))
with self._lock:
+ self.open()
if self.fts_available:
expression = " OR ".join('"' + w + '"' for w in words[:32])
rows = self._db.execute(
@@ -210,11 +244,11 @@ async def prepare_history(
archive = runtime.conversations
sid = session.session_id
- archive.ingest(sid, events)
+ await asyncio.to_thread(archive.ingest, sid, events)
window = max(1024, int(getattr(provider, "context_window", 0) or 32768))
# Reserve room for the system/tool surface, current request and generated output.
budget = max(1024, window * 3 - prompt_chars - len(query))
- through, summary = archive.checkpoint(sid)
+ through, summary = await asyncio.to_thread(archive.checkpoint, sid)
remaining = [e for e in events if int(e.get("seq") or 0) > through]
size = sum(len(event_text(e)) + 32 for e in remaining)
if size + len(summary) > budget or len(summary) > budget // 3:
@@ -290,7 +324,7 @@ async def prepare_history(
raise ValueError("An empty summary cannot replace conversation context")
if boundary is not None:
through = boundary
- archive.save_checkpoint(sid, through, summary)
+ await asyncio.to_thread(archive.save_checkpoint, sid, through, summary)
remaining = [e for e in events if int(e.get("seq") or 0) > through]
if retained_request is not None and int(retained_request.get("seq") or 0) <= through:
# Keep the last human request verbatim even if its unusually long
@@ -298,7 +332,7 @@ async def prepare_history(
remaining.insert(0, retained_request)
history = brain_history_from_events(remaining, max_messages=None)
if summary:
- recalled = archive.search(sid, query, limit=4)
+ recalled = await asyncio.to_thread(archive.search, sid, query, limit=4)
excerpts = "\n".join(
f"{h['source']}: {h['text'][:1500]}" for h in recalled if h["seq"] <= through
)
diff --git a/jarvis/society/conversation_tool.py b/jarvis/society/conversation_tool.py
index 7e388466fe..9a696ece0d 100644
--- a/jarvis/society/conversation_tool.py
+++ b/jarvis/society/conversation_tool.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import asyncio
from typing import Any
from jarvis.core.protocols import ToolResult
@@ -35,14 +36,14 @@ class ConversationRecallTool:
service = self._runtime.chat_service()
archive = self._runtime.conversations
if service is not None:
- archive.ingest(session, service.store.list_events(session))
+ events = await asyncio.to_thread(service.store.list_events, session)
+ await asyncio.to_thread(archive.ingest, session, events)
limit = max(1, min(20, int(args.get("limit") or 5)))
if str(args.get("query") or "").strip():
- return ToolResult(
- True, {"hits": archive.search(session, str(args["query"]), limit=limit)}
- )
- events = archive.read(
- session, after_seq=max(0, int(args.get("after_seq") or 0)), limit=limit
+ hits = await asyncio.to_thread(archive.search, session, str(args["query"]), limit=limit)
+ return ToolResult(True, {"hits": hits})
+ events = await asyncio.to_thread(
+ archive.read, session, after_seq=max(0, int(args.get("after_seq") or 0)), limit=limit
)
return ToolResult(
True, {"events": events, "next_after_seq": events[-1]["seq"] if events else None}
diff --git a/jarvis/society/runtime.py b/jarvis/society/runtime.py
index 29bfcdedba..df2ddf3585 100644
--- a/jarvis/society/runtime.py
+++ b/jarvis/society/runtime.py
@@ -250,7 +250,9 @@ class SocietyRuntime:
self._publish_event = event_publish
#: The society's one memory service; every touch moves the figure to the Memory House.
self.memory = SocietyMemory(self, on_activity=self.checkpoints.note_memory_activity)
- self.conversations = ConversationArchive(self._data_dir / "society-conversations.db")
+ self.conversations = ConversationArchive(
+ self._data_dir / "society-conversations.db", defer_open=True
+ )
self._review_lock = asyncio.Lock()
#: The Quest Board: the person's jobs, routed to one taker, read back off the board.
self.quests = Quests(self)
@@ -329,6 +331,10 @@ class SocietyRuntime:
async def _start_runtime(self) -> SocietyRuntime:
await self.store.open()
self._require_open_owner()
+ # FTS backfill can take seconds on a busy disk. It must never stall
+ # HTTP, microphone controls or the first response from the desktop.
+ await asyncio.to_thread(self.conversations.open)
+ self._require_open_owner()
self.scheduler._budget = self._get_budget() # noqa: SLF001 — the runtime owns its scheduler
self.scheduler.attach()
self._delivery_unsubscribe = self.store.bus.subscribe_all(self._delivery_failed)
@@ -556,13 +562,13 @@ class SocietyRuntime:
async with AsyncExitStack() as cleanup:
cleanup.callback(clear_runtime)
cleanup.push_async_callback(self.store.close)
+ cleanup.push_async_callback(asyncio.to_thread, self.conversations.close)
for release in (
self.world_feed.detach,
self.quests.detach,
self.checkpoints.detach,
self.bridge.detach,
self.scheduler.detach,
- self.conversations.close,
):
cleanup.callback(release)
cleanup.push_async_callback(self.browser.close)
@@ -606,7 +612,7 @@ class SocietyRuntime:
import json
events = json.loads(completion.events_json)
- self.conversations.ingest(session.session_id, events)
+ await asyncio.to_thread(self.conversations.ingest, session.session_id, events)
await self._complete_message_reply(session, completion, events)
terminal = [e for e in events if e.get("kind") == "turn_finished"]
if not terminal or terminal[-1].get("payload", {}).get("status") not in {
@@ -631,7 +637,8 @@ class SocietyRuntime:
# Reviewing them again wastes a model call and can
# duplicate a standing instruction as a conflicting memory.
return
- if self.conversations.queue_review(
+ if await asyncio.to_thread(
+ self.conversations.queue_review,
session.session_id,
completion.turn.turn_id,
events,
diff --git a/tests/unit/society/test_conversation_responsiveness.py b/tests/unit/society/test_conversation_responsiveness.py
new file mode 100644
index 0000000000..f7a90ea1fd
--- /dev/null
+++ b/tests/unit/society/test_conversation_responsiveness.py
@@ -0,0 +1,100 @@
+"""Busy conversation storage must not freeze the desktop's serving loop."""
+from __future__ import annotations
+
+import asyncio
+import sqlite3
+import threading
+
+import pytest
+
+from jarvis.society.conversation import ConversationArchive
+from jarvis.society.runtime import SocietyRuntime
+
+
+def test_deferred_archive_does_not_touch_storage_until_used(tmp_path):
+ path = tmp_path / "unused" / "archive.db"
+ archive = ConversationArchive(path, defer_open=True)
+ assert not path.parent.exists()
+ archive.open()
+ try:
+ archive.ingest(
+ "agent-a", [{"seq": 1, "kind": "user_message", "payload": {"text": "hello"}}]
+ )
+ assert archive.search("agent-a", "hello")[0]["seq"] == 1
+ assert archive.search("agent-b", "hello") == []
+ finally:
+ archive.close()
+
+
+def test_closed_deferred_archive_cannot_create_storage(tmp_path):
+ path = tmp_path / "unused" / "archive.db"
+ archive = ConversationArchive(path, defer_open=True)
+ archive.close()
+ archive.close()
+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):
+ archive.open()
+ assert not path.parent.exists()
+
+
+def test_failed_initialization_releases_connection_and_can_retry(tmp_path, monkeypatch):
+ archive = ConversationArchive(tmp_path / "archive.db", defer_open=True)
+ initialize = archive._initialize_schema
+ attempts = 0
+
+ def initialize_once():
+ nonlocal attempts
+ attempts += 1
+ if attempts == 1:
+ raise sqlite3.OperationalError("test storage unavailable")
+ initialize()
+
+ monkeypatch.setattr(archive, "_initialize_schema", initialize_once)
+ with pytest.raises(sqlite3.OperationalError):
+ archive.open()
+ assert archive._connection is None
+ archive.open()
+ try:
+ assert archive.read("agent-a") == []
+ finally:
+ archive.close()
+
+
+async def test_slow_archive_start_and_cancelled_cleanup_keep_loop_responsive(tmp_path, monkeypatch):
+ loop_thread = threading.get_ident()
+ entered = threading.Event()
+ release = threading.Event()
+ initialize = ConversationArchive._initialize_schema
+
+ def slow_schema(self):
+ assert threading.get_ident() != loop_thread
+ entered.set()
+ assert release.wait(10), "test must release the database initializer"
+ initialize(self)
+
+ monkeypatch.setattr(ConversationArchive, "_initialize_schema", slow_schema)
+ runtime = SocietyRuntime(tmp_path)
+ assert not (tmp_path / "society-conversations.db").exists()
+ startup = asyncio.create_task(runtime.ensure_started())
+ closing = None
+ try:
+ assert await asyncio.to_thread(entered.wait, 5)
+ heartbeat = asyncio.Event()
+ asyncio.get_running_loop().call_soon(heartbeat.set)
+ await asyncio.wait_for(heartbeat.wait(), 1)
+ startup.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await startup
+ closing = asyncio.create_task(runtime.close())
+ heartbeat.clear()
+ asyncio.get_running_loop().call_soon(heartbeat.set)
+ await asyncio.wait_for(heartbeat.wait(), 1)
+ finally:
+ release.set()
+ await asyncio.gather(startup, return_exceptions=True)
+ if closing is not None:
+ await closing
+ else:
+ await runtime.close()
+ assert runtime.conversations._connection is None
+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):
+ runtime.conversations.open()