DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(storage): close SQLite connections after helper operations
dajiaohuang149343c41452
diff --git a/deeptutor/multi_user/session_handoff.py b/deeptutor/multi_user/session_handoff.pyindex 1c52192dd..fa65889de 100644--- a/deeptutor/multi_user/session_handoff.py+++ b/deeptutor/multi_user/session_handoff.py@@ -8,6 +8,7 @@ token.from __future__ import annotations+from contextlib import closingfrom dataclasses import dataclassimport hashlibimport ipaddress@@ -283,7 +284,7 @@ class SessionHandoffStore:now: int | None,) -> None:current = int(time.time() if now is None else now)- with self._connect() as connection:+ with closing(self._connect()) as connection, connection:self._initialize(connection)self._check_rate(connection, bucket, limit, current)@@ -301,7 +302,7 @@ class SessionHandoffStore:self._rate_limit(f"create:{rate_key}", CREATE_RATE_LIMIT, now=now)code = secrets.token_urlsafe(32)expires_at = current + code_lifetime- with self._connect() as connection:+ with closing(self._connect()) as connection, connection:self._initialize(connection)connection.execute("BEGIN IMMEDIATE")try:@@ -341,7 +342,7 @@ class SessionHandoffStore:# on that site. Codes are 256-bit secrets, looked up by hash; a bogus# code costs one indexed lookup and cannot guess a valid record.code_hash = hash_secret(code)- with self._connect() as connection:+ with closing(self._connect()) as connection, connection:self._initialize(connection)connection.execute("BEGIN IMMEDIATE")try:@@ -411,7 +412,7 @@ class SessionHandoffStore:# As with exchange, a shared quota here would let anonymous invalid# tickets prevent every user on this public host from signing in.ticket_hash = hash_secret(ticket)- with self._connect() as connection:+ with closing(self._connect()) as connection, connection:self._initialize(connection)connection.execute("BEGIN IMMEDIATE")try:diff --git a/deeptutor/reading/store.py b/deeptutor/reading/store.pyindex 7b2c5fcd5..1cf741e38 100644--- a/deeptutor/reading/store.py+++ b/deeptutor/reading/store.py@@ -26,7 +26,7 @@ that produced the corrupted-notebook reports.from __future__ import annotations-from contextlib import contextmanager+from contextlib import closing, contextmanagerfrom dataclasses import replace as dataclass_replaceimport hashlibimport json@@ -205,7 +205,7 @@ class ReadingStore:if not db_path.is_file():return Nonetry:- with sqlite3.connect(db_path, timeout=30) as conn:+ with closing(sqlite3.connect(db_path, timeout=30)) as conn:conn.row_factory = sqlite3.Rowreturn conn.execute("SELECT * FROM reading_materials WHERE material_id = ?",diff --git a/deeptutor/services/memory/snapshot/adapters.py b/deeptutor/services/memory/snapshot/adapters.pyindex a87d7f1c9..bf0354953 100644--- a/deeptutor/services/memory/snapshot/adapters.py+++ b/deeptutor/services/memory/snapshot/adapters.py@@ -13,6 +13,7 @@ across refreshes.from __future__ import annotations+from contextlib import closingfrom datetime import datetime, timezoneimport hashlibimport json@@ -470,7 +471,7 @@ def read_chat_entities() -> list[Entity]:if not db_path.exists():continuetry:- with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:+ with closing(sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)) as conn:conn.row_factory = sqlite3.Rowsessions = conn.execute("SELECT id, title, created_at, updated_at FROM sessions ORDER BY updated_at DESC"@@ -522,7 +523,7 @@ def read_quiz_entities() -> list[Entity]:if not db_path.exists():continuetry:- with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:+ with closing(sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)) as conn:conn.row_factory = sqlite3.Rowrows = conn.execute("SELECT id, session_id, turn_id, question_id, question, "@@ -602,7 +603,7 @@ def probe_chat_entities() -> list[EntityStamp]:if not db_path.exists():continuetry:- with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:+ with closing(sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)) as conn:conn.row_factory = sqlite3.Rowlast_msg_id: dict[str, int] = {row["session_id"]: row["id"]diff --git a/deeptutor/services/partners/runtime_status.py b/deeptutor/services/partners/runtime_status.pyindex e05a36db1..ea5ebc463 100644--- a/deeptutor/services/partners/runtime_status.py+++ b/deeptutor/services/partners/runtime_status.py@@ -2,6 +2,7 @@from __future__ import annotations+from contextlib import closingimport jsonfrom pathlib import Pathimport sqlite3@@ -17,7 +18,7 @@ class PartnerRuntimeStatusRepository:def __init__(self, path: Path | None = None) -> None:self.path = (path or (get_data_dir() / "_runtime" / "status.sqlite3")).resolve()self.path.parent.mkdir(parents=True, exist_ok=True)- with self._connect() as connection:+ with closing(self._connect()) as connection, connection:connection.execute("""CREATE TABLE IF NOT EXISTS partner_runtime_status (@@ -65,7 +66,7 @@ class PartnerRuntimeStatusRepository:"runtime_updated_at": updated_at,})- with self._connect() as connection:+ with closing(self._connect()) as connection, connection:connection.execute("""INSERT INTO partner_runtime_status(@@ -95,7 +96,7 @@ class PartnerRuntimeStatusRepository:return safe_payloaddef get(self, partner_id: str) -> dict[str, Any] | None:- with self._connect() as connection:+ with closing(self._connect()) as connection, connection:row = connection.execute("SELECT payload FROM partner_runtime_status WHERE partner_id=?",(partner_id,),@@ -103,14 +104,14 @@ class PartnerRuntimeStatusRepository:return json.loads(str(row[0])) if row else Nonedef list(self) -> dict[str, dict[str, Any]]:- with self._connect() as connection:+ with closing(self._connect()) as connection, connection:rows = connection.execute("SELECT partner_id, payload FROM partner_runtime_status").fetchall()return {str(row[0]): json.loads(str(row[1])) for row in rows}def delete(self, partner_id: str) -> None:- with self._connect() as connection:+ with closing(self._connect()) as connection, connection:connection.execute("DELETE FROM partner_runtime_status WHERE partner_id=?", (partner_id,))diff --git a/deeptutor/services/workspace/data_migration.py b/deeptutor/services/workspace/data_migration.pyindex 0f05cfcce..0b70384fd 100644--- a/deeptutor/services/workspace/data_migration.py+++ b/deeptutor/services/workspace/data_migration.py@@ -8,6 +8,7 @@ in an account-private recovery directory after a successful transfer.from __future__ import annotations+from contextlib import closingfrom datetime import datetime, timezoneimport hashlibimport json@@ -132,7 +133,7 @@ def _empty_scaffold(root: Path) -> bool:database = handle.read(16) == b"SQLite format 3\x00"if not database:return False- with sqlite3.connect(f"file:{path}?mode=ro", uri=True) as conn:+ with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as conn, conn:for (name,) in conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").fetchall():@@ -159,8 +160,9 @@ def _snapshot(source: Path, target: Path, *, included: set[str] | None = None) -database = handle.read(16) == b"SQLite format 3\x00"if database:with (- sqlite3.connect(f"file:{path}?mode=ro", uri=True) as original,- sqlite3.connect(destination) as copy,+ closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as original,+ closing(sqlite3.connect(destination)) as copy,+ copy,):original.backup(copy)if copy.execute("PRAGMA integrity_check").fetchone()[0] != "ok":@@ -180,7 +182,7 @@ def _sqlite_sessions(paths) -> list[dict]:db = paths.get_chat_history_db()if not db.exists():return []- with sqlite3.connect(f"file:{db}?mode=ro", uri=True) as conn:+ with closing(sqlite3.connect(f"file:{db}?mode=ro", uri=True)) as conn, conn:conn.row_factory = sqlite3.Rowreturn [dict(row) for row in conn.execute("SELECT id,title,preferences_json FROM sessions")]@@ -345,7 +347,7 @@ def preview(blockers.append("Choose a custom workspace to retain historical archives.")for paths in (source, target):if paths.get_chat_history_db().exists():- with sqlite3.connect(paths.get_chat_history_db()) as conn:+ with closing(sqlite3.connect(paths.get_chat_history_db())) as conn, conn:if conn.execute("SELECT 1 FROM turns WHERE status IN ('queued','running','waiting_input') LIMIT 1").fetchone():@@ -373,7 +375,7 @@ def preview(root = _feature_path(source, feature)artifacts[feature] = [path.relative_to(root).as_posix() for path in _files(root)]if _backend() == "sqlite" and session_ids and target.get_chat_history_db().exists():- with sqlite3.connect(source.get_chat_history_db()) as conn:+ with closing(sqlite3.connect(source.get_chat_history_db())) as conn, conn:conn.execute("ATTACH DATABASE ? AS target", (str(target.get_chat_history_db()),))placeholders = ",".join("?" for _ in session_ids)for table, owner in (@@ -447,7 +449,7 @@ def _selected_artifact_files(paths, session_ids: set[str]) -> dict[str, list[str)db = paths.get_chat_history_db()if db.exists():- with sqlite3.connect(db) as conn:+ with closing(sqlite3.connect(db)) as conn, conn:identifiers.update(row[0]for row in conn.execute("SELECT id,session_id FROM turns")@@ -819,7 +821,7 @@ def _rebind_feature_urls(root: Path, target_id: str, *, included: set[str] | Nonif original != updated:atomic_write_text(path, updated)elif path.suffix in {".sqlite3", ".sqlite", ".db"}:- with sqlite3.connect(path) as conn:+ with closing(sqlite3.connect(path)) as conn, conn:tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").fetchall()@@ -845,7 +847,11 @@ def _restore_database(snapshot: Path, destination: Path) -> None:if not snapshot.exists():returndestination.parent.mkdir(parents=True, exist_ok=True)- with sqlite3.connect(snapshot) as original, sqlite3.connect(destination) as restored:+ with (+ closing(sqlite3.connect(snapshot)) as original,+ closing(sqlite3.connect(destination)) as restored,+ restored,+ ):original.backup(restored)@@ -871,7 +877,7 @@ def _remove_installed(result: dict, target) -> None:def _prune_session_snapshot(database: Path, session_ids: set[str]) -> None:- with sqlite3.connect(database) as conn:+ with closing(sqlite3.connect(database)) as conn, conn:conn.execute("PRAGMA foreign_keys=ON")conn.execute("CREATE TEMP TABLE retained_sessions (id TEXT PRIMARY KEY)")conn.executemany(diff --git a/deeptutor/services/workspace/dependencies.py b/deeptutor/services/workspace/dependencies.pyindex 89883e23e..eb1c79906 100644--- a/deeptutor/services/workspace/dependencies.py+++ b/deeptutor/services/workspace/dependencies.py@@ -2,6 +2,7 @@from __future__ import annotations+from contextlib import closingimport jsonimport loggingimport sqlite3@@ -193,7 +194,7 @@ def _forward_closure(scanned.add("learning")db = paths.get_workspace_dir() / "learning" / "mastery" / "mastery.sqlite3"if db.exists():- with sqlite3.connect(db) as conn:+ with closing(sqlite3.connect(db)) as conn, conn:tables = {row[0]for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")@@ -222,7 +223,7 @@ def _forward_closure(scanned.add("reading")db = paths.get_workspace_dir() / "reading" / "_catalog.sqlite3"if db.exists():- with sqlite3.connect(db) as conn:+ with closing(sqlite3.connect(db)) as conn, conn:if conn.execute("SELECT 1 FROM sqlite_master WHERE name='reading_workspace_sessions'").fetchone():@@ -293,7 +294,7 @@ def _question_sessions(paths, entry_ids=None) -> set[str]:db = paths.get_chat_history_db()if not db.exists():return set()- with sqlite3.connect(db) as conn:+ with closing(sqlite3.connect(db)) as conn, conn:if not conn.execute("SELECT 1 FROM sqlite_master WHERE name='notebook_entries'").fetchone():return set()if entry_ids:@@ -351,7 +352,7 @@ def _with_historical_references(db = paths.get_chat_history_db()if db.exists():- with sqlite3.connect(db) as conn:+ with closing(sqlite3.connect(db)) as conn, conn:for sid, raw in conn.execute("SELECT session_id,metadata_json FROM messages"):if sid in prefs and raw:try:diff --git a/deeptutor/services/workspace/session_move.py b/deeptutor/services/workspace/session_move.pyindex 8d2ad8e5f..086f2a96c 100644--- a/deeptutor/services/workspace/session_move.py+++ b/deeptutor/services/workspace/session_move.py@@ -2,6 +2,7 @@from __future__ import annotations+from contextlib import closingimport jsonimport sqlite3@@ -113,7 +114,7 @@ def migrate_legacy_bindings() -> int:else:db = paths.get_chat_history_db()if db.exists():- with sqlite3.connect(db) as conn:+ with closing(sqlite3.connect(db)) as conn, conn:for sid, raw in conn.execute("SELECT id,preferences_json FROM sessions"):prefs = json.loads(raw or "{}")if prefs.get("workspace_id"):diff --git a/tests/multi_user/test_handoff_connection_lifetime.py b/tests/multi_user/test_handoff_connection_lifetime.pynew file mode 100644index 000000000..366f66903--- /dev/null+++ b/tests/multi_user/test_handoff_connection_lifetime.py@@ -0,0 +1,46 @@+"""Handoff operations release their native SQLite connections."""++from __future__ import annotations++import sqlite3++import pytest++from deeptutor.multi_user import session_handoff++[email protected]("operation", ["create", "exchange", "consume", "reject"])+def test_handoff_closes_connections(tmp_path, monkeypatch, operation):+ monkeypatch.setattr(session_handoff, "load_or_create_auth_secret", lambda: "test-secret")+ store = session_handoff.SessionHandoffStore(tmp_path / "handoff.sqlite3")+ connections = []+ connect = store._connect++ def observed_connect():+ connection = connect()+ connections.append(connection)+ return connection++ monkeypatch.setattr(store, "_connect", observed_connect)+ ticket = session_handoff.encrypt_ticket_payload({"host": "app.example", "exp": 220})+ record = store.create(+ encrypted_ticket=ticket,+ ticket_hash=session_handoff.hash_secret(ticket),+ public_host="app.example",+ now=100,+ )+ if operation == "reject":+ with pytest.raises(session_handoff.HandoffRejected):+ store.exchange(code=record.code, public_host="other.example", now=101)+ elif operation in {"exchange", "consume"}:+ exchanged = store.exchange(code=record.code, public_host="app.example", now=101)+ if operation == "consume":+ assert (+ store.consume_ticket(ticket=exchanged, public_host="app.example", now=102)+ == exchanged+ )+ assert connections+ for connection in connections:+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):+ connection.execute("SELECT 1")+ store.db_path.rename(tmp_path / "released.sqlite3")diff --git a/tests/reading/test_catalog_connection_lifetime.py b/tests/reading/test_catalog_connection_lifetime.pynew file mode 100644index 000000000..dcc6ba9ba--- /dev/null+++ b/tests/reading/test_catalog_connection_lifetime.py@@ -0,0 +1,41 @@+"""Reading APIs release native catalog handles before returning."""++from pathlib import Path+import sqlite3++import pytest++from deeptutor.reading.catalog_store import ReadingCatalogStore+from deeptutor.reading.store import ReadingStore++[email protected]("operation", ["manifest", "annotations", "exists"])+def test_reading_api_closes_catalog_connections(+ tmp_path: Path, monkeypatch, operation: str+) -> None:+ """Retain real handles so garbage collection cannot hide leaked readers."""+ catalog = ReadingCatalogStore(tmp_path)+ store = ReadingStore(tmp_path)+ material_id = "abcdef0123456789"+ manifest = store.ingest_units(material_id, filename="guide.md", units=["A reading guide."])+ catalog.register_manifest(manifest)+ connect = sqlite3.connect+ connections = []++ def track_connection(*args, **kwargs):+ connection = connect(*args, **kwargs)+ connections.append(connection)+ return connection++ monkeypatch.setattr(sqlite3, "connect", track_connection)+ try:+ result = getattr(store, operation)(material_id)+ assert result is not None+ assert connections+ for connection in connections:+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):+ connection.execute("SELECT 1")+ catalog.db_path.rename(tmp_path / "moved.sqlite3")+ finally:+ for connection in connections:+ connection.close()diff --git a/tests/services/memory/test_snapshot_connection_lifetime.py b/tests/services/memory/test_snapshot_connection_lifetime.pynew file mode 100644index 000000000..629bfc469--- /dev/null+++ b/tests/services/memory/test_snapshot_connection_lifetime.py@@ -0,0 +1,47 @@+"""Snapshot readers release their SQLite connections before returning."""++from pathlib import Path+import sqlite3+from types import SimpleNamespace++import pytest++from deeptutor.services.memory.snapshot import adapters+from deeptutor.services.session.sqlite_store import SQLiteSessionStore+++ "reader", ["read_chat_entities", "read_quiz_entities", "probe_chat_entities"]+)+def test_snapshot_reader_closes_connection(tmp_path: Path, monkeypatch, reader: str) -> None:+ """Keep native handles alive so garbage collection cannot mask a leak."""+ database = tmp_path / "history.db"+ SQLiteSessionStore(database)+ monkeypatch.setattr(+ adapters,+ "get_account_path_service",+ lambda: SimpleNamespace(get_chat_history_db=lambda: database),+ )+ monkeypatch.setattr(+ "deeptutor.services.workspace.get_content_workspace_service",+ lambda: SimpleNamespace(registered_bindings=lambda: []),+ )+ connect = sqlite3.connect+ connections = []++ def track_connection(*args, **kwargs):+ connection = connect(*args, **kwargs)+ connections.append(connection)+ return connection++ monkeypatch.setattr(sqlite3, "connect", track_connection)+ try:+ assert getattr(adapters, reader)() == []+ assert len(connections) == 1+ for connection in connections:+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):+ connection.execute("SELECT 1")+ database.rename(tmp_path / "moved.db")+ finally:+ for connection in connections:+ connection.close()diff --git a/tests/services/test_partner_status_connection_lifetime.py b/tests/services/test_partner_status_connection_lifetime.pynew file mode 100644index 000000000..314f8f70f--- /dev/null+++ b/tests/services/test_partner_status_connection_lifetime.py@@ -0,0 +1,47 @@+"""Partner runtime status operations release their SQLite connections."""++import sqlite3++import pytest++from deeptutor.services.partners.runtime_status import PartnerRuntimeStatusRepository++[email protected]("operation", ["initialize", "set", "get", "list", "delete"])+def test_status_operations_close_native_connections(tmp_path, monkeypatch, operation):+ path = tmp_path / "status.sqlite3"+ repository = PartnerRuntimeStatusRepository(path)+ repository.set("partner", owner_id="owner", running=True, state="running")+ connect = sqlite3.connect+ connections = []++ def track(*args, **kwargs):+ connection = connect(*args, **kwargs)+ connections.append(connection)+ return connection++ monkeypatch.setattr(sqlite3, "connect", track)+ if operation == "initialize":+ PartnerRuntimeStatusRepository(path)+ elif operation == "set":+ repository.set("partner", owner_id="owner", running=False, state="stopped")+ elif operation == "get":+ assert repository.get("partner")["running"] is True+ elif operation == "list":+ assert "partner" in repository.list()+ else:+ repository.delete("partner")+ assert connections+ try:+ for connection in connections:+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):+ connection.execute("SELECT 1")+ finally:+ for connection in connections:+ connection.close()++ with connect(path) as observer:+ row = observer.execute("SELECT running FROM partner_runtime_status").fetchone()+ observer.close()+ expected = {"set": (0,), "delete": None}.get(operation, (1,))+ assert row == expecteddiff --git a/tests/services/workspace/test_connection_lifetime.py b/tests/services/workspace/test_connection_lifetime.pynew file mode 100644index 000000000..2131227fb--- /dev/null+++ b/tests/services/workspace/test_connection_lifetime.py@@ -0,0 +1,61 @@+"""Workspace database helpers release handles before migration continues."""++from contextlib import closing+import sqlite3+from types import SimpleNamespace++import pytest++from deeptutor.services.workspace import data_migration, dependencies+++ "operation", ["snapshot", "restore", "rebind", "scaffold", "sessions", "questions"]+)+def test_workspace_helpers_close_database_connections(tmp_path, monkeypatch, operation):+ source = tmp_path / "source.sqlite"+ destination = tmp_path / "copy.sqlite"+ connect = sqlite3.connect+ with closing(connect(source)) as conn, conn:+ conn.execute("CREATE TABLE sessions (id TEXT, title TEXT, preferences_json TEXT)")+ conn.execute("CREATE TABLE notebook_entries (id TEXT, session_id TEXT)")+ conn.execute("CREATE TABLE links (url TEXT)")+ if operation != "scaffold":+ conn.execute("INSERT INTO sessions VALUES ('session-one', 'Title', '{}')")+ conn.execute("INSERT INTO notebook_entries VALUES ('question-one', 'session-one')")+ conn.execute("INSERT INTO links VALUES ('/files/outputs/result.txt')")+ connections = []++ def tracked_connect(*args, **kwargs):+ conn = connect(*args, **kwargs)+ connections.append(conn)+ return conn++ monkeypatch.setattr(sqlite3, "connect", tracked_connect)+ paths = SimpleNamespace(get_chat_history_db=lambda: source)+ if operation == "snapshot":+ manifest = data_migration._snapshot(source, tmp_path / "snapshot")+ assert source.name in manifest+ destination = tmp_path / "snapshot" / source.name+ elif operation == "restore":+ data_migration._restore_database(source, destination)+ elif operation == "rebind":+ data_migration._rebind_feature_urls(source, "workspace-one")+ with closing(connect(source)) as conn:+ assert (+ conn.execute("SELECT url FROM links").fetchone()[0] != "/files/outputs/result.txt"+ )+ elif operation == "scaffold":+ assert data_migration._empty_scaffold(source)+ elif operation == "sessions":+ assert data_migration._sqlite_sessions(paths)[0]["id"] == "session-one"+ else:+ assert dependencies._question_sessions(paths) == {"session-one"}+ assert connections+ for conn in connections:+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):+ conn.execute("SELECT 1")+ if operation in {"snapshot", "restore"}:+ with closing(connect(destination)) as conn:+ assert conn.execute("SELECT id FROM sessions").fetchone() == ("session-one",)+ source.rename(tmp_path / "released.sqlite")