DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix: close SQLite connections in background repositories
Lamber56cf5b4414c1
diff --git a/deeptutor/services/cron/repository.py b/deeptutor/services/cron/repository.pyindex 8adc752de..6cd13d92b 100644--- a/deeptutor/services/cron/repository.py+++ b/deeptutor/services/cron/repository.py@@ -2,7 +2,7 @@from __future__ import annotations-from collections.abc import Iterable+from collections.abc import Iterable, Iteratorimport contextlibimport jsonfrom pathlib import Path@@ -103,12 +103,18 @@ class SQLiteCronRepository:source.replace(archive)return payloads- def _connect(self) -> sqlite3.Connection:+ @contextlib.contextmanager+ def _connect(self) -> Iterator[sqlite3.Connection]:+ """Preserve transaction handling and close without relying on garbage collection."""connection = sqlite3.connect(self.path, timeout=30, isolation_level=None)- connection.execute("PRAGMA journal_mode=WAL")- connection.execute("PRAGMA busy_timeout=30000")- connection.execute("PRAGMA synchronous=NORMAL")- return connection+ try:+ connection.execute("PRAGMA journal_mode=WAL")+ connection.execute("PRAGMA busy_timeout=30000")+ connection.execute("PRAGMA synchronous=NORMAL")+ with connection:+ yield connection+ finally:+ connection.close()def _initialize(self, initial: Iterable[dict[str, Any]]) -> None:with self._migration_lock(), self._connect() as connection:diff --git a/deeptutor/services/web_source/repository.py b/deeptutor/services/web_source/repository.pyindex 1387d3db9..14d39bdb7 100644--- a/deeptutor/services/web_source/repository.py+++ b/deeptutor/services/web_source/repository.py@@ -2,6 +2,8 @@from __future__ import annotations+from collections.abc import Iterator+import contextlibfrom dataclasses import dataclassfrom pathlib import Pathimport sqlite3@@ -117,13 +119,19 @@ class SQLiteWebSourceSyncRepository:self.path.parent.mkdir(parents=True, exist_ok=True)self._initialize()- def _connect(self) -> sqlite3.Connection:+ @contextlib.contextmanager+ def _connect(self) -> Iterator[sqlite3.Connection]:+ """Preserve transaction handling and close without relying on garbage collection."""connection = sqlite3.connect(self.path, timeout=30, isolation_level=None)- connection.execute("PRAGMA journal_mode=WAL")- connection.execute("PRAGMA busy_timeout=30000")- connection.execute("PRAGMA synchronous=NORMAL")- connection.row_factory = sqlite3.Row- return connection+ try:+ connection.execute("PRAGMA journal_mode=WAL")+ connection.execute("PRAGMA busy_timeout=30000")+ connection.execute("PRAGMA synchronous=NORMAL")+ connection.row_factory = sqlite3.Row+ with connection:+ yield connection+ finally:+ connection.close()def _initialize(self) -> None:with self._connect() as connection:diff --git a/tests/services/test_background_sqlite_connections.py b/tests/services/test_background_sqlite_connections.pynew file mode 100644index 000000000..5c7c12613--- /dev/null+++ b/tests/services/test_background_sqlite_connections.py@@ -0,0 +1,114 @@+"""Background SQLite repositories release connections without waiting for GC."""++from __future__ import annotations++from collections.abc import Iterator+from pathlib import Path+import sqlite3++import pytest++from deeptutor.services.cron.repository import SQLiteCronRepository+from deeptutor.services.web_source.repository import SQLiteWebSourceSyncRepository++[email protected](params=[SQLiteCronRepository, SQLiteWebSourceSyncRepository])+def repository_type(+ request: pytest.FixtureRequest,+) -> type[SQLiteCronRepository] | type[SQLiteWebSourceSyncRepository]:+ """Exercise both repositories used by periodic background tasks."""+ return request.param+++def connections(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[sqlite3.Connection]]:+ """Retain real connections so garbage collection cannot hide a missing close."""+ opened: list[sqlite3.Connection] = []+ connect = sqlite3.connect++ def track_connection(*args, **kwargs) -> sqlite3.Connection:+ connection = connect(*args, **kwargs)+ opened.append(connection)+ return connection++ monkeypatch.setattr(sqlite3, "connect", track_connection)+ try:+ yield opened+ finally:+ # Also clean up when this regression test runs against the unfixed code.+ for connection in opened:+ connection.close()+++def assert_closed(connections: list[sqlite3.Connection]) -> None:+ """Every returned connection must already reject database operations."""+ assert connections+ for connection in connections:+ with pytest.raises(sqlite3.ProgrammingError, match="closed"):+ connection.execute("SELECT 1")+++def test_polling_closes_every_connection(repository_type, connections, tmp_path: Path) -> None:+ """Initialization and repeated empty polling must not retain DB/WAL handles."""+ repository = repository_type(tmp_path / "jobs.sqlite")+ for _ in range(10):+ if isinstance(repository, SQLiteCronRepository):+ assert repository.revision() == 0+ assert repository.list_payloads() == []+ else:+ repository.reconcile_sources(set(), scanned_owner_ids={"local-admin"})+ assert repository.due_jobs() == []+ assert_closed(connections)+++def test_failed_operation_closes_connection(repository_type, connections, tmp_path: Path) -> None:+ """An exception during a repository operation must release its connection."""+ repository = repository_type(tmp_path / "jobs.sqlite")+ with pytest.raises(sqlite3.OperationalError):+ with repository._connect() as connection:+ connection.execute("SELECT * FROM nonexistent_table")+ assert_closed(connections)++[email protected]("pragma", ["journal_mode", "busy_timeout", "synchronous"])+def test_failed_setup_closes_connection(+ repository_type, connections, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, pragma: str+) -> None:+ """Close even when connection setup fails before the context yields."""+ repository = repository_type(tmp_path / "jobs.sqlite")+ connect = sqlite3.connect++ class SetupFailureConnection(sqlite3.Connection):+ def execute(self, sql, *args, **kwargs):+ if sql.startswith(f"PRAGMA {pragma}"):+ raise sqlite3.OperationalError("injected setup failure")+ return super().execute(sql, *args, **kwargs)++ def fail_setup(*args, **kwargs) -> sqlite3.Connection:+ return connect(*args, **kwargs, factory=SetupFailureConnection)++ monkeypatch.setattr(sqlite3, "connect", fail_setup)+ with pytest.raises(sqlite3.OperationalError, match="injected setup failure"):+ with repository._connect():+ pytest.fail("failed setup must not yield a connection")+ assert_closed(connections)+++def test_connection_context_preserves_commit_and_rollback(+ repository_type, connections, tmp_path: Path+) -> None:+ """Closing must preserve the existing SQLite transaction context semantics."""+ repository = repository_type(tmp_path / "jobs.sqlite")+ with repository._connect() as connection:+ connection.execute("CREATE TABLE lifecycle_test (value INTEGER)")+ connection.execute("BEGIN")+ connection.execute("INSERT INTO lifecycle_test VALUES (1)")+ with pytest.raises(RuntimeError, match="abort transaction"):+ with repository._connect() as connection:+ connection.execute("BEGIN")+ connection.execute("INSERT INTO lifecycle_test VALUES (2)")+ raise RuntimeError("abort transaction")+ with repository._connect() as connection:+ rows = connection.execute("SELECT value FROM lifecycle_test").fetchall()+ assert [row[0] for row in rows] == [1]+ assert_closed(connections)