Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix: close sqlite connections in flow persistence and sqlite provider (#7511)
Sharoon Sharif5c33fe4c713a
diff --git a/lib/crewai/src/crewai/flow/persistence/sqlite.py b/lib/crewai/src/crewai/flow/persistence/sqlite.pyindex e0c7994fd..9380caf6b 100644--- a/lib/crewai/src/crewai/flow/persistence/sqlite.py+++ b/lib/crewai/src/crewai/flow/persistence/sqlite.py@@ -2,6 +2,7 @@from __future__ import annotations+from contextlib import closingfrom datetime import date, datetime, timezoneimport jsonimport os@@ -86,7 +87,8 @@ class SQLiteFlowPersistence(FlowPersistence):"""Create the necessary tables if they don't exist."""with (store_lock(self._lock_name),- sqlite3.connect(self.db_path, timeout=30) as conn,+ closing(sqlite3.connect(self.db_path, timeout=30)) as conn,+ conn,):conn.execute("PRAGMA journal_mode=WAL")conn.execute(@@ -188,7 +190,8 @@ class SQLiteFlowPersistence(FlowPersistence):with (store_lock(self._lock_name),- sqlite3.connect(self.db_path, timeout=30) as conn,+ closing(sqlite3.connect(self.db_path, timeout=30)) as conn,+ conn,):self._save_state_sql(conn, flow_uuid, method_name, state_dict)@@ -201,7 +204,7 @@ class SQLiteFlowPersistence(FlowPersistence):Returns:The most recent state as a dictionary, or None if no state exists"""- with sqlite3.connect(self.db_path, timeout=30) as conn:+ with closing(sqlite3.connect(self.db_path, timeout=30)) as conn, conn:cursor = conn.execute("""SELECT state_json@@ -239,7 +242,8 @@ class SQLiteFlowPersistence(FlowPersistence):with (store_lock(self._lock_name),- sqlite3.connect(self.db_path, timeout=30) as conn,+ closing(sqlite3.connect(self.db_path, timeout=30)) as conn,+ conn,):self._save_state_sql(conn, flow_uuid, context.method_name, state_dict)@@ -276,7 +280,7 @@ class SQLiteFlowPersistence(FlowPersistence):# Import here to avoid circular importsfrom crewai.flow.async_feedback.types import PendingFeedbackContext- with sqlite3.connect(self.db_path, timeout=30) as conn:+ with closing(sqlite3.connect(self.db_path, timeout=30)) as conn, conn:cursor = conn.execute("""SELECT state_json, context_json@@ -302,7 +306,8 @@ class SQLiteFlowPersistence(FlowPersistence):"""with (store_lock(self._lock_name),- sqlite3.connect(self.db_path, timeout=30) as conn,+ closing(sqlite3.connect(self.db_path, timeout=30)) as conn,+ conn,):conn.execute("""diff --git a/lib/crewai/src/crewai/state/provider/sqlite_provider.py b/lib/crewai/src/crewai/state/provider/sqlite_provider.pyindex 14fa3425d..8f7f11d04 100644--- a/lib/crewai/src/crewai/state/provider/sqlite_provider.py+++ b/lib/crewai/src/crewai/state/provider/sqlite_provider.py@@ -2,6 +2,7 @@from __future__ import annotations+from contextlib import closingfrom datetime import datetime, timezonefrom pathlib import Pathimport sqlite3@@ -76,7 +77,7 @@ class SqliteProvider(BaseProvider):"""checkpoint_id, ts = _make_id()Path(location).parent.mkdir(parents=True, exist_ok=True)- with sqlite3.connect(location) as conn:+ with closing(sqlite3.connect(location)) as conn, conn:conn.execute("PRAGMA journal_mode=WAL")conn.execute(_CREATE_TABLE)conn.execute(_INSERT, (checkpoint_id, ts, parent_id, branch, data))@@ -113,7 +114,7 @@ class SqliteProvider(BaseProvider):def prune(self, location: str, max_keep: int, *, branch: str = "main") -> int:"""Remove oldest checkpoint rows beyond *max_keep* on a branch."""- with sqlite3.connect(location) as conn:+ with closing(sqlite3.connect(location)) as conn, conn:cursor = conn.execute(_PRUNE, (branch, branch, max_keep))removed: int = cursor.rowcountconn.commit()@@ -136,7 +137,7 @@ class SqliteProvider(BaseProvider):ValueError: If the checkpoint ID is not found."""db_path, checkpoint_id = location.rsplit("#", 1)- with sqlite3.connect(db_path) as conn:+ with closing(sqlite3.connect(db_path)) as conn, conn:row = conn.execute(_SELECT, (checkpoint_id,)).fetchone()if row is None:raise ValueError(f"Checkpoint not found: {checkpoint_id}")diff --git a/lib/crewai/tests/test_checkpoint.py b/lib/crewai/tests/test_checkpoint.pyindex ab49786c5..8bd43e3e7 100644--- a/lib/crewai/tests/test_checkpoint.py+++ b/lib/crewai/tests/test_checkpoint.py@@ -3,6 +3,7 @@from __future__ import annotationsimport inspect+from contextlib import closingimport jsonimport osimport sqlite3@@ -505,7 +506,7 @@ class TestSqliteProviderFork:loc = provider.checkpoint("{}", db, parent_id="p1", branch="exp")cid = provider.extract_id(loc)- with sqlite3.connect(db) as conn:+ with closing(sqlite3.connect(db)) as conn:row = conn.execute("SELECT parent_id, branch FROM checkpoints WHERE id = ?",(cid,),@@ -523,7 +524,7 @@ class TestSqliteProviderFork:provider.prune(db, max_keep=1, branch="main")- with sqlite3.connect(db) as conn:+ with closing(sqlite3.connect(db)) as conn:main_count = conn.execute("SELECT COUNT(*) FROM checkpoints WHERE branch = 'main'").fetchone()[0]@@ -549,7 +550,7 @@ class TestSqliteProviderFork:id2 = state._checkpoint_idassert id2 != id1- with sqlite3.connect(db) as conn:+ with closing(sqlite3.connect(db)) as conn:row = conn.execute("SELECT parent_id FROM checkpoints WHERE id = ?", (id2,)).fetchone()diff --git a/lib/crewai/tests/test_sqlite_connection_lifecycle.py b/lib/crewai/tests/test_sqlite_connection_lifecycle.pynew file mode 100644index 000000000..5b5f4838c--- /dev/null+++ b/lib/crewai/tests/test_sqlite_connection_lifecycle.py@@ -0,0 +1,139 @@+"""Connection-lifecycle tests for the flow persistence and checkpoint SQLite backends."""++from __future__ import annotations++import json+import os+import sqlite3+from pathlib import Path++import pytest+from crewai.flow.async_feedback.types import PendingFeedbackContext+from crewai.flow.persistence.sqlite import SQLiteFlowPersistence+from crewai.state.provider.sqlite_provider import SqliteProvider+++def _track_connections(+ monkeypatch: pytest.MonkeyPatch,+) -> list[sqlite3.Connection]:+ """Record every connection ``sqlite3.connect`` hands out for the rest of the test."""+ opened: list[sqlite3.Connection] = []+ real_connect = sqlite3.connect++ def tracking_connect(*args: object, **kwargs: object) -> sqlite3.Connection:+ conn = real_connect(*args, **kwargs) # type: ignore[arg-type]+ opened.append(conn)+ return conn++ monkeypatch.setattr(sqlite3, "connect", tracking_connect)+ return opened+++def _assert_all_closed(opened: list[sqlite3.Connection]) -> None:+ """Every recorded connection must reject further use."""+ for conn in opened:+ with pytest.raises(sqlite3.ProgrammingError, match="closed database"):+ conn.execute("SELECT 1")+++def _make_context() -> PendingFeedbackContext:+ """Build a minimal pending-feedback context."""+ return PendingFeedbackContext(+ flow_id="flow-1",+ flow_class="tests.ReviewFlow",+ method_name="review",+ method_output="draft",+ message="Please review",+ )+++class _FailingConnection(sqlite3.Connection):+ """Connection whose ``execute`` fails on INSERT, after any preceding statements."""++ def execute(self, sql: str, *args: object) -> sqlite3.Cursor: # type: ignore[override]+ if sql.lstrip().upper().startswith("INSERT"):+ raise sqlite3.OperationalError("simulated failure inside the transaction")+ return super().execute(sql, *args) # type: ignore[arg-type]+++def test_flow_persistence_closes_every_connection(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Each ``SQLiteFlowPersistence`` operation closes the connection it opened."""+ opened = _track_connections(monkeypatch)++ persistence = SQLiteFlowPersistence(str(tmp_path / "flows.db")) # init_db+ persistence.save_state("flow-1", "start", {"step": 1})+ assert persistence.load_state("flow-1") == {"step": 1}+ persistence.save_pending_feedback("flow-1", _make_context(), {"step": 2})+ loaded = persistence.load_pending_feedback("flow-1")+ assert loaded is not None and loaded[0] == {"step": 2}+ persistence.clear_pending_feedback("flow-1")++ assert len(opened) == 6+ _assert_all_closed(opened)+++def test_flow_persistence_db_is_removable_after_use(tmp_path: Path) -> None:+ """The flow database can be deleted right after use, without a GC pass."""+ db_path = tmp_path / "flows.db"+ persistence = SQLiteFlowPersistence(str(db_path))+ persistence.save_state("flow-1", "start", {"step": 1})+ persistence.load_state("flow-1")+ del persistence++ os.remove(db_path)+ assert not db_path.exists()+++def test_flow_persistence_failed_write_rolls_back_and_closes(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """A failing INSERT inside ``save_state`` rolls back and still closes the connection."""+ persistence = SQLiteFlowPersistence(str(tmp_path / "flows.db"))++ opened: list[sqlite3.Connection] = []+ real_connect = sqlite3.connect++ def failing_connect(*args: object, **kwargs: object) -> sqlite3.Connection:+ kwargs["factory"] = _FailingConnection+ conn = real_connect(*args, **kwargs) # type: ignore[arg-type]+ opened.append(conn)+ return conn++ monkeypatch.setattr(sqlite3, "connect", failing_connect)+ with pytest.raises(sqlite3.OperationalError):+ persistence.save_state("flow-1", "start", {"step": 1})+ monkeypatch.undo()++ assert len(opened) == 1+ _assert_all_closed(opened)+ assert persistence.load_state("flow-1") is None+++def test_sqlite_provider_closes_every_connection(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Each ``SqliteProvider`` operation closes the connection it opened."""+ opened = _track_connections(monkeypatch)+ db_path = str(tmp_path / "checkpoints.db")++ provider = SqliteProvider()+ location = provider.checkpoint('{"a": 1}', db_path, branch="main")+ # Stored as jsonb, so the text comes back normalised; compare the parsed value.+ assert json.loads(provider.from_checkpoint(location)) == {"a": 1}+ provider.prune(db_path, 1, branch="main")++ assert len(opened) == 3+ _assert_all_closed(opened)+++def test_sqlite_provider_db_is_removable_after_use(tmp_path: Path) -> None:+ """The checkpoint database can be deleted right after use, without a GC pass."""+ db_path = tmp_path / "checkpoints.db"+ provider = SqliteProvider()+ location = provider.checkpoint('{"a": 1}', str(db_path), branch="main")+ provider.from_checkpoint(location)++ os.remove(db_path)+ assert not db_path.exists()