RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(db): abort poisoned transactions on connection loss (#19530)
Sbaaoui Idriss6a9d4dcf1325
diff --git a/api/db/db_models.py b/api/db/db_models.pyindex 1b59e97e84..4ba7de0d65 100644--- a/api/db/db_models.py+++ b/api/db/db_models.py@@ -14,6 +14,7 @@# limitations under the License.#import hashlib+import heapqimport inspectimport loggingimport operator@@ -343,7 +344,77 @@ class JsonSerializedField(SerializedField):super(JsonSerializedField, self).__init__(serialized_type=SerializedType.JSON, object_hook=object_hook, object_pairs_hook=object_pairs_hook, **kwargs)-class RetryingPooledMySQLDatabase(PooledMySQLDatabase):+class PooledConnectionRetryMixin:+ """Recover pooled connections without replaying work in a dead transaction."""++ def _raise_if_connection_poisoned(self):+ error = getattr(self._state, "connection_loss", None)+ if error is not None:+ raise error++ def _evict_dead_connection(self):+ conn = self._state.conn+ if conn is None:+ return+ key = self.conn_key(conn)+ with self._pool_lock:+ self._in_use.pop(key, None)+ self._connections = [entry for entry in self._connections if entry[2] is not conn]+ heapq.heapify(self._connections)+ try:+ conn.close()+ except Exception:+ logging.debug("Failed to close dead database connection", exc_info=True)++ def _abort_transaction_after_connection_loss(self, error):+ self._evict_dead_connection()+ # Keep transactions until Peewee's context manager pops them. The+ # poison flag makes all SQL and final commit/rollback deterministic.+ self._state.connection_loss = error+ self._state.closed = True+ self._state.conn = None++ def pop_transaction(self):+ transaction = super().pop_transaction()+ if not self._state.transactions and getattr(self._state, "connection_loss", None) is not None:+ self._state.reset()+ self._state.connection_loss = None+ return transaction++ def commit(self):+ if getattr(self._state, "connection_loss", None) is not None:+ logging.debug("Suppressing commit after database connection loss")+ return+ return super().commit()++ def rollback(self):+ if getattr(self._state, "connection_loss", None) is not None:+ logging.debug("Suppressing rollback after database connection loss")+ return+ return super().rollback()++ def _handle_connection_loss(self):+ self._evict_dead_connection()+ self._state.reset()+ self._state.connection_loss = None+ try:+ self.connect()+ except Exception as e:+ logging.error("Failed to reconnect to %s: %s", getattr(self, "database_display_name", self.__class__.__name__), e)+ raise+++def _is_mysql_connection_error(error):+ args = getattr(error, "args", ())+ if args and isinstance(args[0], int) and args[0] in {2006, 2013}:+ return True+ if error.__class__.__name__ == "InterfaceError":+ return True+ message = str(error).lower()+ return any(text in message for text in ("lost connection", "gone away", "connection reset", "connection closed"))+++class RetryingPooledMySQLDatabase(PooledConnectionRetryMixin, PooledMySQLDatabase):def __init__(self, *args, **kwargs):self.max_retries = kwargs.pop("max_retries", 5)self.retry_delay = kwargs.pop("retry_delay", 1)@@ -352,11 +423,14 @@ class RetryingPooledMySQLDatabase(PooledMySQLDatabase):def execute_sql(self, sql, params=None, commit=True):for attempt in range(self.max_retries + 1):try:+ self._raise_if_connection_poisoned()return super().execute_sql(sql, params, commit)except (OperationalError, InterfaceError) as e:- error_codes = [2013, 2006]- error_messages = ["", "Lost connection"]- should_retry = (hasattr(e, "args") and e.args and e.args[0] in error_codes) or (str(e) in error_messages) or (hasattr(e, "__class__") and e.__class__.__name__ == "InterfaceError")+ should_retry = _is_mysql_connection_error(e)++ if should_retry and self.in_transaction():+ self._abort_transaction_after_connection_loss(e)+ raiseif should_retry and attempt < self.max_retries:logging.warning(f"Database connection issue (attempt {attempt + 1}/{self.max_retries}): {e}")@@ -367,33 +441,13 @@ class RetryingPooledMySQLDatabase(PooledMySQLDatabase):raisereturn None- def _handle_connection_loss(self):- # self.close_all()- # self.connect()- try:- self.close()- except Exception:- pass- try:- self.connect()- except Exception as e:- logging.error(f"Failed to reconnect: {e}")- time.sleep(0.1)- try:- self.connect()- except Exception as e2:- logging.error(f"Failed to reconnect on second attempt: {e2}")- raise-def begin(self):for attempt in range(self.max_retries + 1):try:+ self._raise_if_connection_poisoned()return super().begin()except (OperationalError, InterfaceError) as e:- error_codes = [2013, 2006]- error_messages = ["", "Lost connection"]-- should_retry = (hasattr(e, "args") and e.args and e.args[0] in error_codes) or (str(e) in error_messages) or (hasattr(e, "__class__") and e.__class__.__name__ == "InterfaceError")+ should_retry = _is_mysql_connection_error(e)if should_retry and attempt < self.max_retries:logging.warning(f"Lost connection during transaction (attempt {attempt + 1}/{self.max_retries})")@@ -404,7 +458,7 @@ class RetryingPooledMySQLDatabase(PooledMySQLDatabase):return None-class GaussDBPsycopgRetryMixin:+class GaussDBPsycopgRetryMixin(PooledConnectionRetryMixin):"""Connection retry behavior used only by the GaussDB adapter."""database_display_name = "GaussDB"@@ -420,13 +474,15 @@ class GaussDBPsycopgRetryMixin:def execute_sql(self, sql, params=None, commit=True):for attempt in range(self.max_retries + 1):try:+ self._raise_if_connection_poisoned()prepared_sql = self._prepare_sql_for_execution(sql)return super().execute_sql(prepared_sql, params, commit)except (OperationalError, InterfaceError) as e:# A reconnect cannot restore an active transaction. Let the# atomic block fail as a unit instead of retrying one statement# on a different connection.- if self.in_transaction():+ if self.in_transaction() and is_psycopg_connection_error(e):+ self._abort_transaction_after_connection_loss(e)logging.error(f"{self.database_display_name} execution failure: {e}")raise@@ -445,24 +501,12 @@ class GaussDBPsycopgRetryMixin:return Nonedef _handle_connection_loss(self):- try:- self.close()- except Exception:- pass- try:- self.connect()- except Exception as e:- logging.error(f"Failed to reconnect to {self.database_display_name}: {e}")- time.sleep(0.1)- try:- self.connect()- except Exception as e2:- logging.error(f"Failed to reconnect to {self.database_display_name} on second attempt: {e2}")- raise+ return super()._handle_connection_loss()def begin(self):for attempt in range(self.max_retries + 1):try:+ self._raise_if_connection_poisoned()return super().begin()except (OperationalError, InterfaceError) as e:# Apply the same connection-error classification during@@ -478,7 +522,7 @@ class GaussDBPsycopgRetryMixin:return None-class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):+class RetryingPooledPostgresqlDatabase(PooledConnectionRetryMixin, PooledPostgresqlDatabase):def __init__(self, *args, **kwargs):self.max_retries = kwargs.pop("max_retries", 5)self.retry_delay = kwargs.pop("retry_delay", 1)@@ -487,18 +531,14 @@ class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):def execute_sql(self, sql, params=None, commit=True):for attempt in range(self.max_retries + 1):try:+ self._raise_if_connection_poisoned()return super().execute_sql(sql, params, commit)except (OperationalError, InterfaceError) as e:- # PostgreSQL specific error codes- # 57P01: admin_shutdown- # 57P02: crash_shutdown- # 57P03: cannot_connect_now- # 08006: connection_failure- # 08003: connection_does_not_exist- # 08000: connection_exception- error_messages = ["connection", "server closed", "connection refused", "no connection to the server", "terminating connection"]+ should_retry = is_psycopg_connection_error(e)- should_retry = any(msg in str(e).lower() for msg in error_messages)+ if should_retry and self.in_transaction():+ self._abort_transaction_after_connection_loss(e)+ raiseif should_retry and attempt < self.max_retries:logging.warning(f"PostgreSQL connection issue (attempt {attempt + 1}/{self.max_retries}): {e}")@@ -510,29 +550,15 @@ class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):return Nonedef _handle_connection_loss(self):- try:- self.close()- except Exception:- pass- try:- self.connect()- except Exception as e:- logging.error(f"Failed to reconnect to PostgreSQL: {e}")- time.sleep(0.1)- try:- self.connect()- except Exception as e2:- logging.error(f"Failed to reconnect to PostgreSQL on second attempt: {e2}")- raise+ return super()._handle_connection_loss()def begin(self):for attempt in range(self.max_retries + 1):try:+ self._raise_if_connection_poisoned()return super().begin()except (OperationalError, InterfaceError) as e:- error_messages = ["connection", "server closed", "connection refused", "no connection to the server", "terminating connection"]-- should_retry = any(msg in str(e).lower() for msg in error_messages)+ should_retry = is_psycopg_connection_error(e)if should_retry and attempt < self.max_retries:logging.warning(f"PostgreSQL connection lost during transaction (attempt {attempt + 1}/{self.max_retries})")@@ -721,7 +747,7 @@ class RetryingPooledGaussDBDatabase(GaussDBPsycopgRetryMixin, PooledPostgresqlDareturn [table for (table,) in cursor.fetchall()]-class RetryingPooledOceanBaseDatabase(PooledMySQLDatabase):+class RetryingPooledOceanBaseDatabase(PooledConnectionRetryMixin, PooledMySQLDatabase):"""Pooled OceanBase database with retry mechanism.OceanBase is compatible with MySQL protocol, so we inherit from PooledMySQLDatabase.@@ -736,19 +762,14 @@ class RetryingPooledOceanBaseDatabase(PooledMySQLDatabase):def execute_sql(self, sql, params=None, commit=True):for attempt in range(self.max_retries + 1):try:+ self._raise_if_connection_poisoned()return super().execute_sql(sql, params, commit)except (OperationalError, InterfaceError) as e:- # OceanBase/MySQL specific error codes- # 2013: Lost connection to MySQL server during query- # 2006: MySQL server has gone away- error_codes = [2013, 2006]- error_messages = ["", "Lost connection", "gone away"]-- should_retry = (- (hasattr(e, "args") and e.args and e.args[0] in error_codes)- or any(msg in str(e).lower() for msg in error_messages)- or (hasattr(e, "__class__") and e.__class__.__name__ == "InterfaceError")- )+ should_retry = _is_mysql_connection_error(e)++ if should_retry and self.in_transaction():+ self._abort_transaction_after_connection_loss(e)+ raiseif should_retry and attempt < self.max_retries:logging.warning(f"OceanBase connection issue (attempt {attempt + 1}/{self.max_retries}): {e}")@@ -760,30 +781,15 @@ class RetryingPooledOceanBaseDatabase(PooledMySQLDatabase):return Nonedef _handle_connection_loss(self):- try:- self.close()- except Exception:- pass- try:- self.connect()- except Exception as e:- logging.error(f"Failed to reconnect to OceanBase: {e}")- time.sleep(0.1)- try:- self.connect()- except Exception as e2:- logging.error(f"Failed to reconnect to OceanBase on second attempt: {e2}")- raise+ return super()._handle_connection_loss()def begin(self):for attempt in range(self.max_retries + 1):try:+ self._raise_if_connection_poisoned()return super().begin()except (OperationalError, InterfaceError) as e:- error_codes = [2013, 2006]- error_messages = ["", "Lost connection"]-- should_retry = (hasattr(e, "args") and e.args and e.args[0] in error_codes) or (str(e) in error_messages) or (hasattr(e, "__class__") and e.__class__.__name__ == "InterfaceError")+ should_retry = _is_mysql_connection_error(e)if should_retry and attempt < self.max_retries:logging.warning(f"Lost connection during transaction (attempt {attempt + 1}/{self.max_retries})")diff --git a/test/unit_test/api/db/test_connection_retry.py b/test/unit_test/api/db/test_connection_retry.pynew file mode 100644index 0000000000..96e8e11caf--- /dev/null+++ b/test/unit_test/api/db/test_connection_retry.py@@ -0,0 +1,79 @@+import pytest+from peewee import OperationalError+from playhouse.pool import PooledMySQLDatabase++from api.db.db_models import RetryingPooledMySQLDatabase, _is_mysql_connection_error+++class DeadConnection:+ def __init__(self):+ self.closed = False++ def close(self):+ self.closed = True+++def test_mysql_connection_loss_retries_once_outside_transaction(monkeypatch):+ db = RetryingPooledMySQLDatabase(None, max_retries=1, retry_delay=0)+ calls = []+ reconnects = []++ def execute(_self, sql, params=None, commit=True):+ calls.append(sql)+ if len(calls) == 1:+ raise OperationalError(2013, "Lost connection")+ return "cursor"++ monkeypatch.setattr(PooledMySQLDatabase, "execute_sql", execute)+ monkeypatch.setattr(db, "in_transaction", lambda: False)+ monkeypatch.setattr(db, "_handle_connection_loss", lambda: reconnects.append(None))++ assert db.execute_sql("SELECT 1") == "cursor"+ assert calls == ["SELECT 1", "SELECT 1"]+ assert reconnects == [None]+++def test_mysql_connection_loss_poisoning_evicts_every_dead_transaction_connection(monkeypatch):+ db = RetryingPooledMySQLDatabase(None, max_retries=1, retry_delay=0)+ monkeypatch.setattr(db, "in_transaction", lambda: True)++ def execute(_self, sql, params=None, commit=True):+ raise OperationalError(2013, "Lost connection")++ monkeypatch.setattr(PooledMySQLDatabase, "execute_sql", execute)++ for _ in range(50):+ connection = DeadConnection()+ db._state.conn = connection+ db._state.transactions.append(object())+ db._in_use[db.conn_key(connection)] = object()+ db._connections = [(0, object(), connection)]++ with pytest.raises(OperationalError, match="Lost connection"):+ db.execute_sql("SELECT 1")++ assert connection.closed+ assert not db._in_use+ assert not db._connections+ db.pop_transaction()+ assert getattr(db._state, "connection_loss", None) is None+++def test_mysql_non_connection_error_is_not_retried(monkeypatch):+ db = RetryingPooledMySQLDatabase(None, max_retries=1, retry_delay=0)+ calls = []++ def execute(_self, sql, params=None, commit=True):+ calls.append(sql)+ raise OperationalError("syntax error")++ monkeypatch.setattr(PooledMySQLDatabase, "execute_sql", execute)++ with pytest.raises(OperationalError, match="syntax error"):+ db.execute_sql("SELECT 1")+ assert calls == ["SELECT 1"]++[email protected]("error", [OperationalError([]), OperationalError({})])+def test_mysql_connection_error_classifier_accepts_unhashable_codes(error):+ assert not _is_mysql_connection_error(error)