RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(doc_store): retry transient connection errors in pool init (#17196)
Harsh Kashyap824f9bfdcba5
diff --git a/common/doc_store/es_conn_pool.py b/common/doc_store/es_conn_pool.pyindex c9139430cb..4f46d684fa 100644--- a/common/doc_store/es_conn_pool.py+++ b/common/doc_store/es_conn_pool.py@@ -20,7 +20,9 @@ from elasticsearch import Elasticsearchfrom common import settingsfrom common.decorator import singleton-ATTEMPT_TIME = 2+MAX_RETRIES = 6+ATTEMPT_TIME = MAX_RETRIES+HEALTH_CHECK_BASE_DELAY_SECONDS = 5@singleton@@ -31,16 +33,32 @@ class ElasticSearchConnectionPool:else:self.ES_CONFIG = settings.get_base_config("es", {})- for _ in range(ATTEMPT_TIME):+ for attempt in range(MAX_RETRIES):try:if self._connect():breakexcept Exception as e:- logging.warning(f"{str(e)}. Waiting Elasticsearch {self.ES_CONFIG['hosts']} to be healthy.")- time.sleep(5)+ logging.warning(+ "Elasticsearch %s connection attempt %d/%d failed: %s",+ self.ES_CONFIG["hosts"],+ attempt + 1,+ MAX_RETRIES,+ e,+ )+ if attempt == MAX_RETRIES - 1:+ raise+ if hasattr(self, "es_conn") and self.es_conn:+ try:+ self.es_conn.close()+ except Exception:+ logging.exception(+ "Failed to close partially-initialized Elasticsearch client before retry.",+ )+ time.sleep(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt))+ continueif not hasattr(self, "es_conn") or not self.es_conn or not self.es_conn.ping():- msg = f"Elasticsearch {self.ES_CONFIG['hosts']} is unhealthy in 10s."+ msg = f"Elasticsearch {self.ES_CONFIG['hosts']} is unhealthy after {MAX_RETRIES} attempts."logging.error(msg)raise Exception(msg)v = self.info.get("version", {"number": "8.11.3"})diff --git a/common/doc_store/infinity_conn_base.py b/common/doc_store/infinity_conn_base.pyindex 26520a601b..52c3fbef00 100644--- a/common/doc_store/infinity_conn_base.py+++ b/common/doc_store/infinity_conn_base.py@@ -85,6 +85,19 @@ def _int_env(name: str, default: int) -> int:_META_RETRY_MAX = _int_env("INFINITY_META_RETRY_MAX", 5)_META_RETRY_BASE_DELAY_MS = _int_env("INFINITY_META_RETRY_BASE_DELAY_MS", 50)+# Health-check retry budget for the connection-pool initializers+# (InfinityConnectionBase.__init__ and InfinityConnectionPool.__init__).+# The startup wait budget: 8 attempts with base 5s, capped at+# HEALTH_CHECK_MAX_DELAY_SECONDS per sleep, gives a worst case of+# ~255s (5+10+20+40+60+60+60 for attempts 0-6, since the last attempt+# never sleeps) -- well beyond the original 120s ceiling, but still+# absorbing the brief blips that used to crash the worker at module+# import. Keep this in mind when sizing startup timeouts and liveness+# probes that wait on the initial connection.+MAX_RETRIES = 8+HEALTH_CHECK_BASE_DELAY_SECONDS = 5+HEALTH_CHECK_MAX_DELAY_SECONDS = 60+def _is_meta_contention_error(exc: BaseException) -> bool:"""Return True iff ``exc`` is the RocksDB metadata-counter "Resource busy".@@ -168,7 +181,7 @@ class InfinityConnectionBase(DocStoreConnection):self.connPool = Noneself.logger.info(f"Use Infinity {infinity_uri} as the doc engine.")conn_pool = INFINITY_CONN.get_conn_pool()- for _ in range(24):+ for attempt in range(MAX_RETRIES):try:inf_conn = conn_pool.get_conn()res = inf_conn.show_current_node()@@ -178,14 +191,32 @@ class InfinityConnectionBase(DocStoreConnection):conn_pool.release_conn(inf_conn)breakconn_pool.release_conn(inf_conn)- self.logger.warning(f"Infinity status: {res.server_status}. Waiting Infinity {infinity_uri} to be healthy.")- time.sleep(5)+ self.logger.warning(+ "Infinity status %s on attempt %d/%d, waiting Infinity %s to be healthy.",+ res.server_status,+ attempt + 1,+ MAX_RETRIES,+ infinity_uri,+ )+ if attempt == MAX_RETRIES - 1:+ msg = f"Infinity {infinity_uri} status {res.server_status} after {MAX_RETRIES} attempts."+ self.logger.error(msg)+ raise Exception(msg)+ time.sleep(min(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt), HEALTH_CHECK_MAX_DELAY_SECONDS))except Exception as e:+ self.logger.warning(+ "Infinity connection attempt %d/%d to %s failed: %s",+ attempt + 1,+ MAX_RETRIES,+ infinity_uri,+ e,+ )+ if attempt == MAX_RETRIES - 1:+ raiseconn_pool = INFINITY_CONN.refresh_conn_pool()- self.logger.warning(f"{str(e)}. Waiting Infinity {infinity_uri} to be healthy.")- time.sleep(5)+ time.sleep(min(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt), HEALTH_CHECK_MAX_DELAY_SECONDS))if self.connPool is None:- msg = f"Infinity {infinity_uri} is unhealthy in 120s."+ msg = f"Infinity {infinity_uri} is unhealthy after {MAX_RETRIES} attempts."self.logger.error(msg)raise Exception(msg)self.logger.info(f"Infinity {infinity_uri} is healthy.")diff --git a/common/doc_store/infinity_conn_pool.py b/common/doc_store/infinity_conn_pool.pyindex 8eab676f75..88b93874bc 100644--- a/common/doc_store/infinity_conn_pool.py+++ b/common/doc_store/infinity_conn_pool.py@@ -24,6 +24,10 @@ from infinity.errors import ErrorCodefrom common import settingsfrom common.decorator import singleton+MAX_RETRIES = 8+HEALTH_CHECK_BASE_DELAY_SECONDS = 5+HEALTH_CHECK_MAX_DELAY_SECONDS = 60+@singletonclass InfinityConnectionPool:@@ -47,7 +51,7 @@ class InfinityConnectionPool:self.infinity_uri = infinity.common.NetworkAddress(host, int(port))self.conn_pool = None- for _ in range(24):+ for attempt in range(MAX_RETRIES):conn_pool = Noneinf_conn = Nonetry:@@ -57,11 +61,29 @@ class InfinityConnectionPool:if res.error_code == ErrorCode.OK and res.server_status in ["started", "alive"]:self.conn_pool = conn_poolbreak- logging.warning(f"Infinity status: {res.server_status}. Waiting Infinity {infinity_uri} to be healthy.")- time.sleep(5)+ logging.warning(+ "Infinity status %s on attempt %d/%d, waiting Infinity %s to be healthy.",+ res.server_status,+ attempt + 1,+ MAX_RETRIES,+ infinity_uri,+ )+ if attempt == MAX_RETRIES - 1:+ msg = f"Infinity {infinity_uri} status {res.server_status} after {MAX_RETRIES} attempts."+ logging.error(msg)+ raise Exception(msg)+ time.sleep(min(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt), HEALTH_CHECK_MAX_DELAY_SECONDS))except Exception as e:- logging.warning(f"{str(e)}. Waiting Infinity {infinity_uri} to be healthy.")- time.sleep(5)+ logging.warning(+ "Infinity connection attempt %d/%d to %s failed: %s",+ attempt + 1,+ MAX_RETRIES,+ infinity_uri,+ e,+ )+ if attempt == MAX_RETRIES - 1:+ raise+ time.sleep(min(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt), HEALTH_CHECK_MAX_DELAY_SECONDS))finally:if inf_conn is not None and conn_pool is not None:conn_pool.release_conn(inf_conn)@@ -69,7 +91,7 @@ class InfinityConnectionPool:conn_pool.destroy()if self.conn_pool is None:- msg = f"Infinity {infinity_uri} is unhealthy in 120s."+ msg = f"Infinity {infinity_uri} is unhealthy after {MAX_RETRIES} attempts."logging.error(msg)raise Exception(msg)diff --git a/common/doc_store/ob_conn_pool.py b/common/doc_store/ob_conn_pool.pyindex d7e23ab748..5b7017e02e 100644--- a/common/doc_store/ob_conn_pool.py+++ b/common/doc_store/ob_conn_pool.py@@ -25,7 +25,9 @@ from pyobvector.util import ObVersionfrom common import settingsfrom common.decorator import singleton-ATTEMPT_TIME = 2+MAX_RETRIES = 6+ATTEMPT_TIME = MAX_RETRIES+HEALTH_CHECK_BASE_DELAY_SECONDS = 5OB_QUERY_TIMEOUT = int(os.environ.get("OB_QUERY_TIMEOUT", "100_000_000"))logger = logging.getLogger("ragflow.ob_conn_pool")@@ -69,7 +71,7 @@ class OceanBaseConnectionPool:max_overflow = int(os.environ.get("OB_MAX_OVERFLOW", max(max_connections // 2, 10)))pool_timeout = int(os.environ.get("OB_POOL_TIMEOUT", "30"))- for _ in range(ATTEMPT_TIME):+ for attempt in range(MAX_RETRIES):try:self.client = ObVecClient(uri=self.uri,@@ -84,11 +86,27 @@ class OceanBaseConnectionPool:)breakexcept Exception as e:- logger.warning(f"{str(e)}. Waiting OceanBase {self.uri} to be healthy.")- time.sleep(5)+ logger.warning(+ "OceanBase %s connection attempt %d/%d failed: %s",+ self.uri,+ attempt + 1,+ MAX_RETRIES,+ e,+ )+ if attempt == MAX_RETRIES - 1:+ raise+ if self.client is not None:+ try:+ self.client.engine.dispose()+ except Exception:+ logger.exception(+ "Failed to dispose partially-initialized OceanBase engine before retry.",+ )+ time.sleep(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt))+ continueif self.client is None:- msg = f"OceanBase {self.uri} connection failed after {ATTEMPT_TIME} attempts."+ msg = f"OceanBase {self.uri} connection failed after {MAX_RETRIES} attempts."logger.error(msg)raise Exception(msg)