Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(doc_store): retry transient connection errors in pool init (#17196)

Harsh Kashyap824f9bfdcba5
diff --git a/common/doc_store/es_conn_pool.py b/common/doc_store/es_conn_pool.py
index c9139430cb..4f46d684fa 100644
--- a/common/doc_store/es_conn_pool.py
+++ b/common/doc_store/es_conn_pool.py
@@ -20,7 +20,9 @@ from elasticsearch import Elasticsearch
from common import settings
from common.decorator import singleton
-ATTEMPT_TIME = 2
+MAX_RETRIES = 6
+ATTEMPT_TIME = MAX_RETRIES
+HEALTH_CHECK_BASE_DELAY_SECONDS = 5
@singleton
@@ -31,16 +33,32 @@ class ElasticSearchConnectionPool:
else:
self.ES_CONFIG = settings.get_base_config("es", {})
- for _ in range(ATTEMPT_TIME):
+ for attempt in range(MAX_RETRIES):
try:
if self._connect():
break
except Exception as e:
- logging.warning(f"{str(e)}. Waiting Elasticsearch {self.ES_CONFIG['hosts']} to be healthy.")
- time.sleep(5)
+ logging.warning(
+ "Elasticsearch %s connection attempt %d/%d failed: %s",
+ self.ES_CONFIG["hosts"],
+ attempt + 1,
+ MAX_RETRIES,
+ e,
+ )
+ if attempt == MAX_RETRIES - 1:
+ raise
+ if hasattr(self, "es_conn") and self.es_conn:
+ try:
+ self.es_conn.close()
+ except Exception:
+ logging.exception(
+ "Failed to close partially-initialized Elasticsearch client before retry.",
+ )
+ time.sleep(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt))
+ continue
if not hasattr(self, "es_conn") or not self.es_conn or not self.es_conn.ping():
- msg = f"Elasticsearch {self.ES_CONFIG['hosts']} is unhealthy in 10s."
+ msg = f"Elasticsearch {self.ES_CONFIG['hosts']} is unhealthy after {MAX_RETRIES} attempts."
logging.error(msg)
raise Exception(msg)
v = self.info.get("version", {"number": "8.11.3"})
diff --git a/common/doc_store/infinity_conn_base.py b/common/doc_store/infinity_conn_base.py
index 26520a601b..52c3fbef00 100644
--- a/common/doc_store/infinity_conn_base.py
+++ b/common/doc_store/infinity_conn_base.py
@@ -85,6 +85,19 @@ def _int_env(name: str, default: int) -> int:
_META_RETRY_MAX = _int_env("INFINITY_META_RETRY_MAX", 5)
_META_RETRY_BASE_DELAY_MS = _int_env("INFINITY_META_RETRY_BASE_DELAY_MS", 50)
+# Health-check retry budget for the connection-pool initializers
+# (InfinityConnectionBase.__init__ and InfinityConnectionPool.__init__).
+# The startup wait budget: 8 attempts with base 5s, capped at
+# HEALTH_CHECK_MAX_DELAY_SECONDS per sleep, gives a worst case of
+# ~255s (5+10+20+40+60+60+60 for attempts 0-6, since the last attempt
+# never sleeps) -- well beyond the original 120s ceiling, but still
+# absorbing the brief blips that used to crash the worker at module
+# import. Keep this in mind when sizing startup timeouts and liveness
+# probes that wait on the initial connection.
+MAX_RETRIES = 8
+HEALTH_CHECK_BASE_DELAY_SECONDS = 5
+HEALTH_CHECK_MAX_DELAY_SECONDS = 60
+
def _is_meta_contention_error(exc: BaseException) -> bool:
"""Return True iff ``exc`` is the RocksDB metadata-counter "Resource busy".
@@ -168,7 +181,7 @@ class InfinityConnectionBase(DocStoreConnection):
self.connPool = None
self.logger.info(f"Use Infinity {infinity_uri} as the doc engine.")
conn_pool = INFINITY_CONN.get_conn_pool()
- for _ in range(24):
+ for attempt in range(MAX_RETRIES):
try:
inf_conn = conn_pool.get_conn()
res = inf_conn.show_current_node()
@@ -178,14 +191,32 @@ class InfinityConnectionBase(DocStoreConnection):
conn_pool.release_conn(inf_conn)
break
conn_pool.release_conn(inf_conn)
- self.logger.warning(f"Infinity status: {res.server_status}. Waiting Infinity {infinity_uri} to be healthy.")
- time.sleep(5)
+ self.logger.warning(
+ "Infinity status %s on attempt %d/%d, waiting Infinity %s to be healthy.",
+ res.server_status,
+ attempt + 1,
+ MAX_RETRIES,
+ infinity_uri,
+ )
+ if attempt == MAX_RETRIES - 1:
+ msg = f"Infinity {infinity_uri} status {res.server_status} after {MAX_RETRIES} attempts."
+ self.logger.error(msg)
+ raise Exception(msg)
+ time.sleep(min(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt), HEALTH_CHECK_MAX_DELAY_SECONDS))
except Exception as e:
+ self.logger.warning(
+ "Infinity connection attempt %d/%d to %s failed: %s",
+ attempt + 1,
+ MAX_RETRIES,
+ infinity_uri,
+ e,
+ )
+ if attempt == MAX_RETRIES - 1:
+ raise
conn_pool = INFINITY_CONN.refresh_conn_pool()
- self.logger.warning(f"{str(e)}. Waiting Infinity {infinity_uri} to be healthy.")
- time.sleep(5)
+ time.sleep(min(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt), HEALTH_CHECK_MAX_DELAY_SECONDS))
if self.connPool is None:
- msg = f"Infinity {infinity_uri} is unhealthy in 120s."
+ msg = f"Infinity {infinity_uri} is unhealthy after {MAX_RETRIES} attempts."
self.logger.error(msg)
raise Exception(msg)
self.logger.info(f"Infinity {infinity_uri} is healthy.")
diff --git a/common/doc_store/infinity_conn_pool.py b/common/doc_store/infinity_conn_pool.py
index 8eab676f75..88b93874bc 100644
--- a/common/doc_store/infinity_conn_pool.py
+++ b/common/doc_store/infinity_conn_pool.py
@@ -24,6 +24,10 @@ from infinity.errors import ErrorCode
from common import settings
from common.decorator import singleton
+MAX_RETRIES = 8
+HEALTH_CHECK_BASE_DELAY_SECONDS = 5
+HEALTH_CHECK_MAX_DELAY_SECONDS = 60
+
@singleton
class InfinityConnectionPool:
@@ -47,7 +51,7 @@ class InfinityConnectionPool:
self.infinity_uri = infinity.common.NetworkAddress(host, int(port))
self.conn_pool = None
- for _ in range(24):
+ for attempt in range(MAX_RETRIES):
conn_pool = None
inf_conn = None
try:
@@ -57,11 +61,29 @@ class InfinityConnectionPool:
if res.error_code == ErrorCode.OK and res.server_status in ["started", "alive"]:
self.conn_pool = conn_pool
break
- logging.warning(f"Infinity status: {res.server_status}. Waiting Infinity {infinity_uri} to be healthy.")
- time.sleep(5)
+ logging.warning(
+ "Infinity status %s on attempt %d/%d, waiting Infinity %s to be healthy.",
+ res.server_status,
+ attempt + 1,
+ MAX_RETRIES,
+ infinity_uri,
+ )
+ if attempt == MAX_RETRIES - 1:
+ msg = f"Infinity {infinity_uri} status {res.server_status} after {MAX_RETRIES} attempts."
+ logging.error(msg)
+ raise Exception(msg)
+ time.sleep(min(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt), HEALTH_CHECK_MAX_DELAY_SECONDS))
except Exception as e:
- logging.warning(f"{str(e)}. Waiting Infinity {infinity_uri} to be healthy.")
- time.sleep(5)
+ logging.warning(
+ "Infinity connection attempt %d/%d to %s failed: %s",
+ attempt + 1,
+ MAX_RETRIES,
+ infinity_uri,
+ e,
+ )
+ if attempt == MAX_RETRIES - 1:
+ raise
+ time.sleep(min(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt), HEALTH_CHECK_MAX_DELAY_SECONDS))
finally:
if inf_conn is not None and conn_pool is not None:
conn_pool.release_conn(inf_conn)
@@ -69,7 +91,7 @@ class InfinityConnectionPool:
conn_pool.destroy()
if self.conn_pool is None:
- msg = f"Infinity {infinity_uri} is unhealthy in 120s."
+ msg = f"Infinity {infinity_uri} is unhealthy after {MAX_RETRIES} attempts."
logging.error(msg)
raise Exception(msg)
diff --git a/common/doc_store/ob_conn_pool.py b/common/doc_store/ob_conn_pool.py
index d7e23ab748..5b7017e02e 100644
--- a/common/doc_store/ob_conn_pool.py
+++ b/common/doc_store/ob_conn_pool.py
@@ -25,7 +25,9 @@ from pyobvector.util import ObVersion
from common import settings
from common.decorator import singleton
-ATTEMPT_TIME = 2
+MAX_RETRIES = 6
+ATTEMPT_TIME = MAX_RETRIES
+HEALTH_CHECK_BASE_DELAY_SECONDS = 5
OB_QUERY_TIMEOUT = int(os.environ.get("OB_QUERY_TIMEOUT", "100_000_000"))
logger = logging.getLogger("ragflow.ob_conn_pool")
@@ -69,7 +71,7 @@ class OceanBaseConnectionPool:
max_overflow = int(os.environ.get("OB_MAX_OVERFLOW", max(max_connections // 2, 10)))
pool_timeout = int(os.environ.get("OB_POOL_TIMEOUT", "30"))
- for _ in range(ATTEMPT_TIME):
+ for attempt in range(MAX_RETRIES):
try:
self.client = ObVecClient(
uri=self.uri,
@@ -84,11 +86,27 @@ class OceanBaseConnectionPool:
)
break
except Exception as e:
- logger.warning(f"{str(e)}. Waiting OceanBase {self.uri} to be healthy.")
- time.sleep(5)
+ logger.warning(
+ "OceanBase %s connection attempt %d/%d failed: %s",
+ self.uri,
+ attempt + 1,
+ MAX_RETRIES,
+ e,
+ )
+ if attempt == MAX_RETRIES - 1:
+ raise
+ if self.client is not None:
+ try:
+ self.client.engine.dispose()
+ except Exception:
+ logger.exception(
+ "Failed to dispose partially-initialized OceanBase engine before retry.",
+ )
+ time.sleep(HEALTH_CHECK_BASE_DELAY_SECONDS * (2**attempt))
+ continue
if self.client is None:
- msg = f"OceanBase {self.uri} connection failed after {ATTEMPT_TIME} attempts."
+ msg = f"OceanBase {self.uri} connection failed after {MAX_RETRIES} attempts."
logger.error(msg)
raise Exception(msg)