RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(storage): retry transient S3 failures (#19526)
Sbaaoui Idriss2dbce829904b
diff --git a/rag/utils/s3_conn.py b/rag/utils/s3_conn.pyindex f284594026..5af1ce9c45 100644--- a/rag/utils/s3_conn.py+++ b/rag/utils/s3_conn.py@@ -23,6 +23,8 @@ from io import BytesIOfrom common.decorator import singletonfrom common import settings+MAX_ATTEMPTS = 3+@singletonclass RAGFlowS3:@@ -72,7 +74,7 @@ class RAGFlowS3:if self.conn:self.__close__()except Exception:- pass+ self.conn = Nonetry:s3_params = {}@@ -100,8 +102,11 @@ class RAGFlowS3:s3_params["config"] = Config(**config_kwargs)self.conn = [boto3.client("s3", **s3_params)]+ return Trueexcept Exception:+ self.conn = Nonelogging.exception(f"Fail to connect at region {self.region_name} or endpoint {self.endpoint_url}")+ return Falsedef __close__(self):del self.conn[0]@@ -144,7 +149,7 @@ class RAGFlowS3:that use ``auto`` to select their own location."""logging.debug(f"bucket name {bucket}; filename :{fnm}:")- for _ in range(1):+ for attempt in range(MAX_ATTEMPTS):try:if not self.bucket_exists(bucket):bucket_config = {"Bucket": bucket}@@ -159,9 +164,12 @@ class RAGFlowS3:return rexcept Exception:- logging.exception(f"Fail put {bucket}/{fnm}")- self.__open__()- time.sleep(1)+ logging.exception(f"Fail put {bucket}/{fnm}, attempt {attempt + 1}/{MAX_ATTEMPTS}")+ if attempt == MAX_ATTEMPTS - 1:+ raise+ if not self.__open__():+ raise+ time.sleep(2**attempt)@use_prefix_path@use_default_bucket@@ -174,16 +182,30 @@ class RAGFlowS3:@use_prefix_path@use_default_bucketdef get(self, bucket, fnm, *args, **kwargs):- for _ in range(1):+ for attempt in range(MAX_ATTEMPTS):try:r = self.conn[0].get_object(Bucket=bucket, Key=fnm)object_data = r["Body"].read()return object_data+ except ClientError as e:+ response = e.response if isinstance(e.response, dict) else {}+ error = response.get("Error", {})+ code = error.get("Code") if isinstance(error, dict) else None+ if code in ("404", "NoSuchKey", "NotFound"):+ return None+ logging.exception(f"fail get {bucket}/{fnm}, attempt {attempt + 1}/{MAX_ATTEMPTS}")+ if attempt == MAX_ATTEMPTS - 1:+ raise+ if not self.__open__():+ raise+ time.sleep(2**attempt)except Exception:- logging.exception(f"fail get {bucket}/{fnm}")- self.__open__()- time.sleep(1)- return None+ logging.exception(f"fail get {bucket}/{fnm}, attempt {attempt + 1}/{MAX_ATTEMPTS}")+ if attempt == MAX_ATTEMPTS - 1:+ raise+ if not self.__open__():+ raise+ time.sleep(2**attempt)@use_prefix_path@use_default_bucketdiff --git a/test/unit_test/rag/utils/test_s3_conn.py b/test/unit_test/rag/utils/test_s3_conn.pyindex c77178c8ee..0171256973 100644--- a/test/unit_test/rag/utils/test_s3_conn.py+++ b/test/unit_test/rag/utils/test_s3_conn.py@@ -17,6 +17,9 @@import importlibfrom unittest.mock import Mock+import pytest+from botocore.exceptions import ClientError+# rag.utils.s3_conn and common.settings import each other (s3_conn needs# settings.S3; settings re-imports RAGFlowS3). The cycle resolves only when# common.settings is imported first, the same order the app uses. Importing it@@ -63,3 +66,149 @@ def test_s3_health_returns_false_on_client_error(monkeypatch):client.head_bucket.side_effect = ConnectionError("unavailable")assert storage.health() is False+++def _client_error(code):+ return ClientError({"Error": {"Code": code}}, "GetObject")+++def test_s3_put_succeeds_after_transient_failure(monkeypatch):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})+ client.upload_fileobj.side_effect = [ConnectionError("temporary"), None]+ reconnect = Mock(return_value=True)+ monkeypatch.setattr(storage, "__open__", reconnect)+ sleep = Mock()+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)++ assert storage.put("kb", "file.txt", b"data") is None+ assert client.upload_fileobj.call_count == 2+ reconnect.assert_called_once_with()+ sleep.assert_called_once_with(1)+++def test_s3_put_succeeds_on_first_attempt(monkeypatch):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})++ assert storage.put("kb", "file.txt", b"data") is client.upload_fileobj.return_value+ client.upload_fileobj.assert_called_once()+++def test_s3_put_succeeds_after_two_transient_failures(monkeypatch):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})+ client.upload_fileobj.side_effect = [ConnectionError("one"), ConnectionError("two"), None]+ reconnect = Mock(return_value=True)+ monkeypatch.setattr(storage, "__open__", reconnect)+ sleep = Mock()+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)++ assert storage.put("kb", "file.txt", b"data") is None+ assert client.upload_fileobj.call_count == 3+ assert reconnect.call_count == 2+ assert sleep.call_args_list == [((1,), {}), ((2,), {})]+++def test_s3_put_exhaustion_reraises_and_uses_expected_retries(monkeypatch):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})+ failure = ConnectionError("temporary")+ client.upload_fileobj.side_effect = failure+ reconnect = Mock(return_value=True)+ monkeypatch.setattr(storage, "__open__", reconnect)+ sleep = Mock()+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)++ with pytest.raises(ConnectionError) as raised:+ storage.put("kb", "file.txt", b"data")++ assert raised.value is failure+ assert client.upload_fileobj.call_count == 3+ assert reconnect.call_count == 2+ assert sleep.call_args_list == [((1,), {}), ((2,), {})]+++def test_s3_put_reconnect_failure_preserves_operation_error(monkeypatch):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})+ failure = ConnectionError("upload failed")+ client.upload_fileobj.side_effect = failure+ reconnect = Mock(return_value=False)+ monkeypatch.setattr(storage, "__open__", reconnect)+ sleep = Mock()+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)++ with pytest.raises(ConnectionError) as raised:+ storage.put("kb", "file.txt", b"data")++ assert raised.value is failure+ assert client.upload_fileobj.call_count == 1+ sleep.assert_not_called()+++def test_s3_get_succeeds_after_transient_failure(monkeypatch):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})+ client.get_object.side_effect = [ConnectionError("temporary"), {"Body": Mock(read=Mock(return_value=b"data"))}]+ reconnect = Mock(return_value=True)+ monkeypatch.setattr(storage, "__open__", reconnect)+ sleep = Mock()+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)++ assert storage.get("kb", "file.txt") == b"data"+ assert client.get_object.call_count == 2+ reconnect.assert_called_once_with()+ sleep.assert_called_once_with(1)+++def test_s3_get_succeeds_on_first_attempt(monkeypatch):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})+ client.get_object.return_value = {"Body": Mock(read=Mock(return_value=b"data"))}++ assert storage.get("kb", "file.txt") == b"data"+ client.get_object.assert_called_once()++[email protected]("code", ["404", "NoSuchKey", "NotFound"])+def test_s3_get_missing_object_codes_return_none_without_retry(monkeypatch, code):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})+ client.get_object.side_effect = _client_error(code)+ reconnect = Mock(return_value=True)+ monkeypatch.setattr(storage, "__open__", reconnect)+ sleep = Mock()+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)++ assert storage.get("kb", "missing.txt") is None+ client.get_object.assert_called_once_with(Bucket="ragflow", Key="missing.txt")+ reconnect.assert_not_called()+ sleep.assert_not_called()+++def test_s3_get_non_not_found_client_error_exhausts_retries(monkeypatch):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})+ failure = _client_error("AccessDenied")+ client.get_object.side_effect = failure+ reconnect = Mock(return_value=True)+ monkeypatch.setattr(storage, "__open__", reconnect)+ sleep = Mock()+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)++ with pytest.raises(ClientError) as raised:+ storage.get("kb", "file.txt")++ assert raised.value is failure+ assert client.get_object.call_count == 3+ assert reconnect.call_count == 2+ assert sleep.call_args_list == [((1,), {}), ((2,), {})]+++def test_s3_get_reconnect_failure_preserves_operation_error(monkeypatch):+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})+ failure = ConnectionError("download failed")+ client.get_object.side_effect = failure+ reconnect = Mock(return_value=False)+ monkeypatch.setattr(storage, "__open__", reconnect)+ sleep = Mock()+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)++ with pytest.raises(ConnectionError) as raised:+ storage.get("kb", "file.txt")++ assert raised.value is failure+ assert client.get_object.call_count == 1+ sleep.assert_not_called()