Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(storage): retry transient S3 failures (#19526)

Sbaaoui Idriss2dbce829904b
diff --git a/rag/utils/s3_conn.py b/rag/utils/s3_conn.py
index f284594026..5af1ce9c45 100644
--- a/rag/utils/s3_conn.py
+++ b/rag/utils/s3_conn.py
@@ -23,6 +23,8 @@ from io import BytesIO
from common.decorator import singleton
from common import settings
+MAX_ATTEMPTS = 3
+
@singleton
class RAGFlowS3:
@@ -72,7 +74,7 @@ class RAGFlowS3:
if self.conn:
self.__close__()
except Exception:
- pass
+ self.conn = None
try:
s3_params = {}
@@ -100,8 +102,11 @@ class RAGFlowS3:
s3_params["config"] = Config(**config_kwargs)
self.conn = [boto3.client("s3", **s3_params)]
+ return True
except Exception:
+ self.conn = None
logging.exception(f"Fail to connect at region {self.region_name} or endpoint {self.endpoint_url}")
+ return False
def __close__(self):
del self.conn[0]
@@ -144,7 +149,7 @@ class RAGFlowS3:
that use ``auto`` to select their own location.
"""
logging.debug(f"bucket name {bucket}; filename :{fnm}:")
- for _ in range(1):
+ for attempt in range(MAX_ATTEMPTS):
try:
if not self.bucket_exists(bucket):
bucket_config = {"Bucket": bucket}
@@ -159,9 +164,12 @@ class RAGFlowS3:
return r
except Exception:
- logging.exception(f"Fail put {bucket}/{fnm}")
- self.__open__()
- time.sleep(1)
+ logging.exception(f"Fail put {bucket}/{fnm}, attempt {attempt + 1}/{MAX_ATTEMPTS}")
+ if attempt == MAX_ATTEMPTS - 1:
+ raise
+ if not self.__open__():
+ raise
+ time.sleep(2**attempt)
@use_prefix_path
@use_default_bucket
@@ -174,16 +182,30 @@ class RAGFlowS3:
@use_prefix_path
@use_default_bucket
def get(self, bucket, fnm, *args, **kwargs):
- for _ in range(1):
+ for attempt in range(MAX_ATTEMPTS):
try:
r = self.conn[0].get_object(Bucket=bucket, Key=fnm)
object_data = r["Body"].read()
return object_data
+ except ClientError as e:
+ response = e.response if isinstance(e.response, dict) else {}
+ error = response.get("Error", {})
+ code = error.get("Code") if isinstance(error, dict) else None
+ if code in ("404", "NoSuchKey", "NotFound"):
+ return None
+ logging.exception(f"fail get {bucket}/{fnm}, attempt {attempt + 1}/{MAX_ATTEMPTS}")
+ if attempt == MAX_ATTEMPTS - 1:
+ raise
+ if not self.__open__():
+ raise
+ time.sleep(2**attempt)
except Exception:
- logging.exception(f"fail get {bucket}/{fnm}")
- self.__open__()
- time.sleep(1)
- return None
+ logging.exception(f"fail get {bucket}/{fnm}, attempt {attempt + 1}/{MAX_ATTEMPTS}")
+ if attempt == MAX_ATTEMPTS - 1:
+ raise
+ if not self.__open__():
+ raise
+ time.sleep(2**attempt)
@use_prefix_path
@use_default_bucket
diff --git a/test/unit_test/rag/utils/test_s3_conn.py b/test/unit_test/rag/utils/test_s3_conn.py
index c77178c8ee..0171256973 100644
--- a/test/unit_test/rag/utils/test_s3_conn.py
+++ b/test/unit_test/rag/utils/test_s3_conn.py
@@ -17,6 +17,9 @@
import importlib
from unittest.mock import Mock
+import pytest
+from botocore.exceptions import ClientError
+
# rag.utils.s3_conn and common.settings import each other (s3_conn needs
# settings.S3; settings re-imports RAGFlowS3). The cycle resolves only when
# common.settings is imported first, the same order the app uses. Importing it
@@ -63,3 +66,149 @@ def test_s3_health_returns_false_on_client_error(monkeypatch):
client.head_bucket.side_effect = ConnectionError("unavailable")
assert storage.health() is False
+
+
+def _client_error(code):
+ return ClientError({"Error": {"Code": code}}, "GetObject")
+
+
+def test_s3_put_succeeds_after_transient_failure(monkeypatch):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ client.upload_fileobj.side_effect = [ConnectionError("temporary"), None]
+ reconnect = Mock(return_value=True)
+ monkeypatch.setattr(storage, "__open__", reconnect)
+ sleep = Mock()
+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)
+
+ assert storage.put("kb", "file.txt", b"data") is None
+ assert client.upload_fileobj.call_count == 2
+ reconnect.assert_called_once_with()
+ sleep.assert_called_once_with(1)
+
+
+def test_s3_put_succeeds_on_first_attempt(monkeypatch):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+
+ assert storage.put("kb", "file.txt", b"data") is client.upload_fileobj.return_value
+ client.upload_fileobj.assert_called_once()
+
+
+def test_s3_put_succeeds_after_two_transient_failures(monkeypatch):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ client.upload_fileobj.side_effect = [ConnectionError("one"), ConnectionError("two"), None]
+ reconnect = Mock(return_value=True)
+ monkeypatch.setattr(storage, "__open__", reconnect)
+ sleep = Mock()
+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)
+
+ assert storage.put("kb", "file.txt", b"data") is None
+ assert client.upload_fileobj.call_count == 3
+ assert reconnect.call_count == 2
+ assert sleep.call_args_list == [((1,), {}), ((2,), {})]
+
+
+def test_s3_put_exhaustion_reraises_and_uses_expected_retries(monkeypatch):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ failure = ConnectionError("temporary")
+ client.upload_fileobj.side_effect = failure
+ reconnect = Mock(return_value=True)
+ monkeypatch.setattr(storage, "__open__", reconnect)
+ sleep = Mock()
+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)
+
+ with pytest.raises(ConnectionError) as raised:
+ storage.put("kb", "file.txt", b"data")
+
+ assert raised.value is failure
+ assert client.upload_fileobj.call_count == 3
+ assert reconnect.call_count == 2
+ assert sleep.call_args_list == [((1,), {}), ((2,), {})]
+
+
+def test_s3_put_reconnect_failure_preserves_operation_error(monkeypatch):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ failure = ConnectionError("upload failed")
+ client.upload_fileobj.side_effect = failure
+ reconnect = Mock(return_value=False)
+ monkeypatch.setattr(storage, "__open__", reconnect)
+ sleep = Mock()
+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)
+
+ with pytest.raises(ConnectionError) as raised:
+ storage.put("kb", "file.txt", b"data")
+
+ assert raised.value is failure
+ assert client.upload_fileobj.call_count == 1
+ sleep.assert_not_called()
+
+
+def test_s3_get_succeeds_after_transient_failure(monkeypatch):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ client.get_object.side_effect = [ConnectionError("temporary"), {"Body": Mock(read=Mock(return_value=b"data"))}]
+ reconnect = Mock(return_value=True)
+ monkeypatch.setattr(storage, "__open__", reconnect)
+ sleep = Mock()
+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)
+
+ assert storage.get("kb", "file.txt") == b"data"
+ assert client.get_object.call_count == 2
+ reconnect.assert_called_once_with()
+ sleep.assert_called_once_with(1)
+
+
+def test_s3_get_succeeds_on_first_attempt(monkeypatch):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ client.get_object.return_value = {"Body": Mock(read=Mock(return_value=b"data"))}
+
+ assert storage.get("kb", "file.txt") == b"data"
+ client.get_object.assert_called_once()
+
+
[email protected]("code", ["404", "NoSuchKey", "NotFound"])
+def test_s3_get_missing_object_codes_return_none_without_retry(monkeypatch, code):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ client.get_object.side_effect = _client_error(code)
+ reconnect = Mock(return_value=True)
+ monkeypatch.setattr(storage, "__open__", reconnect)
+ sleep = Mock()
+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)
+
+ assert storage.get("kb", "missing.txt") is None
+ client.get_object.assert_called_once_with(Bucket="ragflow", Key="missing.txt")
+ reconnect.assert_not_called()
+ sleep.assert_not_called()
+
+
+def test_s3_get_non_not_found_client_error_exhausts_retries(monkeypatch):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ failure = _client_error("AccessDenied")
+ client.get_object.side_effect = failure
+ reconnect = Mock(return_value=True)
+ monkeypatch.setattr(storage, "__open__", reconnect)
+ sleep = Mock()
+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)
+
+ with pytest.raises(ClientError) as raised:
+ storage.get("kb", "file.txt")
+
+ assert raised.value is failure
+ assert client.get_object.call_count == 3
+ assert reconnect.call_count == 2
+ assert sleep.call_args_list == [((1,), {}), ((2,), {})]
+
+
+def test_s3_get_reconnect_failure_preserves_operation_error(monkeypatch):
+ storage, client, _ = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ failure = ConnectionError("download failed")
+ client.get_object.side_effect = failure
+ reconnect = Mock(return_value=False)
+ monkeypatch.setattr(storage, "__open__", reconnect)
+ sleep = Mock()
+ monkeypatch.setattr(s3_conn.time, "sleep", sleep)
+
+ with pytest.raises(ConnectionError) as raised:
+ storage.get("kb", "file.txt")
+
+ assert raised.value is failure
+ assert client.get_object.call_count == 1
+ sleep.assert_not_called()