RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(storage):preserve errors during backend retries (#19578)
Sbaaoui Idriss9e932547e5b8
diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.pyindex 7428312e2e..10e477181a 100644--- a/api/apps/restful_apis/document_api.py+++ b/api/apps/restful_apis/document_api.py@@ -1797,7 +1797,7 @@ def _parse_document_image_id(image_id: str) -> tuple[str, str] | None:``(bucket, object_key)`` when valid, otherwise ``None``."""parts = image_id.split("-", 1)- if len(parts) != 2 or not parts[0] or not parts[1]:+ if len(parts) != 2 or not re.fullmatch(r"[0-9a-fA-F]{32}", parts[0]) or not parts[1]:return Nonereturn parts[0], parts[1]diff --git a/rag/utils/azure_sas_conn.py b/rag/utils/azure_sas_conn.pyindex 3706a7abd4..f89ab34cee 100644--- a/rag/utils/azure_sas_conn.py+++ b/rag/utils/azure_sas_conn.py@@ -53,13 +53,15 @@ class RAGFlowAzureSasBlob:def put(self, bucket, fnm, binary, tenant_id=None):blob_name = f"{bucket}/{fnm}"- for _ in range(3):+ for attempt in range(3):try:return self.conn.upload_blob(name=blob_name, data=BytesIO(binary), length=len(binary), overwrite=True)except Exception:logging.exception(f"Fail put {blob_name}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)def rm(self, bucket, fnm, tenant_id=None):try:@@ -69,14 +71,16 @@ class RAGFlowAzureSasBlob:def get(self, bucket, fnm, tenant_id=None):blob_name = f"{bucket}/{fnm}"- for _ in range(1):+ for attempt in range(3):try:r = self.conn.download_blob(blob_name)return r.read()except Exception:logging.exception(f"fail get {blob_name}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return Nonedef obj_exist(self, bucket, fnm, tenant_id=None):@@ -89,11 +93,13 @@ class RAGFlowAzureSasBlob:def get_presigned_url(self, bucket, fnm, expires):blob_name = f"{bucket}/{fnm}"- for _ in range(10):+ for attempt in range(3):try:return self.conn.get_presigned_url("GET", bucket, blob_name, expires)except Exception:logging.exception(f"fail get {blob_name}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return Nonediff --git a/rag/utils/azure_spn_conn.py b/rag/utils/azure_spn_conn.pyindex abd64b4f82..85148e72cf 100644--- a/rag/utils/azure_spn_conn.py+++ b/rag/utils/azure_spn_conn.py@@ -68,16 +68,17 @@ class RAGFlowAzureSpnBlob:def put(self, bucket, fnm, binary, tenant_id=None):blob = f"{bucket}/{fnm}"- for _ in range(3):+ for attempt in range(3):try:f = self.conn.create_file(f"{blob}")f.append_data(binary, offset=0, length=len(binary))return f.flush_data(len(binary))except Exception:logging.exception(f"Fail put {blob}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)- return None+ time.sleep(2**attempt)return Nonedef rm(self, bucket, fnm, tenant_id=None):@@ -89,15 +90,17 @@ class RAGFlowAzureSpnBlob:def get(self, bucket, fnm, tenant_id=None):blob = f"{bucket}/{fnm}"- for _ in range(1):+ for attempt in range(3):try:client = self.conn.get_file_client(f"{blob}")r = client.download_file()return r.read()except Exception:logging.exception(f"fail get {blob}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return Nonedef obj_exist(self, bucket, fnm, tenant_id=None):@@ -111,11 +114,13 @@ class RAGFlowAzureSpnBlob:def get_presigned_url(self, bucket, fnm, expires):f_path = f"{bucket}/{fnm}"- for _ in range(10):+ for attempt in range(3):try:return self.conn.get_presigned_url("GET", bucket, f_path, expires)except Exception:logging.exception(f"fail get {bucket}/{fnm}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return Nonediff --git a/rag/utils/gcs_conn.py b/rag/utils/gcs_conn.pyindex a46db79ec0..280b5a8869 100644--- a/rag/utils/gcs_conn.py+++ b/rag/utils/gcs_conn.py@@ -67,7 +67,7 @@ class RAGFlowGCS:def put(self, bucket, fnm, binary, tenant_id=None):# RENAMED PARAMETER: bucket_name -> bucket (to match interface)- for _ in range(3):+ for attempt in range(3):try:bucket_obj = self.client.bucket(self.bucket_name)blob_path = self._get_blob_path(bucket, fnm)@@ -80,8 +80,10 @@ class RAGFlowGCS:return Falseexcept Exception:logging.exception(f"Fail to put {bucket}/{fnm}:")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return Falsedef rm(self, bucket, fnm, tenant_id=None):@@ -98,7 +100,7 @@ class RAGFlowGCS:def get(self, bucket, filename, tenant_id=None):# RENAMED PARAMETER: bucket_name -> bucket- for _ in range(1):+ for attempt in range(3):try:bucket_obj = self.client.bucket(self.bucket_name)blob_path = self._get_blob_path(bucket, filename)@@ -109,8 +111,10 @@ class RAGFlowGCS:return Noneexcept Exception:logging.exception(f"Fail to get {bucket}/{filename}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return Nonedef obj_exist(self, bucket, filename, tenant_id=None):@@ -135,7 +139,7 @@ class RAGFlowGCS:def get_presigned_url(self, bucket, fnm, expires, tenant_id=None):# RENAMED PARAMETER: bucket_name -> bucket- for _ in range(10):+ for attempt in range(3):try:bucket_obj = self.client.bucket(self.bucket_name)blob_path = self._get_blob_path(bucket, fnm)@@ -149,8 +153,10 @@ class RAGFlowGCS:return urlexcept Exception:logging.exception(f"Fail to get_presigned {bucket}/{fnm}:")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return Nonedef remove_bucket(self, bucket):diff --git a/rag/utils/minio_conn.py b/rag/utils/minio_conn.pyindex 63dc4738ff..8ac6ddb4df 100644--- a/rag/utils/minio_conn.py+++ b/rag/utils/minio_conn.py@@ -143,7 +143,7 @@ class RAGFlowMinio:@use_default_bucket@use_prefix_pathdef put(self, bucket, fnm, binary, tenant_id=None):- for _ in range(3):+ for attempt in range(3):try:# Note: bucket must already exist - we don't have permission to create bucketsif not self.bucket and not self.conn.bucket_exists(bucket):@@ -153,8 +153,10 @@ class RAGFlowMinio:return rexcept Exception:logging.exception(f"Fail to put {bucket}/{fnm}:")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)@use_default_bucket@use_prefix_path@@ -167,14 +169,16 @@ class RAGFlowMinio:@use_default_bucket@use_prefix_pathdef get(self, bucket, filename, tenant_id=None):- for _ in range(1):+ for attempt in range(3):try:r = self.conn.get_object(bucket, filename)return r.read()except Exception:logging.exception(f"Fail to get {bucket}/{filename}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return@use_default_bucket@@ -211,13 +215,15 @@ class RAGFlowMinio:@use_default_bucket@use_prefix_pathdef get_presigned_url(self, bucket, fnm, expires, tenant_id=None):- for _ in range(10):+ for attempt in range(3):try:return self.conn.get_presigned_url("GET", bucket, fnm, expires)except Exception:logging.exception(f"Fail to get_presigned {bucket}/{fnm}:")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return@use_default_bucketdiff --git a/rag/utils/oss_conn.py b/rag/utils/oss_conn.pyindex 8e372b2c95..7fed29536e 100644--- a/rag/utils/oss_conn.py+++ b/rag/utils/oss_conn.py@@ -114,7 +114,7 @@ class RAGFlowOSS:@use_default_bucketdef put(self, bucket, fnm, binary, tenant_id=None):logging.debug(f"bucket name {bucket}; filename :{fnm}:")- for _ in range(1):+ for attempt in range(3):try:if not self.bucket_exists(bucket):self.conn.create_bucket(Bucket=bucket)@@ -124,8 +124,10 @@ class RAGFlowOSS:return rexcept Exception:logging.exception(f"Fail put {bucket}/{fnm}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)@use_prefix_path@use_default_bucket@@ -138,15 +140,17 @@ class RAGFlowOSS:@use_prefix_path@use_default_bucketdef get(self, bucket, fnm, tenant_id=None):- for _ in range(1):+ for attempt in range(3):try:r = self.conn.get_object(Bucket=bucket, Key=fnm)object_data = r["Body"].read()return object_dataexcept Exception:logging.exception(f"fail get {bucket}/{fnm}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return None@use_prefix_path@@ -164,13 +168,15 @@ class RAGFlowOSS:@use_prefix_path@use_default_bucketdef get_presigned_url(self, bucket, fnm, expires, tenant_id=None):- for _ in range(10):+ for attempt in range(3):try:r = self.conn.generate_presigned_url("get_object", Params={"Bucket": bucket, "Key": fnm}, ExpiresIn=expires)return rexcept Exception:logging.exception(f"fail get url {bucket}/{fnm}")+ if attempt == 2:+ raiseself.__open__()- time.sleep(1)+ time.sleep(2**attempt)return None