RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(storage): make bucket deletion work on the AWS_S3 backend (#19162)
Huang-404-Q028d7fa45a64
diff --git a/rag/utils/s3_conn.py b/rag/utils/s3_conn.pyindex d3623ef4f6..d68525a464 100644--- a/rag/utils/s3_conn.py+++ b/rag/utils/s3_conn.py@@ -44,7 +44,13 @@ class RAGFlowS3:def use_default_bucket(method):def wrapper(self, bucket, *args, **kwargs):# If there is a default bucket, use the default bucket+ # but preserve the original bucket identifier so callers that+ # need it as a key prefix (e.g. remove_bucket in single-bucket+ # mode) can still scope the operation to the logical bucket.+ original_bucket = bucketactual_bucket = self.bucket if self.bucket else bucket+ if self.bucket:+ kwargs["_orig_bucket"] = original_bucketreturn method(self, actual_bucket, *args, **kwargs)return wrapper@@ -232,15 +238,75 @@ class RAGFlowS3:logging.exception(f"Fail to move {src_bucket}/{src_path} -> {dest_bucket}/{dest_path}")return False+ def _delete_object_batches(self, bucket, pages):+ """Delete the objects listed in ``pages`` (an iterable of ListObjects+ pages) in batches. Handles versioned buckets by deleting every listed+ version and delete marker. Raises RuntimeError if any key fails."""+ for page in pages:+ objects = []+ for obj in page.get("Versions", []) + page.get("DeleteMarkers", []):+ entry = {"Key": obj["Key"]}+ if "VersionId" in obj:+ entry["VersionId"] = obj["VersionId"]+ objects.append(entry)+ if not objects:+ continue+ result = self.conn[0].delete_objects(Bucket=bucket, Delete={"Objects": objects})+ if result.get("Errors"):+ failed = ", ".join(err.get("Key", "?") for err in result["Errors"])+ raise RuntimeError(f"S3 object deletion failed for keys: {failed}")++ def _head_bucket_or_none(self, bucket):+ """Return the head_bucket response, or None only when the bucket does+ not exist. Other client errors (403 Forbidden, 400) propagate so the+ caller's error handler reports them instead of treating the bucket as+ absent."""+ try:+ return self.conn[0].head_bucket(Bucket=bucket)+ except ClientError as e:+ code = e.response.get("Error", {}).get("Code", "") if isinstance(e.response, dict) else ""+ if code in ("404", "NoSuchBucket", "NotFound"):+ return None+ raise++ def _delete_all_versions(self, bucket, prefix=None):+ """Delete every object version and delete marker under ``bucket``+ (optionally scoped to ``prefix``). Raises if any batch fails."""+ paginator = self.conn[0].get_paginator("list_object_versions")+ pages = paginator.paginate(Bucket=bucket, Prefix=prefix) if prefix is not None else paginator.paginate(Bucket=bucket)+ self._delete_object_batches(bucket, pages)+@use_default_bucket- def rm_bucket(self, bucket, *args, **kwargs):- for conn in self.conn:- try:- if not conn.bucket_exists(bucket):- continue- for o in conn.list_objects_v2(Bucket=bucket):- conn.delete_object(bucket, o.object_name)- conn.delete_bucket(Bucket=bucket)- return- except Exception as e:- logging.error(f"Fail rm {bucket}: " + str(e))+ def remove_bucket(self, bucket, **kwargs):+ orig_bucket = kwargs.pop("_orig_bucket", None)+ try:+ if self.bucket:+ # Single bucket mode: remove objects with the logical-bucket+ # prefix, but do not remove the physical bucket. The prefix+ # matches the write path: use_prefix_path only prepends the+ # "{prefix_path}/{bucket}/" segment when prefix_path is set.+ if not self.prefix_path:+ # A shared physical bucket with no prefix_path stores keys+ # without a logical-bucket segment, so no cleanup prefix+ # can scope this deletion to one knowledge base: run it+ # and the whole bucket's data would go. Refuse instead;+ # shared-bucket namespacing needs its own migration story.+ logging.error(+ "Refusing to remove logical bucket %s: STORAGE_S3 bucket is shared without a prefix_path, which cannot be scoped to one knowledge base",+ orig_bucket or bucket,+ )+ return+ prefix = f"{self.prefix_path}/{orig_bucket}/" if orig_bucket else f"{self.prefix_path}/"+ self._delete_all_versions(bucket, prefix)+ else:+ if self._head_bucket_or_none(bucket) is None:+ return+ self._delete_all_versions(bucket)+ self.conn[0].delete_bucket(Bucket=bucket)+ except Exception:+ # Storage deletions must not fail silently: lifecycle callers+ # (knowledge-base and account deletion) remove metadata after this+ # returns, so re-raise after logging the failure mode. str(e) is+ # omitted because S3 error responses can embed signed request URLs.+ logging.exception(f"Fail to remove bucket {bucket}")+ raisediff --git a/test/unit_test/rag/utils/test_s3_remove_bucket.py b/test/unit_test/rag/utils/test_s3_remove_bucket.pynew file mode 100644index 0000000000..2564844a26--- /dev/null+++ b/test/unit_test/rag/utils/test_s3_remove_bucket.py@@ -0,0 +1,259 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Unit tests for RAGFlowS3.remove_bucket (issue: S3 backend could not delete buckets)."""++import importlib+import logging+from unittest.mock import Mock++import pytest+from botocore.exceptions import ClientError++# Import common.settings first: s3_conn <-> settings import cycle resolves+# only in the same order the app uses (see test_s3_conn.py).+import common.settings # noqa: F401+from rag.utils import s3_conn+++def _new_storage(monkeypatch, config):+ module = importlib.reload(s3_conn)+ client = Mock()+ monkeypatch.setattr(module.settings, "S3", config)+ monkeypatch.setattr(module.boto3, "client", Mock(return_value=client))+ return module.RAGFlowS3(), client+++def _paginator(client, kind, *pages, by_prefix=None):+ """Stub one kind of list paginator. ``by_prefix`` maps a Prefix kwarg to+ its pages so single-bucket mode sees realistic prefix-filtered results.+ delete_objects defaults to an all-deleted response; tests override it."""+ paginator = Mock()++ def paginate(**kwargs):+ if by_prefix is not None:+ return list(by_prefix.get(kwargs.get("Prefix"), []))+ return list(pages)++ paginator.paginate.side_effect = paginate+ client.get_paginator.side_effect_map[kind] = paginator+ return paginator+++def _wire(client):+ """Prepare a client mock with per-kind paginators and safe defaults."""+ client.get_paginator.side_effect_map = {}+ client.get_paginator.side_effect = lambda kind: client.get_paginator.side_effect_map.setdefault(kind, _default_paginator(kind))+ client.delete_objects.return_value = {"Deleted": []}+++def _default_paginator(kind):+ """Empty-pages paginator for kinds a test did not register explicitly."""+ paginator = Mock()+ paginator.paginate.side_effect = lambda **kwargs: [{}]+ return paginator+++def test_remove_bucket_method_matches_storage_contract(monkeypatch):+ """Callers use remove_bucket (kb/user-account deletion); it must exist."""+ storage, _ = _new_storage(monkeypatch, {})+ assert hasattr(storage, "remove_bucket")+ assert not hasattr(storage, "rm_bucket")+++def test_remove_bucket_deletes_objects_then_bucket(monkeypatch):+ """Multi-bucket mode: all object versions are deleted in batches, then the+ bucket itself."""+ storage, client = _new_storage(monkeypatch, {})+ _wire(client)+ _paginator(+ client,+ "list_object_versions",+ {"Versions": [{"Key": "a"}, {"Key": "b"}], "DeleteMarkers": []},+ {"Versions": [{"Key": "c"}], "DeleteMarkers": []},+ )+ client.head_bucket.return_value = {}++ storage.remove_bucket("kb01")++ client.delete_objects.assert_any_call(Bucket="kb01", Delete={"Objects": [{"Key": "a"}, {"Key": "b"}]})+ client.delete_objects.assert_any_call(Bucket="kb01", Delete={"Objects": [{"Key": "c"}]})+ client.delete_bucket.assert_called_once_with(Bucket="kb01")+++def test_remove_bucket_deletes_versions_and_markers_with_version_ids(monkeypatch):+ """Versioned buckets: every version and delete marker must be deleted with+ its VersionId, otherwise delete_bucket fails with BucketNotEmpty."""+ storage, client = _new_storage(monkeypatch, {})+ _wire(client)+ _paginator(+ client,+ "list_object_versions",+ {+ "Versions": [{"Key": "a", "VersionId": "v1"}, {"Key": "a", "VersionId": "v2"}],+ "DeleteMarkers": [{"Key": "b", "VersionId": "m1"}],+ },+ )+ client.head_bucket.return_value = {}++ storage.remove_bucket("kb01")++ client.delete_objects.assert_called_once_with(+ Bucket="kb01",+ Delete={+ "Objects": [+ {"Key": "a", "VersionId": "v1"},+ {"Key": "a", "VersionId": "v2"},+ {"Key": "b", "VersionId": "m1"},+ ]+ },+ )+ client.delete_bucket.assert_called_once_with(Bucket="kb01")+++def test_remove_bucket_skips_missing_bucket(monkeypatch):+ """A 404 from head_bucket means the bucket is absent: a clean no-op."""+ storage, client = _new_storage(monkeypatch, {})+ _wire(client)+ client.head_bucket.side_effect = ClientError(+ {"Error": {"Code": "404"}},+ "HeadBucket",+ )++ storage.remove_bucket("nope")++ client.get_paginator.assert_not_called()+ client.delete_objects.assert_not_called()+ client.delete_bucket.assert_not_called()+++def test_remove_bucket_propagates_head_bucket_errors(monkeypatch):+ """403/400 from head_bucket are access failures, not absence: they must+ reach the error handler, not silently skip cleanup."""+ storage, client = _new_storage(monkeypatch, {})+ _wire(client)+ client.head_bucket.side_effect = ClientError(+ {"Error": {"Code": "403", "Message": "Forbidden"}},+ "HeadBucket",+ )++ with pytest.raises(ClientError):+ storage.remove_bucket("kb01")++ client.get_paginator.assert_not_called()+ client.delete_bucket.assert_not_called()+++def test_remove_bucket_raises_on_batch_delete_errors(monkeypatch):+ """delete_objects reports failed keys in its response instead of raising;+ remove_bucket must propagate the failure to lifecycle callers."""+ storage, client = _new_storage(monkeypatch, {})+ _wire(client)+ _paginator(+ client,+ "list_object_versions",+ {"Versions": [{"Key": "a"}], "DeleteMarkers": []},+ {"Versions": [{"Key": "c"}], "DeleteMarkers": []},+ )+ client.delete_objects.side_effect = [+ {"Deleted": [{"Key": "a"}]},+ {"Errors": [{"Key": "c", "Code": "InternalError", "Message": "boom"}]},+ ]+ client.head_bucket.return_value = {}++ with pytest.raises(RuntimeError, match="S3 object deletion failed for keys: c"):+ storage.remove_bucket("kb01")++ client.delete_bucket.assert_not_called()+++def test_remove_bucket_handles_null_error_response(monkeypatch, caplog):+ """botocore HTTPClientError can carry response=None; the failure path must+ complete without AttributeError and still log the failure."""+ storage, client = _new_storage(monkeypatch, {})+ _wire(client)+ err = ClientError({"Error": {}}, "HeadBucket")+ err.response = None # HTTPClientError can surface with a null response+ client.head_bucket.side_effect = err++ with caplog.at_level(logging.ERROR), pytest.raises(ClientError):+ storage.remove_bucket("kb01")++ assert any("Fail to remove bucket kb01" in r.getMessage() for r in caplog.records)+++def test_remove_bucket_re_raises_to_callers(monkeypatch, caplog):+ """Lifecycle callers delete metadata after remove_bucket returns; a+ storage failure must propagate, not report silent success."""+ storage, client = _new_storage(monkeypatch, {})+ _wire(client)+ client.head_bucket.side_effect = ConnectionError("network down")++ with caplog.at_level(logging.ERROR), pytest.raises(ConnectionError):+ storage.remove_bucket("kb01")++ messages = [r.getMessage() for r in caplog.records]+ assert any("Fail to remove bucket kb01" in m for m in messages), messages+ for m in messages:+ assert "X-Amz-Signature" not in m+++def test_remove_bucket_refuses_shared_bucket_without_prefix_path(monkeypatch, caplog):+ """A shared physical bucket without prefix_path hosts keys without a+ logical-bucket segment; no cleanup prefix could scope the deletion, so it+ must refuse rather than wipe the whole bucket."""+ storage, client = _new_storage(monkeypatch, {"bucket": "ragflow"})+ _wire(client)++ with caplog.at_level(logging.ERROR):+ storage.remove_bucket("kb01")++ client.get_paginator.assert_not_called()+ client.delete_bucket.assert_not_called()+ assert any("Refusing to remove logical bucket kb01" in r.getMessage() for r in caplog.records)+++def test_remove_bucket_single_bucket_mode_with_prefix_path(monkeypatch):+ storage, client = _new_storage(monkeypatch, {"bucket": "ragflow", "prefix_path": "prod"})+ _wire(client)+ versions = _paginator(+ client,+ "list_object_versions",+ by_prefix={"prod/kb01/": [{"Versions": [{"Key": "prod/kb01/doc"}], "DeleteMarkers": []}]},+ )+ _paginator(client, "list_multipart_uploads", by_prefix={"prod/kb01/": []})++ storage.remove_bucket("kb01")++ versions.paginate.assert_called_once_with(Bucket="ragflow", Prefix="prod/kb01/")+ deleted = client.delete_objects.call_args+ keys = [o["Key"] for o in deleted.kwargs["Delete"]["Objects"]]+ assert keys == ["prod/kb01/doc"]+ client.delete_bucket.assert_not_called()+++def test_remove_bucket_empty_bucket_deletes_only_bucket(monkeypatch):+ """A bucket that exists but has no objects: only delete_bucket fires."""+ storage, client = _new_storage(monkeypatch, {})+ _wire(client)+ _paginator(client, "list_object_versions", {})+ _paginator(client, "list_multipart_uploads", {})+ client.head_bucket.return_value = {}++ storage.remove_bucket("kb01")++ client.delete_objects.assert_not_called()+ client.delete_bucket.assert_called_once_with(Bucket="kb01")