Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(storage): make bucket deletion work on the AWS_S3 backend (#19162)

Huang-404-Q028d7fa45a64
diff --git a/rag/utils/s3_conn.py b/rag/utils/s3_conn.py
index d3623ef4f6..d68525a464 100644
--- a/rag/utils/s3_conn.py
+++ b/rag/utils/s3_conn.py
@@ -44,7 +44,13 @@ class RAGFlowS3:
def use_default_bucket(method):
def wrapper(self, bucket, *args, **kwargs):
# If there is a default bucket, use the default bucket
+ # but preserve the original bucket identifier so callers that
+ # need it as a key prefix (e.g. remove_bucket in single-bucket
+ # mode) can still scope the operation to the logical bucket.
+ original_bucket = bucket
actual_bucket = self.bucket if self.bucket else bucket
+ if self.bucket:
+ kwargs["_orig_bucket"] = original_bucket
return method(self, actual_bucket, *args, **kwargs)
return wrapper
@@ -232,15 +238,75 @@ class RAGFlowS3:
logging.exception(f"Fail to move {src_bucket}/{src_path} -> {dest_bucket}/{dest_path}")
return False
+ def _delete_object_batches(self, bucket, pages):
+ """Delete the objects listed in ``pages`` (an iterable of ListObjects
+ pages) in batches. Handles versioned buckets by deleting every listed
+ version and delete marker. Raises RuntimeError if any key fails."""
+ for page in pages:
+ objects = []
+ for obj in page.get("Versions", []) + page.get("DeleteMarkers", []):
+ entry = {"Key": obj["Key"]}
+ if "VersionId" in obj:
+ entry["VersionId"] = obj["VersionId"]
+ objects.append(entry)
+ if not objects:
+ continue
+ result = self.conn[0].delete_objects(Bucket=bucket, Delete={"Objects": objects})
+ if result.get("Errors"):
+ failed = ", ".join(err.get("Key", "?") for err in result["Errors"])
+ raise RuntimeError(f"S3 object deletion failed for keys: {failed}")
+
+ def _head_bucket_or_none(self, bucket):
+ """Return the head_bucket response, or None only when the bucket does
+ not exist. Other client errors (403 Forbidden, 400) propagate so the
+ caller's error handler reports them instead of treating the bucket as
+ absent."""
+ try:
+ return self.conn[0].head_bucket(Bucket=bucket)
+ except ClientError as e:
+ code = e.response.get("Error", {}).get("Code", "") if isinstance(e.response, dict) else ""
+ if code in ("404", "NoSuchBucket", "NotFound"):
+ return None
+ raise
+
+ def _delete_all_versions(self, bucket, prefix=None):
+ """Delete every object version and delete marker under ``bucket``
+ (optionally scoped to ``prefix``). Raises if any batch fails."""
+ paginator = self.conn[0].get_paginator("list_object_versions")
+ pages = paginator.paginate(Bucket=bucket, Prefix=prefix) if prefix is not None else paginator.paginate(Bucket=bucket)
+ self._delete_object_batches(bucket, pages)
+
@use_default_bucket
- def rm_bucket(self, bucket, *args, **kwargs):
- for conn in self.conn:
- try:
- if not conn.bucket_exists(bucket):
- continue
- for o in conn.list_objects_v2(Bucket=bucket):
- conn.delete_object(bucket, o.object_name)
- conn.delete_bucket(Bucket=bucket)
- return
- except Exception as e:
- logging.error(f"Fail rm {bucket}: " + str(e))
+ def remove_bucket(self, bucket, **kwargs):
+ orig_bucket = kwargs.pop("_orig_bucket", None)
+ try:
+ if self.bucket:
+ # Single bucket mode: remove objects with the logical-bucket
+ # prefix, but do not remove the physical bucket. The prefix
+ # matches the write path: use_prefix_path only prepends the
+ # "{prefix_path}/{bucket}/" segment when prefix_path is set.
+ if not self.prefix_path:
+ # A shared physical bucket with no prefix_path stores keys
+ # without a logical-bucket segment, so no cleanup prefix
+ # can scope this deletion to one knowledge base: run it
+ # and the whole bucket's data would go. Refuse instead;
+ # shared-bucket namespacing needs its own migration story.
+ logging.error(
+ "Refusing to remove logical bucket %s: STORAGE_S3 bucket is shared without a prefix_path, which cannot be scoped to one knowledge base",
+ orig_bucket or bucket,
+ )
+ return
+ prefix = f"{self.prefix_path}/{orig_bucket}/" if orig_bucket else f"{self.prefix_path}/"
+ self._delete_all_versions(bucket, prefix)
+ else:
+ if self._head_bucket_or_none(bucket) is None:
+ return
+ self._delete_all_versions(bucket)
+ self.conn[0].delete_bucket(Bucket=bucket)
+ except Exception:
+ # Storage deletions must not fail silently: lifecycle callers
+ # (knowledge-base and account deletion) remove metadata after this
+ # returns, so re-raise after logging the failure mode. str(e) is
+ # omitted because S3 error responses can embed signed request URLs.
+ logging.exception(f"Fail to remove bucket {bucket}")
+ raise
diff --git a/test/unit_test/rag/utils/test_s3_remove_bucket.py b/test/unit_test/rag/utils/test_s3_remove_bucket.py
new file mode 100644
index 0000000000..2564844a26
--- /dev/null
+++ b/test/unit_test/rag/utils/test_s3_remove_bucket.py
@@ -0,0 +1,259 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unit tests for RAGFlowS3.remove_bucket (issue: S3 backend could not delete buckets)."""
+
+import importlib
+import logging
+from unittest.mock import Mock
+
+import pytest
+from botocore.exceptions import ClientError
+
+# Import common.settings first: s3_conn <-> settings import cycle resolves
+# only in the same order the app uses (see test_s3_conn.py).
+import common.settings # noqa: F401
+from rag.utils import s3_conn
+
+
+def _new_storage(monkeypatch, config):
+ module = importlib.reload(s3_conn)
+ client = Mock()
+ monkeypatch.setattr(module.settings, "S3", config)
+ monkeypatch.setattr(module.boto3, "client", Mock(return_value=client))
+ return module.RAGFlowS3(), client
+
+
+def _paginator(client, kind, *pages, by_prefix=None):
+ """Stub one kind of list paginator. ``by_prefix`` maps a Prefix kwarg to
+ its pages so single-bucket mode sees realistic prefix-filtered results.
+ delete_objects defaults to an all-deleted response; tests override it."""
+ paginator = Mock()
+
+ def paginate(**kwargs):
+ if by_prefix is not None:
+ return list(by_prefix.get(kwargs.get("Prefix"), []))
+ return list(pages)
+
+ paginator.paginate.side_effect = paginate
+ client.get_paginator.side_effect_map[kind] = paginator
+ return paginator
+
+
+def _wire(client):
+ """Prepare a client mock with per-kind paginators and safe defaults."""
+ client.get_paginator.side_effect_map = {}
+ client.get_paginator.side_effect = lambda kind: client.get_paginator.side_effect_map.setdefault(kind, _default_paginator(kind))
+ client.delete_objects.return_value = {"Deleted": []}
+
+
+def _default_paginator(kind):
+ """Empty-pages paginator for kinds a test did not register explicitly."""
+ paginator = Mock()
+ paginator.paginate.side_effect = lambda **kwargs: [{}]
+ return paginator
+
+
+def test_remove_bucket_method_matches_storage_contract(monkeypatch):
+ """Callers use remove_bucket (kb/user-account deletion); it must exist."""
+ storage, _ = _new_storage(monkeypatch, {})
+ assert hasattr(storage, "remove_bucket")
+ assert not hasattr(storage, "rm_bucket")
+
+
+def test_remove_bucket_deletes_objects_then_bucket(monkeypatch):
+ """Multi-bucket mode: all object versions are deleted in batches, then the
+ bucket itself."""
+ storage, client = _new_storage(monkeypatch, {})
+ _wire(client)
+ _paginator(
+ client,
+ "list_object_versions",
+ {"Versions": [{"Key": "a"}, {"Key": "b"}], "DeleteMarkers": []},
+ {"Versions": [{"Key": "c"}], "DeleteMarkers": []},
+ )
+ client.head_bucket.return_value = {}
+
+ storage.remove_bucket("kb01")
+
+ client.delete_objects.assert_any_call(Bucket="kb01", Delete={"Objects": [{"Key": "a"}, {"Key": "b"}]})
+ client.delete_objects.assert_any_call(Bucket="kb01", Delete={"Objects": [{"Key": "c"}]})
+ client.delete_bucket.assert_called_once_with(Bucket="kb01")
+
+
+def test_remove_bucket_deletes_versions_and_markers_with_version_ids(monkeypatch):
+ """Versioned buckets: every version and delete marker must be deleted with
+ its VersionId, otherwise delete_bucket fails with BucketNotEmpty."""
+ storage, client = _new_storage(monkeypatch, {})
+ _wire(client)
+ _paginator(
+ client,
+ "list_object_versions",
+ {
+ "Versions": [{"Key": "a", "VersionId": "v1"}, {"Key": "a", "VersionId": "v2"}],
+ "DeleteMarkers": [{"Key": "b", "VersionId": "m1"}],
+ },
+ )
+ client.head_bucket.return_value = {}
+
+ storage.remove_bucket("kb01")
+
+ client.delete_objects.assert_called_once_with(
+ Bucket="kb01",
+ Delete={
+ "Objects": [
+ {"Key": "a", "VersionId": "v1"},
+ {"Key": "a", "VersionId": "v2"},
+ {"Key": "b", "VersionId": "m1"},
+ ]
+ },
+ )
+ client.delete_bucket.assert_called_once_with(Bucket="kb01")
+
+
+def test_remove_bucket_skips_missing_bucket(monkeypatch):
+ """A 404 from head_bucket means the bucket is absent: a clean no-op."""
+ storage, client = _new_storage(monkeypatch, {})
+ _wire(client)
+ client.head_bucket.side_effect = ClientError(
+ {"Error": {"Code": "404"}},
+ "HeadBucket",
+ )
+
+ storage.remove_bucket("nope")
+
+ client.get_paginator.assert_not_called()
+ client.delete_objects.assert_not_called()
+ client.delete_bucket.assert_not_called()
+
+
+def test_remove_bucket_propagates_head_bucket_errors(monkeypatch):
+ """403/400 from head_bucket are access failures, not absence: they must
+ reach the error handler, not silently skip cleanup."""
+ storage, client = _new_storage(monkeypatch, {})
+ _wire(client)
+ client.head_bucket.side_effect = ClientError(
+ {"Error": {"Code": "403", "Message": "Forbidden"}},
+ "HeadBucket",
+ )
+
+ with pytest.raises(ClientError):
+ storage.remove_bucket("kb01")
+
+ client.get_paginator.assert_not_called()
+ client.delete_bucket.assert_not_called()
+
+
+def test_remove_bucket_raises_on_batch_delete_errors(monkeypatch):
+ """delete_objects reports failed keys in its response instead of raising;
+ remove_bucket must propagate the failure to lifecycle callers."""
+ storage, client = _new_storage(monkeypatch, {})
+ _wire(client)
+ _paginator(
+ client,
+ "list_object_versions",
+ {"Versions": [{"Key": "a"}], "DeleteMarkers": []},
+ {"Versions": [{"Key": "c"}], "DeleteMarkers": []},
+ )
+ client.delete_objects.side_effect = [
+ {"Deleted": [{"Key": "a"}]},
+ {"Errors": [{"Key": "c", "Code": "InternalError", "Message": "boom"}]},
+ ]
+ client.head_bucket.return_value = {}
+
+ with pytest.raises(RuntimeError, match="S3 object deletion failed for keys: c"):
+ storage.remove_bucket("kb01")
+
+ client.delete_bucket.assert_not_called()
+
+
+def test_remove_bucket_handles_null_error_response(monkeypatch, caplog):
+ """botocore HTTPClientError can carry response=None; the failure path must
+ complete without AttributeError and still log the failure."""
+ storage, client = _new_storage(monkeypatch, {})
+ _wire(client)
+ err = ClientError({"Error": {}}, "HeadBucket")
+ err.response = None # HTTPClientError can surface with a null response
+ client.head_bucket.side_effect = err
+
+ with caplog.at_level(logging.ERROR), pytest.raises(ClientError):
+ storage.remove_bucket("kb01")
+
+ assert any("Fail to remove bucket kb01" in r.getMessage() for r in caplog.records)
+
+
+def test_remove_bucket_re_raises_to_callers(monkeypatch, caplog):
+ """Lifecycle callers delete metadata after remove_bucket returns; a
+ storage failure must propagate, not report silent success."""
+ storage, client = _new_storage(monkeypatch, {})
+ _wire(client)
+ client.head_bucket.side_effect = ConnectionError("network down")
+
+ with caplog.at_level(logging.ERROR), pytest.raises(ConnectionError):
+ storage.remove_bucket("kb01")
+
+ messages = [r.getMessage() for r in caplog.records]
+ assert any("Fail to remove bucket kb01" in m for m in messages), messages
+ for m in messages:
+ assert "X-Amz-Signature" not in m
+
+
+def test_remove_bucket_refuses_shared_bucket_without_prefix_path(monkeypatch, caplog):
+ """A shared physical bucket without prefix_path hosts keys without a
+ logical-bucket segment; no cleanup prefix could scope the deletion, so it
+ must refuse rather than wipe the whole bucket."""
+ storage, client = _new_storage(monkeypatch, {"bucket": "ragflow"})
+ _wire(client)
+
+ with caplog.at_level(logging.ERROR):
+ storage.remove_bucket("kb01")
+
+ client.get_paginator.assert_not_called()
+ client.delete_bucket.assert_not_called()
+ assert any("Refusing to remove logical bucket kb01" in r.getMessage() for r in caplog.records)
+
+
+def test_remove_bucket_single_bucket_mode_with_prefix_path(monkeypatch):
+ storage, client = _new_storage(monkeypatch, {"bucket": "ragflow", "prefix_path": "prod"})
+ _wire(client)
+ versions = _paginator(
+ client,
+ "list_object_versions",
+ by_prefix={"prod/kb01/": [{"Versions": [{"Key": "prod/kb01/doc"}], "DeleteMarkers": []}]},
+ )
+ _paginator(client, "list_multipart_uploads", by_prefix={"prod/kb01/": []})
+
+ storage.remove_bucket("kb01")
+
+ versions.paginate.assert_called_once_with(Bucket="ragflow", Prefix="prod/kb01/")
+ deleted = client.delete_objects.call_args
+ keys = [o["Key"] for o in deleted.kwargs["Delete"]["Objects"]]
+ assert keys == ["prod/kb01/doc"]
+ client.delete_bucket.assert_not_called()
+
+
+def test_remove_bucket_empty_bucket_deletes_only_bucket(monkeypatch):
+ """A bucket that exists but has no objects: only delete_bucket fires."""
+ storage, client = _new_storage(monkeypatch, {})
+ _wire(client)
+ _paginator(client, "list_object_versions", {})
+ _paginator(client, "list_multipart_uploads", {})
+ client.head_bucket.return_value = {}
+
+ storage.remove_bucket("kb01")
+
+ client.delete_objects.assert_not_called()
+ client.delete_bucket.assert_called_once_with(Bucket="kb01")