Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(storage): honor client region in Python S3 bucket creation (#19291)

Wenchao An9d6f76d2173a
diff --git a/internal/server/config/storage_engine_config.go b/internal/server/config/storage_engine_config.go
index 2d3191d5a6..21b514e3b5 100644
--- a/internal/server/config/storage_engine_config.go
+++ b/internal/server/config/storage_engine_config.go
@@ -152,7 +152,11 @@ func (c *Config) parseS3Config(v *viper.Viper) {
c.storageEngine.S3.SecretKey = sub.GetString("secret_key")
}
- if sub.IsSet("region") {
+ if sub.IsSet("region_name") {
+ c.storageEngine.S3.Region = sub.GetString("region_name")
+ }
+
+ if c.storageEngine.S3.Region == "" && sub.IsSet("region") {
c.storageEngine.S3.Region = sub.GetString("region")
}
diff --git a/internal/server/config/storage_engine_config_test.go b/internal/server/config/storage_engine_config_test.go
new file mode 100644
index 0000000000..03c51cf6eb
--- /dev/null
+++ b/internal/server/config/storage_engine_config_test.go
@@ -0,0 +1,46 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+ "testing"
+
+ "github.com/spf13/viper"
+)
+
+func TestParseS3ConfigRegionAliases(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ values map[string]interface{}
+ want string
+ }{
+ {name: "region name", values: map[string]interface{}{"region_name": "eu-west-1"}, want: "eu-west-1"},
+ {name: "region fallback", values: map[string]interface{}{"region": "ap-southeast-2"}, want: "ap-southeast-2"},
+ {name: "region name precedence", values: map[string]interface{}{"region_name": "eu-central-1", "region": "us-east-1"}, want: "eu-central-1"},
+ {name: "empty region name", values: map[string]interface{}{"region_name": "", "region": "us-west-2"}, want: "us-west-2"},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ v := viper.New()
+ v.Set("s3", test.values)
+ cfg := &Config{}
+ cfg.parseS3Config(v)
+ if got := cfg.GetS3Config().Region; got != test.want {
+ t.Fatalf("S3 region = %q, want %q", got, test.want)
+ }
+ })
+ }
+}
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
index 96798c0f91..be3522adfc 100644
--- a/internal/storage/s3.go
+++ b/internal/storage/s3.go
@@ -120,6 +120,18 @@ func (s *S3Storage) resolveBucketAndPath(bucket, fnm string) (string, string) {
return actualBucket, actualPath
}
+func (s *S3Storage) createBucket(ctx context.Context, bucket string) error {
+ input := &s3.CreateBucketInput{Bucket: aws.String(bucket)}
+ region := s.client.Options().Region
+ if region != "" && region != "us-east-1" && region != "auto" {
+ input.CreateBucketConfiguration = &types.CreateBucketConfiguration{
+ LocationConstraint: types.BucketLocationConstraint(region),
+ }
+ }
+ _, err := s.client.CreateBucket(ctx, input)
+ return err
+}
+
func (s *S3Storage) Type() string { return "s3" }
// Health checks S3 service availability
@@ -137,9 +149,7 @@ func (s *S3Storage) Health(ctx context.Context) bool {
// Ensure bucket exists
if !s.BucketExists(ctx, bucket) {
- _, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{
- Bucket: aws.String(bucket),
- })
+ err := s.createBucket(ctx, bucket)
if err != nil {
common.Error("Failed to create bucket for health check", err, zap.String("bucket", bucket), zap.Error(err))
return false
@@ -169,9 +179,7 @@ func (s *S3Storage) Put(ctx context.Context, bucket, fnm string, binary []byte,
for i := 0; i < 2; i++ {
// Ensure bucket exists
if !s.BucketExists(ctx, bucket) {
- _, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{
- Bucket: aws.String(bucket),
- })
+ err := s.createBucket(ctx, bucket)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr
diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go
index a4cdae9caa..f643791094 100644
--- a/internal/storage/s3_test.go
+++ b/internal/storage/s3_test.go
@@ -16,9 +16,9 @@ type s3RoundTripper func(*http.Request) (*http.Response, error)
func (f s3RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
-func newS3TestStorage(handler s3RoundTripper) *S3Storage {
+func newS3TestStorage(region string, handler s3RoundTripper) *S3Storage {
client := s3.NewFromConfig(aws.Config{
- Region: "us-east-1",
+ Region: region,
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),
HTTPClient: &http.Client{Transport: handler},
}, func(options *s3.Options) {
@@ -33,9 +33,79 @@ func s3Response(status int, body string) *http.Response {
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}
}
+func TestS3StoragePutCreatesBucketInResolvedRegion(t *testing.T) {
+ for _, test := range []struct {
+ region string
+ wantConstraint bool
+ }{
+ {region: "eu-west-1", wantConstraint: true},
+ {region: "us-east-1", wantConstraint: false},
+ {region: "auto", wantConstraint: false},
+ } {
+ t.Run(test.region, func(t *testing.T) {
+ var createBody string
+ storage := newS3TestStorage(test.region, func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead && req.URL.Path == "/kb01":
+ return s3Response(http.StatusNotFound, `<Error><Code>NotFound</Code></Error>`), nil
+ case req.Method == http.MethodPut && req.URL.Path == "/kb01":
+ if req.Body != nil {
+ body, err := io.ReadAll(req.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ createBody = string(body)
+ }
+ return s3Response(http.StatusOK, ""), nil
+ case req.Method == http.MethodPut && req.URL.Path == "/kb01/document":
+ return s3Response(http.StatusOK, ""), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+
+ if err := storage.Put(t.Context(), "kb01", "document", []byte("content")); err != nil {
+ t.Fatal(err)
+ }
+ hasConstraint := strings.Contains(createBody, "<LocationConstraint>"+test.region+"</LocationConstraint>")
+ if hasConstraint != test.wantConstraint {
+ t.Fatalf("CreateBucket body = %q, want location constraint: %t", createBody, test.wantConstraint)
+ }
+ })
+ }
+}
+
+func TestS3StorageHealthCreatesBucketInResolvedRegion(t *testing.T) {
+ var createBody string
+ storage := newS3TestStorage("ap-southeast-2", func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead && req.URL.Path == "/health-check-bucket":
+ return s3Response(http.StatusNotFound, `<Error><Code>NotFound</Code></Error>`), nil
+ case req.Method == http.MethodPut && req.URL.Path == "/health-check-bucket":
+ body, err := io.ReadAll(req.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ createBody = string(body)
+ return s3Response(http.StatusOK, ""), nil
+ case req.Method == http.MethodPut && req.URL.Path == "/health-check-bucket/txtxtxtxt1":
+ return s3Response(http.StatusOK, ""), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+
+ if !storage.Health(t.Context()) {
+ t.Fatal("Health() = false, want true")
+ }
+ if !strings.Contains(createBody, "<LocationConstraint>ap-southeast-2</LocationConstraint>") {
+ t.Fatalf("CreateBucket body = %q, want ap-southeast-2 location constraint", createBody)
+ }
+}
+
func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {
bucketDeleted := false
- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
switch {
case req.Method == http.MethodHead:
return s3Response(http.StatusOK, ""), nil
@@ -61,7 +131,7 @@ func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {
func TestS3StorageRemoveBucketSingleBucketMode(t *testing.T) {
var deleteBody string
- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
switch {
case req.Method == http.MethodHead:
if req.URL.Path != "/physical" {
@@ -95,7 +165,7 @@ func TestS3StorageRemoveBucketSingleBucketMode(t *testing.T) {
func TestS3StorageRemoveBucketDeletesVersionsAndMarkers(t *testing.T) {
var deleteBody string
- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
switch {
case req.Method == http.MethodHead:
return s3Response(http.StatusOK, ""), nil
@@ -123,7 +193,7 @@ func TestS3StorageRemoveBucketDeletesVersionsAndMarkers(t *testing.T) {
func TestS3StorageRemoveBucketDeletesNullVersion(t *testing.T) {
var deleteBody string
- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
switch {
case req.Method == http.MethodHead:
return s3Response(http.StatusOK, ""), nil
@@ -149,7 +219,7 @@ func TestS3StorageRemoveBucketDeletesNullVersion(t *testing.T) {
func TestS3StorageRemoveBucketVersionPagination(t *testing.T) {
versionCalls := 0
- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
switch {
case req.Method == http.MethodHead:
return s3Response(http.StatusOK, ""), nil
@@ -181,7 +251,7 @@ func TestS3StorageRemoveBucketVersionPagination(t *testing.T) {
func TestS3StorageRemoveBucketFailures(t *testing.T) {
t.Run("batch limit", func(t *testing.T) {
requests := 0
- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
if !req.URL.Query().Has("delete") {
t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
}
@@ -209,7 +279,7 @@ func TestS3StorageRemoveBucketFailures(t *testing.T) {
} {
t.Run(test.name, func(t *testing.T) {
bucketDeleted := false
- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
switch {
case req.Method == http.MethodHead:
return s3Response(http.StatusOK, ""), nil
@@ -249,7 +319,7 @@ func TestS3StorageRemoveBucketHeadBucketErrors(t *testing.T) {
} {
t.Run(test.name, func(t *testing.T) {
destructiveRequest := false
- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
if req.Method == http.MethodHead {
return s3Response(test.status, `<Error><Code>`+test.code+`</Code></Error>`), nil
}
diff --git a/rag/utils/s3_conn.py b/rag/utils/s3_conn.py
index d68525a464..f284594026 100644
--- a/rag/utils/s3_conn.py
+++ b/rag/utils/s3_conn.py
@@ -138,11 +138,22 @@ class RAGFlowS3:
@use_prefix_path
@use_default_bucket
def put(self, bucket, fnm, binary, *args, **kwargs):
+ """Upload bytes, creating a missing bucket in the client's region.
+
+ Omit the location constraint for the default AWS region and services
+ that use ``auto`` to select their own location.
+ """
logging.debug(f"bucket name {bucket}; filename :{fnm}:")
for _ in range(1):
try:
if not self.bucket_exists(bucket):
- self.conn[0].create_bucket(Bucket=bucket)
+ bucket_config = {"Bucket": bucket}
+ region = self.conn[0].meta.region_name
+ # AWS requires a location constraint outside us-east-1.
+ # "auto" lets globally distributed S3 services choose it.
+ if region and region not in ("us-east-1", "auto"):
+ bucket_config["CreateBucketConfiguration"] = {"LocationConstraint": region}
+ self.conn[0].create_bucket(**bucket_config)
logging.info(f"create bucket {bucket} ********")
r = self.conn[0].upload_fileobj(BytesIO(binary), bucket, fnm)
diff --git a/test/unit_test/rag/utils/test_s3_bucket_region.py b/test/unit_test/rag/utils/test_s3_bucket_region.py
new file mode 100644
index 0000000000..367706ac00
--- /dev/null
+++ b/test/unit_test/rag/utils/test_s3_bucket_region.py
@@ -0,0 +1,117 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import importlib
+from io import BytesIO
+from unittest.mock import Mock
+
+import pytest
+from botocore.response import StreamingBody
+from botocore.stub import ANY, Stubber
+
+import common.settings # noqa: F401 -- initialize the settings/connector import cycle.
+from rag.utils import s3_conn
+
+pytestmark = pytest.mark.p2
+
+
+def make_storage(monkeypatch, tmp_path, config):
+ """Create a fresh S3 singleton with isolated AWS configuration per test."""
+ module = importlib.reload(s3_conn)
+ # Explicit test credentials and empty config files keep the SDK independent
+ # of the developer's AWS profiles. Stubber rejects any unexpected request.
+ monkeypatch.setenv("AWS_CONFIG_FILE", str(tmp_path / "aws-config"))
+ monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", str(tmp_path / "aws-credentials"))
+ monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-central-1")
+ monkeypatch.setattr(
+ module.settings,
+ "S3",
+ {"access_key": "test-access", "secret_key": "test-secret", "endpoint_url": "https://s3.test", **config},
+ )
+ storage = module.RAGFlowS3()
+ # A failed base request must not reconnect or wait before we inspect the
+ # stub queue. The successful write path still uses the real SDK uploader.
+ monkeypatch.setattr(storage, "__open__", Mock())
+ monkeypatch.setattr(module.time, "sleep", Mock())
+ return storage
+
+
+ "config,region,location",
+ [
+ ({"region": "eu-west-1"}, "eu-west-1", "eu-west-1"),
+ ({"region_name": "ap-southeast-2"}, "ap-southeast-2", "ap-southeast-2"),
+ ({"region": "us-east-1", "region_name": "eu-west-1"}, "eu-west-1", "eu-west-1"),
+ ({}, "eu-central-1", "eu-central-1"),
+ ({"region_name": ""}, "eu-central-1", "eu-central-1"),
+ ({"region": "us-east-1"}, "us-east-1", None),
+ ({"region": "auto"}, "auto", None),
+ ],
+)
[email protected]("shared_bucket", [False, True])
+def test_put_creates_bucket_in_resolved_region_then_uploads(monkeypatch, tmp_path, config, region, location, shared_bucket):
+ """Verify bucket location, object routing and payload with the real SDK."""
+ if shared_bucket:
+ config = {**config, "bucket": "physical-bucket", "prefix_path": "documents"}
+ storage = make_storage(monkeypatch, tmp_path, config)
+ client = storage.conn[0]
+ assert client.meta.region_name == region
+ bucket = "physical-bucket" if shared_bucket else "knowledge-base"
+ key = "documents/knowledge-base/file.txt" if shared_bucket else "file.txt"
+ payload = b"document contents"
+ uploaded = []
+
+ def capture_upload(params, **_kwargs):
+ """Record the upload bytes and rewind the body for the SDK request."""
+ uploaded.append(params["Body"].read())
+ params["Body"].seek(0)
+
+ client.meta.events.register("before-parameter-build.s3.PutObject", capture_upload)
+ create_params = {"Bucket": bucket}
+ if location:
+ create_params["CreateBucketConfiguration"] = {"LocationConstraint": location}
+ try:
+ with Stubber(client) as stub:
+ stub.add_client_error("head_bucket", service_error_code="404", http_status_code=404, expected_params={"Bucket": bucket})
+ stub.add_response("create_bucket", {}, create_params)
+ stub.add_response("put_object", {}, {"Bucket": bucket, "Key": key, "Body": ANY, "ChecksumAlgorithm": ANY})
+ stub.add_response("get_object", {"Body": StreamingBody(BytesIO(payload), len(payload))}, {"Bucket": bucket, "Key": key})
+
+ storage.put("knowledge-base", "file.txt", payload)
+
+ assert uploaded == [payload]
+ assert storage.get("knowledge-base", "file.txt") == payload
+ stub.assert_no_pending_responses()
+ storage.__open__.assert_not_called()
+ finally:
+ client.close()
+
+
+def test_put_to_existing_regional_bucket_skips_creation(monkeypatch, tmp_path):
+ """Verify an existing regional bucket accepts uploads without creation."""
+ storage = make_storage(monkeypatch, tmp_path, {"region": "eu-west-1"})
+ client = storage.conn[0]
+ try:
+ with Stubber(client) as stub:
+ stub.add_response("head_bucket", {}, {"Bucket": "knowledge-base"})
+ stub.add_response("put_object", {}, {"Bucket": "knowledge-base", "Key": "file.txt", "Body": ANY, "ChecksumAlgorithm": ANY})
+
+ storage.put("knowledge-base", "file.txt", b"document contents")
+
+ stub.assert_no_pending_responses()
+ storage.__open__.assert_not_called()
+ finally:
+ client.close()