RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(storage): honor client region in Python S3 bucket creation (#19291)
Wenchao An9d6f76d2173a
diff --git a/internal/server/config/storage_engine_config.go b/internal/server/config/storage_engine_config.goindex 2d3191d5a6..21b514e3b5 100644--- a/internal/server/config/storage_engine_config.go+++ b/internal/server/config/storage_engine_config.go@@ -152,7 +152,11 @@ func (c *Config) parseS3Config(v *viper.Viper) {c.storageEngine.S3.SecretKey = sub.GetString("secret_key")}- if sub.IsSet("region") {+ if sub.IsSet("region_name") {+ c.storageEngine.S3.Region = sub.GetString("region_name")+ }++ if c.storageEngine.S3.Region == "" && sub.IsSet("region") {c.storageEngine.S3.Region = sub.GetString("region")}diff --git a/internal/server/config/storage_engine_config_test.go b/internal/server/config/storage_engine_config_test.gonew file mode 100644index 0000000000..03c51cf6eb--- /dev/null+++ b/internal/server/config/storage_engine_config_test.go@@ -0,0 +1,46 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package config++import (+ "testing"++ "github.com/spf13/viper"+)++func TestParseS3ConfigRegionAliases(t *testing.T) {+ for _, test := range []struct {+ name string+ values map[string]interface{}+ want string+ }{+ {name: "region name", values: map[string]interface{}{"region_name": "eu-west-1"}, want: "eu-west-1"},+ {name: "region fallback", values: map[string]interface{}{"region": "ap-southeast-2"}, want: "ap-southeast-2"},+ {name: "region name precedence", values: map[string]interface{}{"region_name": "eu-central-1", "region": "us-east-1"}, want: "eu-central-1"},+ {name: "empty region name", values: map[string]interface{}{"region_name": "", "region": "us-west-2"}, want: "us-west-2"},+ } {+ t.Run(test.name, func(t *testing.T) {+ v := viper.New()+ v.Set("s3", test.values)+ cfg := &Config{}+ cfg.parseS3Config(v)+ if got := cfg.GetS3Config().Region; got != test.want {+ t.Fatalf("S3 region = %q, want %q", got, test.want)+ }+ })+ }+}diff --git a/internal/storage/s3.go b/internal/storage/s3.goindex 96798c0f91..be3522adfc 100644--- a/internal/storage/s3.go+++ b/internal/storage/s3.go@@ -120,6 +120,18 @@ func (s *S3Storage) resolveBucketAndPath(bucket, fnm string) (string, string) {return actualBucket, actualPath}+func (s *S3Storage) createBucket(ctx context.Context, bucket string) error {+ input := &s3.CreateBucketInput{Bucket: aws.String(bucket)}+ region := s.client.Options().Region+ if region != "" && region != "us-east-1" && region != "auto" {+ input.CreateBucketConfiguration = &types.CreateBucketConfiguration{+ LocationConstraint: types.BucketLocationConstraint(region),+ }+ }+ _, err := s.client.CreateBucket(ctx, input)+ return err+}+func (s *S3Storage) Type() string { return "s3" }// Health checks S3 service availability@@ -137,9 +149,7 @@ func (s *S3Storage) Health(ctx context.Context) bool {// Ensure bucket existsif !s.BucketExists(ctx, bucket) {- _, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{- Bucket: aws.String(bucket),- })+ err := s.createBucket(ctx, bucket)if err != nil {common.Error("Failed to create bucket for health check", err, zap.String("bucket", bucket), zap.Error(err))return false@@ -169,9 +179,7 @@ func (s *S3Storage) Put(ctx context.Context, bucket, fnm string, binary []byte,for i := 0; i < 2; i++ {// Ensure bucket existsif !s.BucketExists(ctx, bucket) {- _, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{- Bucket: aws.String(bucket),- })+ err := s.createBucket(ctx, bucket)if err != nil {if ctxErr := ctx.Err(); ctxErr != nil {return ctxErrdiff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.goindex a4cdae9caa..f643791094 100644--- a/internal/storage/s3_test.go+++ b/internal/storage/s3_test.go@@ -16,9 +16,9 @@ type s3RoundTripper func(*http.Request) (*http.Response, error)func (f s3RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }-func newS3TestStorage(handler s3RoundTripper) *S3Storage {+func newS3TestStorage(region string, handler s3RoundTripper) *S3Storage {client := s3.NewFromConfig(aws.Config{- Region: "us-east-1",+ Region: region,Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),HTTPClient: &http.Client{Transport: handler},}, func(options *s3.Options) {@@ -33,9 +33,79 @@ func s3Response(status int, body string) *http.Response {return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}}+func TestS3StoragePutCreatesBucketInResolvedRegion(t *testing.T) {+ for _, test := range []struct {+ region string+ wantConstraint bool+ }{+ {region: "eu-west-1", wantConstraint: true},+ {region: "us-east-1", wantConstraint: false},+ {region: "auto", wantConstraint: false},+ } {+ t.Run(test.region, func(t *testing.T) {+ var createBody string+ storage := newS3TestStorage(test.region, func(req *http.Request) (*http.Response, error) {+ switch {+ case req.Method == http.MethodHead && req.URL.Path == "/kb01":+ return s3Response(http.StatusNotFound, `<Error><Code>NotFound</Code></Error>`), nil+ case req.Method == http.MethodPut && req.URL.Path == "/kb01":+ if req.Body != nil {+ body, err := io.ReadAll(req.Body)+ if err != nil {+ t.Fatal(err)+ }+ createBody = string(body)+ }+ return s3Response(http.StatusOK, ""), nil+ case req.Method == http.MethodPut && req.URL.Path == "/kb01/document":+ return s3Response(http.StatusOK, ""), nil+ }+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)+ return nil, nil+ })++ if err := storage.Put(t.Context(), "kb01", "document", []byte("content")); err != nil {+ t.Fatal(err)+ }+ hasConstraint := strings.Contains(createBody, "<LocationConstraint>"+test.region+"</LocationConstraint>")+ if hasConstraint != test.wantConstraint {+ t.Fatalf("CreateBucket body = %q, want location constraint: %t", createBody, test.wantConstraint)+ }+ })+ }+}++func TestS3StorageHealthCreatesBucketInResolvedRegion(t *testing.T) {+ var createBody string+ storage := newS3TestStorage("ap-southeast-2", func(req *http.Request) (*http.Response, error) {+ switch {+ case req.Method == http.MethodHead && req.URL.Path == "/health-check-bucket":+ return s3Response(http.StatusNotFound, `<Error><Code>NotFound</Code></Error>`), nil+ case req.Method == http.MethodPut && req.URL.Path == "/health-check-bucket":+ body, err := io.ReadAll(req.Body)+ if err != nil {+ t.Fatal(err)+ }+ createBody = string(body)+ return s3Response(http.StatusOK, ""), nil+ case req.Method == http.MethodPut && req.URL.Path == "/health-check-bucket/txtxtxtxt1":+ return s3Response(http.StatusOK, ""), nil+ }+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)+ return nil, nil+ })++ if !storage.Health(t.Context()) {+ t.Fatal("Health() = false, want true")+ }+ if !strings.Contains(createBody, "<LocationConstraint>ap-southeast-2</LocationConstraint>") {+ t.Fatalf("CreateBucket body = %q, want ap-southeast-2 location constraint", createBody)+ }+}+func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {bucketDeleted := false- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {switch {case req.Method == http.MethodHead:return s3Response(http.StatusOK, ""), nil@@ -61,7 +131,7 @@ func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {func TestS3StorageRemoveBucketSingleBucketMode(t *testing.T) {var deleteBody string- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {switch {case req.Method == http.MethodHead:if req.URL.Path != "/physical" {@@ -95,7 +165,7 @@ func TestS3StorageRemoveBucketSingleBucketMode(t *testing.T) {func TestS3StorageRemoveBucketDeletesVersionsAndMarkers(t *testing.T) {var deleteBody string- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {switch {case req.Method == http.MethodHead:return s3Response(http.StatusOK, ""), nil@@ -123,7 +193,7 @@ func TestS3StorageRemoveBucketDeletesVersionsAndMarkers(t *testing.T) {func TestS3StorageRemoveBucketDeletesNullVersion(t *testing.T) {var deleteBody string- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {switch {case req.Method == http.MethodHead:return s3Response(http.StatusOK, ""), nil@@ -149,7 +219,7 @@ func TestS3StorageRemoveBucketDeletesNullVersion(t *testing.T) {func TestS3StorageRemoveBucketVersionPagination(t *testing.T) {versionCalls := 0- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {switch {case req.Method == http.MethodHead:return s3Response(http.StatusOK, ""), nil@@ -181,7 +251,7 @@ func TestS3StorageRemoveBucketVersionPagination(t *testing.T) {func TestS3StorageRemoveBucketFailures(t *testing.T) {t.Run("batch limit", func(t *testing.T) {requests := 0- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {if !req.URL.Query().Has("delete") {t.Fatalf("unexpected request: %s %s", req.Method, req.URL)}@@ -209,7 +279,7 @@ func TestS3StorageRemoveBucketFailures(t *testing.T) {} {t.Run(test.name, func(t *testing.T) {bucketDeleted := false- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {switch {case req.Method == http.MethodHead:return s3Response(http.StatusOK, ""), nil@@ -249,7 +319,7 @@ func TestS3StorageRemoveBucketHeadBucketErrors(t *testing.T) {} {t.Run(test.name, func(t *testing.T) {destructiveRequest := false- storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {if req.Method == http.MethodHead {return s3Response(test.status, `<Error><Code>`+test.code+`</Code></Error>`), nil}diff --git a/rag/utils/s3_conn.py b/rag/utils/s3_conn.pyindex d68525a464..f284594026 100644--- a/rag/utils/s3_conn.py+++ b/rag/utils/s3_conn.py@@ -138,11 +138,22 @@ class RAGFlowS3:@use_prefix_path@use_default_bucketdef put(self, bucket, fnm, binary, *args, **kwargs):+ """Upload bytes, creating a missing bucket in the client's region.++ Omit the location constraint for the default AWS region and services+ that use ``auto`` to select their own location.+ """logging.debug(f"bucket name {bucket}; filename :{fnm}:")for _ in range(1):try:if not self.bucket_exists(bucket):- self.conn[0].create_bucket(Bucket=bucket)+ bucket_config = {"Bucket": bucket}+ region = self.conn[0].meta.region_name+ # AWS requires a location constraint outside us-east-1.+ # "auto" lets globally distributed S3 services choose it.+ if region and region not in ("us-east-1", "auto"):+ bucket_config["CreateBucketConfiguration"] = {"LocationConstraint": region}+ self.conn[0].create_bucket(**bucket_config)logging.info(f"create bucket {bucket} ********")r = self.conn[0].upload_fileobj(BytesIO(binary), bucket, fnm)diff --git a/test/unit_test/rag/utils/test_s3_bucket_region.py b/test/unit_test/rag/utils/test_s3_bucket_region.pynew file mode 100644index 0000000000..367706ac00--- /dev/null+++ b/test/unit_test/rag/utils/test_s3_bucket_region.py@@ -0,0 +1,117 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import importlib+from io import BytesIO+from unittest.mock import Mock++import pytest+from botocore.response import StreamingBody+from botocore.stub import ANY, Stubber++import common.settings # noqa: F401 -- initialize the settings/connector import cycle.+from rag.utils import s3_conn++pytestmark = pytest.mark.p2+++def make_storage(monkeypatch, tmp_path, config):+ """Create a fresh S3 singleton with isolated AWS configuration per test."""+ module = importlib.reload(s3_conn)+ # Explicit test credentials and empty config files keep the SDK independent+ # of the developer's AWS profiles. Stubber rejects any unexpected request.+ monkeypatch.setenv("AWS_CONFIG_FILE", str(tmp_path / "aws-config"))+ monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", str(tmp_path / "aws-credentials"))+ monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-central-1")+ monkeypatch.setattr(+ module.settings,+ "S3",+ {"access_key": "test-access", "secret_key": "test-secret", "endpoint_url": "https://s3.test", **config},+ )+ storage = module.RAGFlowS3()+ # A failed base request must not reconnect or wait before we inspect the+ # stub queue. The successful write path still uses the real SDK uploader.+ monkeypatch.setattr(storage, "__open__", Mock())+ monkeypatch.setattr(module.time, "sleep", Mock())+ return storage+++ "config,region,location",+ [+ ({"region": "eu-west-1"}, "eu-west-1", "eu-west-1"),+ ({"region_name": "ap-southeast-2"}, "ap-southeast-2", "ap-southeast-2"),+ ({"region": "us-east-1", "region_name": "eu-west-1"}, "eu-west-1", "eu-west-1"),+ ({}, "eu-central-1", "eu-central-1"),+ ({"region_name": ""}, "eu-central-1", "eu-central-1"),+ ({"region": "us-east-1"}, "us-east-1", None),+ ({"region": "auto"}, "auto", None),+ ],+)[email protected]("shared_bucket", [False, True])+def test_put_creates_bucket_in_resolved_region_then_uploads(monkeypatch, tmp_path, config, region, location, shared_bucket):+ """Verify bucket location, object routing and payload with the real SDK."""+ if shared_bucket:+ config = {**config, "bucket": "physical-bucket", "prefix_path": "documents"}+ storage = make_storage(monkeypatch, tmp_path, config)+ client = storage.conn[0]+ assert client.meta.region_name == region+ bucket = "physical-bucket" if shared_bucket else "knowledge-base"+ key = "documents/knowledge-base/file.txt" if shared_bucket else "file.txt"+ payload = b"document contents"+ uploaded = []++ def capture_upload(params, **_kwargs):+ """Record the upload bytes and rewind the body for the SDK request."""+ uploaded.append(params["Body"].read())+ params["Body"].seek(0)++ client.meta.events.register("before-parameter-build.s3.PutObject", capture_upload)+ create_params = {"Bucket": bucket}+ if location:+ create_params["CreateBucketConfiguration"] = {"LocationConstraint": location}+ try:+ with Stubber(client) as stub:+ stub.add_client_error("head_bucket", service_error_code="404", http_status_code=404, expected_params={"Bucket": bucket})+ stub.add_response("create_bucket", {}, create_params)+ stub.add_response("put_object", {}, {"Bucket": bucket, "Key": key, "Body": ANY, "ChecksumAlgorithm": ANY})+ stub.add_response("get_object", {"Body": StreamingBody(BytesIO(payload), len(payload))}, {"Bucket": bucket, "Key": key})++ storage.put("knowledge-base", "file.txt", payload)++ assert uploaded == [payload]+ assert storage.get("knowledge-base", "file.txt") == payload+ stub.assert_no_pending_responses()+ storage.__open__.assert_not_called()+ finally:+ client.close()+++def test_put_to_existing_regional_bucket_skips_creation(monkeypatch, tmp_path):+ """Verify an existing regional bucket accepts uploads without creation."""+ storage = make_storage(monkeypatch, tmp_path, {"region": "eu-west-1"})+ client = storage.conn[0]+ try:+ with Stubber(client) as stub:+ stub.add_response("head_bucket", {}, {"Bucket": "knowledge-base"})+ stub.add_response("put_object", {}, {"Bucket": "knowledge-base", "Key": "file.txt", "Body": ANY, "ChecksumAlgorithm": ANY})++ storage.put("knowledge-base", "file.txt", b"document contents")++ stub.assert_no_pending_responses()+ storage.__open__.assert_not_called()+ finally:+ client.close()