Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat: support host style in archive s3 client (#43066)

Yunlu Wen4f8bf32b72e0
diff --git a/api/.env.example b/api/.env.example
index 7d3ff606049..a85466da2e8 100644
--- a/api/.env.example
+++ b/api/.env.example
@@ -156,6 +156,7 @@ ARCHIVE_STORAGE_EXPORT_BUCKET=
ARCHIVE_STORAGE_ACCESS_KEY=
ARCHIVE_STORAGE_SECRET_KEY=
ARCHIVE_STORAGE_REGION=auto
+ARCHIVE_STORAGE_ADDRESS_STYLE=path
# Azure Blob Storage configuration
AZURE_BLOB_ACCOUNT_NAME=your-account-name
diff --git a/api/configs/extra/archive_config.py b/api/configs/extra/archive_config.py
index a85628fa612..23b467253cc 100644
--- a/api/configs/extra/archive_config.py
+++ b/api/configs/extra/archive_config.py
@@ -1,3 +1,5 @@
+from typing import Literal
+
from pydantic import Field
from pydantic_settings import BaseSettings
@@ -41,3 +43,8 @@ class ArchiveStorageConfig(BaseSettings):
description="Region for storage (use 'auto' if the provider supports it)",
default="auto",
)
+
+ ARCHIVE_STORAGE_ADDRESS_STYLE: Literal["path", "virtual", "auto"] = Field(
+ description="S3 addressing style for archive storage: 'path' (default), 'virtual' (virtual-hosted), or 'auto'",
+ default="path",
+ )
diff --git a/api/libs/archive_storage.py b/api/libs/archive_storage.py
index 5df13830b35..1e6d5e7169b 100644
--- a/api/libs/archive_storage.py
+++ b/api/libs/archive_storage.py
@@ -77,7 +77,7 @@ class ArchiveStorage:
aws_secret_access_key=dify_config.ARCHIVE_STORAGE_SECRET_KEY,
region_name=dify_config.ARCHIVE_STORAGE_REGION,
config=Config(
- s3={"addressing_style": "path"},
+ s3={"addressing_style": dify_config.ARCHIVE_STORAGE_ADDRESS_STYLE},
max_pool_connections=64,
),
)
diff --git a/api/tests/unit_tests/libs/test_archive_storage.py b/api/tests/unit_tests/libs/test_archive_storage.py
index 4c0d51a1907..b49a762992c 100644
--- a/api/tests/unit_tests/libs/test_archive_storage.py
+++ b/api/tests/unit_tests/libs/test_archive_storage.py
@@ -4,8 +4,10 @@ from datetime import datetime
from unittest.mock import ANY, MagicMock
import pytest
+from botocore.awsrequest import AWSResponse
from botocore.exceptions import ClientError, EndpointConnectionError
+from configs.extra.archive_config import ArchiveStorageConfig
from libs import archive_storage as storage_module
from libs.archive_storage import (
ArchiveStorage,
@@ -117,6 +119,118 @@ def test_init_sets_client(monkeypatch: pytest.MonkeyPatch):
assert storage.bucket == BUCKET_NAME
+def test_init_default_address_style_is_path(monkeypatch: pytest.MonkeyPatch):
+ # _configure_storage does not set ARCHIVE_STORAGE_ADDRESS_STYLE, so the
+ # pydantic field default is exercised.
+ _configure_storage(monkeypatch)
+ _, boto_client = _mock_client(monkeypatch)
+
+ ArchiveStorage(bucket=BUCKET_NAME)
+
+ assert ArchiveStorageConfig.model_fields["ARCHIVE_STORAGE_ADDRESS_STYLE"].default == "path"
+ config = boto_client.call_args.kwargs["config"]
+ assert config.s3 == {"addressing_style": "path"}
+
+
[email protected]("address_style", ["virtual", "auto"])
+def test_init_address_style_override(monkeypatch: pytest.MonkeyPatch, address_style: str):
+ _configure_storage(monkeypatch, ARCHIVE_STORAGE_ADDRESS_STYLE=address_style)
+ _, boto_client = _mock_client(monkeypatch)
+
+ ArchiveStorage(bucket=BUCKET_NAME)
+
+ config = boto_client.call_args.kwargs["config"]
+ assert config.s3 == {"addressing_style": address_style}
+
+
+class _EmptyRawBody:
+ def stream(self, **_kwargs):
+ yield b""
+
+
+def _record_real_client_requests(monkeypatch: pytest.MonkeyPatch) -> list[str]:
+ urls: list[str] = []
+ real_client = storage_module.boto3.client
+
+ def make_client(*args, **kwargs):
+ client = real_client(*args, **kwargs)
+
+ def handler(request, **_kwargs):
+ urls.append(request.url)
+ return AWSResponse(request.url, 200, {}, _EmptyRawBody())
+
+ client.meta.events.register("before-send.s3", handler)
+ return client
+
+ monkeypatch.setattr(storage_module.boto3, "client", make_client)
+ return urls
+
+
+ ("address_style", "expected_bucket_url", "expected_object_url"),
+ [
+ (
+ None,
+ "https://account123.r2.example.com/archive-bucket",
+ "https://account123.r2.example.com/archive-bucket/some/key.json",
+ ),
+ (
+ "path",
+ "https://account123.r2.example.com/archive-bucket",
+ "https://account123.r2.example.com/archive-bucket/some/key.json",
+ ),
+ (
+ "virtual",
+ "https://archive-bucket.account123.r2.example.com/",
+ "https://archive-bucket.account123.r2.example.com/some/key.json",
+ ),
+ ],
+)
+def test_real_client_request_urls_follow_address_style(
+ monkeypatch: pytest.MonkeyPatch,
+ address_style: str | None,
+ expected_bucket_url: str,
+ expected_object_url: str,
+):
+ overrides: dict[str, object] = {"ARCHIVE_STORAGE_ENDPOINT": "https://account123.r2.example.com"}
+ if address_style is not None:
+ overrides["ARCHIVE_STORAGE_ADDRESS_STYLE"] = address_style
+ _configure_storage(monkeypatch, **overrides)
+ urls = _record_real_client_requests(monkeypatch)
+
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+ storage.object_exists("some/key.json")
+
+ assert len(urls) == 2
+ assert urls[0].startswith(expected_bucket_url)
+ assert urls[1].startswith(expected_object_url)
+
+
+ ("address_style", "expected_prefix"),
+ [
+ (None, "https://account123.r2.example.com/archive-bucket/some/key.json?"),
+ ("path", "https://account123.r2.example.com/archive-bucket/some/key.json?"),
+ ("virtual", "https://archive-bucket.account123.r2.example.com/some/key.json?"),
+ ],
+)
+def test_real_client_presigned_url_follows_address_style(
+ monkeypatch: pytest.MonkeyPatch,
+ address_style: str | None,
+ expected_prefix: str,
+):
+ overrides: dict[str, object] = {"ARCHIVE_STORAGE_ENDPOINT": "https://account123.r2.example.com"}
+ if address_style is not None:
+ overrides["ARCHIVE_STORAGE_ADDRESS_STYLE"] = address_style
+ _configure_storage(monkeypatch, **overrides)
+ _record_real_client_requests(monkeypatch)
+
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+ url = storage.generate_presigned_url("some/key.json", expires_in=123)
+
+ assert url.startswith(expected_prefix)
+
+
def test_put_object_returns_checksum(monkeypatch: pytest.MonkeyPatch):
_configure_storage(monkeypatch)
client, _ = _mock_client(monkeypatch)
diff --git a/docker/envs/core-services/shared.env.example b/docker/envs/core-services/shared.env.example
index 2615732b34c..8e07ebef499 100644
--- a/docker/envs/core-services/shared.env.example
+++ b/docker/envs/core-services/shared.env.example
@@ -349,6 +349,7 @@ ARCHIVE_STORAGE_ENDPOINT=
ARCHIVE_STORAGE_ARCHIVE_BUCKET=
ARCHIVE_STORAGE_EXPORT_BUCKET=
ARCHIVE_STORAGE_REGION=auto
+ARCHIVE_STORAGE_ADDRESS_STYLE=path
AZURE_BLOB_ACCOUNT_NAME=difyai
AZURE_BLOB_CONTAINER_NAME=difyai-container
AZURE_KEYVAULT_VAULT_URL=