Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
feat: support host style in archive s3 client (#43066)
Yunlu Wen4f8bf32b72e0
diff --git a/api/.env.example b/api/.env.exampleindex 7d3ff606049..a85466da2e8 100644--- a/api/.env.example+++ b/api/.env.example@@ -156,6 +156,7 @@ ARCHIVE_STORAGE_EXPORT_BUCKET=ARCHIVE_STORAGE_ACCESS_KEY=ARCHIVE_STORAGE_SECRET_KEY=ARCHIVE_STORAGE_REGION=auto+ARCHIVE_STORAGE_ADDRESS_STYLE=path# Azure Blob Storage configurationAZURE_BLOB_ACCOUNT_NAME=your-account-namediff --git a/api/configs/extra/archive_config.py b/api/configs/extra/archive_config.pyindex a85628fa612..23b467253cc 100644--- a/api/configs/extra/archive_config.py+++ b/api/configs/extra/archive_config.py@@ -1,3 +1,5 @@+from typing import Literal+from pydantic import Fieldfrom pydantic_settings import BaseSettings@@ -41,3 +43,8 @@ class ArchiveStorageConfig(BaseSettings):description="Region for storage (use 'auto' if the provider supports it)",default="auto",)++ ARCHIVE_STORAGE_ADDRESS_STYLE: Literal["path", "virtual", "auto"] = Field(+ description="S3 addressing style for archive storage: 'path' (default), 'virtual' (virtual-hosted), or 'auto'",+ default="path",+ )diff --git a/api/libs/archive_storage.py b/api/libs/archive_storage.pyindex 5df13830b35..1e6d5e7169b 100644--- a/api/libs/archive_storage.py+++ b/api/libs/archive_storage.py@@ -77,7 +77,7 @@ class ArchiveStorage:aws_secret_access_key=dify_config.ARCHIVE_STORAGE_SECRET_KEY,region_name=dify_config.ARCHIVE_STORAGE_REGION,config=Config(- s3={"addressing_style": "path"},+ s3={"addressing_style": dify_config.ARCHIVE_STORAGE_ADDRESS_STYLE},max_pool_connections=64,),)diff --git a/api/tests/unit_tests/libs/test_archive_storage.py b/api/tests/unit_tests/libs/test_archive_storage.pyindex 4c0d51a1907..b49a762992c 100644--- a/api/tests/unit_tests/libs/test_archive_storage.py+++ b/api/tests/unit_tests/libs/test_archive_storage.py@@ -4,8 +4,10 @@ from datetime import datetimefrom unittest.mock import ANY, MagicMockimport pytest+from botocore.awsrequest import AWSResponsefrom botocore.exceptions import ClientError, EndpointConnectionError+from configs.extra.archive_config import ArchiveStorageConfigfrom libs import archive_storage as storage_modulefrom libs.archive_storage import (ArchiveStorage,@@ -117,6 +119,118 @@ def test_init_sets_client(monkeypatch: pytest.MonkeyPatch):assert storage.bucket == BUCKET_NAME+def test_init_default_address_style_is_path(monkeypatch: pytest.MonkeyPatch):+ # _configure_storage does not set ARCHIVE_STORAGE_ADDRESS_STYLE, so the+ # pydantic field default is exercised.+ _configure_storage(monkeypatch)+ _, boto_client = _mock_client(monkeypatch)++ ArchiveStorage(bucket=BUCKET_NAME)++ assert ArchiveStorageConfig.model_fields["ARCHIVE_STORAGE_ADDRESS_STYLE"].default == "path"+ config = boto_client.call_args.kwargs["config"]+ assert config.s3 == {"addressing_style": "path"}++[email protected]("address_style", ["virtual", "auto"])+def test_init_address_style_override(monkeypatch: pytest.MonkeyPatch, address_style: str):+ _configure_storage(monkeypatch, ARCHIVE_STORAGE_ADDRESS_STYLE=address_style)+ _, boto_client = _mock_client(monkeypatch)++ ArchiveStorage(bucket=BUCKET_NAME)++ config = boto_client.call_args.kwargs["config"]+ assert config.s3 == {"addressing_style": address_style}+++class _EmptyRawBody:+ def stream(self, **_kwargs):+ yield b""+++def _record_real_client_requests(monkeypatch: pytest.MonkeyPatch) -> list[str]:+ urls: list[str] = []+ real_client = storage_module.boto3.client++ def make_client(*args, **kwargs):+ client = real_client(*args, **kwargs)++ def handler(request, **_kwargs):+ urls.append(request.url)+ return AWSResponse(request.url, 200, {}, _EmptyRawBody())++ client.meta.events.register("before-send.s3", handler)+ return client++ monkeypatch.setattr(storage_module.boto3, "client", make_client)+ return urls+++ ("address_style", "expected_bucket_url", "expected_object_url"),+ [+ (+ None,+ "https://account123.r2.example.com/archive-bucket",+ "https://account123.r2.example.com/archive-bucket/some/key.json",+ ),+ (+ "path",+ "https://account123.r2.example.com/archive-bucket",+ "https://account123.r2.example.com/archive-bucket/some/key.json",+ ),+ (+ "virtual",+ "https://archive-bucket.account123.r2.example.com/",+ "https://archive-bucket.account123.r2.example.com/some/key.json",+ ),+ ],+)+def test_real_client_request_urls_follow_address_style(+ monkeypatch: pytest.MonkeyPatch,+ address_style: str | None,+ expected_bucket_url: str,+ expected_object_url: str,+):+ overrides: dict[str, object] = {"ARCHIVE_STORAGE_ENDPOINT": "https://account123.r2.example.com"}+ if address_style is not None:+ overrides["ARCHIVE_STORAGE_ADDRESS_STYLE"] = address_style+ _configure_storage(monkeypatch, **overrides)+ urls = _record_real_client_requests(monkeypatch)++ storage = ArchiveStorage(bucket=BUCKET_NAME)+ storage.object_exists("some/key.json")++ assert len(urls) == 2+ assert urls[0].startswith(expected_bucket_url)+ assert urls[1].startswith(expected_object_url)+++ ("address_style", "expected_prefix"),+ [+ (None, "https://account123.r2.example.com/archive-bucket/some/key.json?"),+ ("path", "https://account123.r2.example.com/archive-bucket/some/key.json?"),+ ("virtual", "https://archive-bucket.account123.r2.example.com/some/key.json?"),+ ],+)+def test_real_client_presigned_url_follows_address_style(+ monkeypatch: pytest.MonkeyPatch,+ address_style: str | None,+ expected_prefix: str,+):+ overrides: dict[str, object] = {"ARCHIVE_STORAGE_ENDPOINT": "https://account123.r2.example.com"}+ if address_style is not None:+ overrides["ARCHIVE_STORAGE_ADDRESS_STYLE"] = address_style+ _configure_storage(monkeypatch, **overrides)+ _record_real_client_requests(monkeypatch)++ storage = ArchiveStorage(bucket=BUCKET_NAME)+ url = storage.generate_presigned_url("some/key.json", expires_in=123)++ assert url.startswith(expected_prefix)++def test_put_object_returns_checksum(monkeypatch: pytest.MonkeyPatch):_configure_storage(monkeypatch)client, _ = _mock_client(monkeypatch)diff --git a/docker/envs/core-services/shared.env.example b/docker/envs/core-services/shared.env.exampleindex 2615732b34c..8e07ebef499 100644--- a/docker/envs/core-services/shared.env.example+++ b/docker/envs/core-services/shared.env.example@@ -349,6 +349,7 @@ ARCHIVE_STORAGE_ENDPOINT=ARCHIVE_STORAGE_ARCHIVE_BUCKET=ARCHIVE_STORAGE_EXPORT_BUCKET=ARCHIVE_STORAGE_REGION=auto+ARCHIVE_STORAGE_ADDRESS_STYLE=pathAZURE_BLOB_ACCOUNT_NAME=difyaiAZURE_BLOB_CONTAINER_NAME=difyai-containerAZURE_KEYVAULT_VAULT_URL=