Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix: Retry failed requests with correct region in case s3 client rece… (#18405)

jordibosch2063e86bad6586
diff --git a/packages/shared/src/server/services/StorageService.test.ts b/packages/shared/src/server/services/StorageService.test.ts
index f0652e3d6a..19ca8bbf85 100644
--- a/packages/shared/src/server/services/StorageService.test.ts
+++ b/packages/shared/src/server/services/StorageService.test.ts
@@ -87,6 +87,58 @@ describe("S3StorageService region normalization", () => {
await expect(client.config.region()).resolves.toBe(region);
},
);
+
+ it("follows S3's region redirect when region is auto on AWS", async () => {
+ const service = StorageServiceFactory.getInstance({
+ accessKeyId: "test-access-key",
+ secretAccessKey: "test-secret-key",
+ bucketName: "test-bucket",
+ endpoint: undefined,
+ region: "auto",
+ forcePathStyle: false,
+ useAzureBlob: false,
+ useGoogleCloudStorage: false,
+ useOCIObjectStorage: false,
+ awsSse: undefined,
+ awsSseKmsKeyId: undefined,
+ });
+ const client = (service as unknown as { client: S3Client }).client;
+ const hostnames: string[] = [];
+
+ // Answer the first request like S3 does for a bucket in another region,
+ // then succeed. No network I/O happens.
+ const fakeS3 = () => async (args: { request: unknown }) => {
+ hostnames.push((args.request as { hostname: string }).hostname);
+ const redirect = hostnames.length === 1;
+ return {
+ response: {
+ statusCode: redirect ? 301 : 200,
+ headers: {
+ "content-type": "application/xml",
+ ...(redirect ? { "x-amz-bucket-region": "eu-west-1" } : {}),
+ },
+ body: Readable.from([
+ Buffer.from(
+ redirect
+ ? "<Error><Code>PermanentRedirect</Code></Error>"
+ : "<ListBucketResult></ListBucketResult>",
+ ),
+ ]),
+ },
+ };
+ };
+ client.middlewareStack.add(
+ fakeS3 as unknown as Parameters<typeof client.middlewareStack.add>[0],
+ { step: "deserialize", priority: "low", name: "fakeS3", override: true },
+ );
+
+ await service.listFiles("exports/");
+
+ expect(hostnames).toEqual([
+ "test-bucket.s3.us-east-1.amazonaws.com",
+ "test-bucket.s3.eu-west-1.amazonaws.com",
+ ]);
+ });
});
describe("resolveMediaStorageEndpoints", () => {
diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.ts
index 83402c0317..7b8f8d8e6b 100644
--- a/packages/shared/src/server/services/StorageService.ts
+++ b/packages/shared/src/server/services/StorageService.ts
@@ -774,13 +774,20 @@ class S3StorageService implements StorageService {
params.region === undefined
? undefined
: normalizeBlobStorageRegion(params.region);
+ // `auto` is a real region for S3-compatible stores such as Cloudflare R2,
+ // but not for Amazon S3. There, sign for us-east-1 and let the SDK follow
+ // S3's redirect (`x-amz-bucket-region`) to the bucket's actual region.
+ const isAutoRegionOnAws =
+ region?.toLowerCase() === "auto" && !params.endpoint;
+ const clientRegion = isAutoRegionOnAws ? "us-east-1" : region;
// Create the main client for S3 operations using the internal endpoint
this.client = new S3Client({
credentials,
endpoint: params.endpoint,
- region,
+ region: clientRegion,
forcePathStyle: params.forcePathStyle,
+ followRegionRedirects: isAutoRegionOnAws,
// Restore pre-v3.729 default so CompleteMultipartUpload doesn't send a
// composite CRC32 header, which GCS's S3-compat layer rejects with 412.
requestChecksumCalculation: "WHEN_REQUIRED",