Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(media): time out S3 media requests (#18636)

Valery Meleshkindc02d2364031
diff --git a/.env.dev.example b/.env.dev.example
index 4f45290cbf..fc1bf5937a 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -157,6 +157,8 @@ LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
## Necessary for minio compatibility
LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+# Request timeout in milliseconds; defaults to 30000.
+# LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS=30000
# Opt in to extracting and uploading media embedded in OTEL span attributes.
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED=false
# Early extraction requires both the toggle and a project selection.
diff --git a/.env.prod.example b/.env.prod.example
index 601cad8072..ee29393cdf 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -205,6 +205,9 @@ OTEL_SERVICE_NAME="langfuse"
# with the default value first, then setting it to a desired limit.
# LANGFUSE_S3_EVENT_KEY_MAX_SEGMENT_BYTES=2048
#
+# S3 media request timeout in milliseconds; defaults to 30000.
+# LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS=30000
+#
# Whether to use blob_storage_file_log table to manage blob storage events
# Can be set to `false` if `event` entities are managed using lifecycle policies in the blob storage bucket.
LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
diff --git a/docker-compose.yml b/docker-compose.yml
index db4651ad88..090a4e2f7d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,6 +56,7 @@ services:
LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+ LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS: ${LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS:-30000}
LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED: ${LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED:-false}
LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES: ${LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES:-2097152}
LANGFUSE_S3_BATCH_EXPORT_ENABLED: ${LANGFUSE_S3_BATCH_EXPORT_ENABLED:-false}
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index 2412473f56..0664fddf5e 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -312,6 +312,11 @@ const EnvSchema = z.object({
.default("false"),
LANGFUSE_S3_MEDIA_UPLOAD_SSE: z.enum(["AES256", "aws:kms"]).optional(),
LANGFUSE_S3_MEDIA_UPLOAD_SSE_KMS_KEY_ID: z.string().optional(),
+ LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(30_000),
LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
LANGFUSE_AZURE_SKIP_CONTAINER_CHECK: z
.enum(["true", "false"])
diff --git a/packages/shared/src/server/s3/index.ts b/packages/shared/src/server/s3/index.ts
index 0b393d70d7..c50ccb14cb 100644
--- a/packages/shared/src/server/s3/index.ts
+++ b/packages/shared/src/server/s3/index.ts
@@ -34,6 +34,7 @@ export const getS3MediaStorageClient = (bucketName: string): StorageService => {
forcePathStyle: env.LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE === "true",
awsSse: env.LANGFUSE_S3_MEDIA_UPLOAD_SSE,
awsSseKmsKeyId: env.LANGFUSE_S3_MEDIA_UPLOAD_SSE_KMS_KEY_ID,
+ s3RequestTimeoutMs: env.LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS,
});
}
return s3MediaStorageClient;
diff --git a/packages/shared/src/server/services/StorageService.test.ts b/packages/shared/src/server/services/StorageService.test.ts
index bbb4208e47..be10098d61 100644
--- a/packages/shared/src/server/services/StorageService.test.ts
+++ b/packages/shared/src/server/services/StorageService.test.ts
@@ -1,9 +1,11 @@
import { createHash, getFips } from "crypto";
+import { createServer } from "http";
import { PassThrough, Readable } from "stream";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { S3Client } from "@aws-sdk/client-s3";
+import type { NodeHttpHandler } from "@smithy/node-http-handler";
import { BLOB_STORAGE_REGION_INVALID_MESSAGE } from "../../utils/stringChecks";
import { env } from "../../env";
@@ -29,6 +31,53 @@ vi.mock("@aws-sdk/lib-storage", () => ({
}));
describe("S3StorageService region normalization", () => {
+ it("configures a failing request timeout when requested", async () => {
+ const server = createServer(() => {});
+ await new Promise<void>((resolve) =>
+ server.listen(0, "127.0.0.1", resolve),
+ );
+ const address = server.address();
+ if (!address || typeof address === "string") {
+ throw new Error("Expected a local HTTP port");
+ }
+
+ try {
+ const service = StorageServiceFactory.getInstance({
+ accessKeyId: "test-access-key",
+ secretAccessKey: "test-secret-key",
+ bucketName: "test-bucket",
+ endpoint: undefined,
+ region: "us-west-2",
+ forcePathStyle: false,
+ useAzureBlob: false,
+ useGoogleCloudStorage: false,
+ useOCIObjectStorage: false,
+ awsSse: undefined,
+ awsSseKmsKeyId: undefined,
+ s3RequestTimeoutMs: 50,
+ });
+ const client = (service as unknown as { client: S3Client }).client;
+ const handler = client.config.requestHandler as NodeHttpHandler;
+ await expect(
+ handler.handle({
+ protocol: "http:",
+ hostname: "127.0.0.1",
+ port: address.port,
+ method: "GET",
+ path: "/",
+ } as Parameters<NodeHttpHandler["handle"]>[0]),
+ ).rejects.toMatchObject({ name: "TimeoutError" });
+
+ expect(handler.httpHandlerConfigs()).toMatchObject({
+ requestTimeout: 50,
+ throwOnRequestTimeout: true,
+ });
+ } finally {
+ server.closeAllConnections();
+ server.close();
+ }
+ });
+
it("trims a persisted region before configuring the AWS client", async () => {
const service = StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.ts
index 220f839596..ddcb2d90ab 100644
--- a/packages/shared/src/server/services/StorageService.ts
+++ b/packages/shared/src/server/services/StorageService.ts
@@ -129,13 +129,19 @@ function handleStorageError(err: unknown, operation: string): never {
function createS3RequestHandler(
connectionValidation?: OutboundUrlConnectionValidationOptions,
+ s3RequestTimeoutMs?: number,
): NodeHttpHandler {
const maxSockets = env.LANGFUSE_S3_CONCURRENT_WRITES;
+ const timeout =
+ s3RequestTimeoutMs === undefined
+ ? {}
+ : { requestTimeout: s3RequestTimeoutMs, throwOnRequestTimeout: true };
if (!connectionValidation) {
// Both agents need the limit: plain-HTTP endpoints (e.g. MinIO) go through
// httpAgent, which otherwise stays at the SDK default of 50 sockets.
return new NodeHttpHandler({
+ ...timeout,
httpAgent: { maxSockets },
httpsAgent: { maxSockets },
});
@@ -147,6 +153,7 @@ function createS3RequestHandler(
);
return new NodeHttpHandler({
+ ...timeout,
httpAgent,
httpsAgent,
});
@@ -360,6 +367,7 @@ export class StorageServiceFactory {
* @param params.logSlowDownAsWarning - Log S3 SlowDown as a warning. Set only when the caller escalates to an error after its own retries are spent.
* @param params.maxAttempts - Maximum AWS SDK attempts for each S3 operation.
* @param params.connectionValidation - Optional connection-time DNS/IP validation for user-controlled endpoints.
+ * @param params.s3RequestTimeoutMs - Optional S3 HTTP request deadline; timeout rejects the request.
*/
public static getInstance(params: {
accessKeyId: string | undefined;
@@ -378,6 +386,7 @@ export class StorageServiceFactory {
logSlowDownAsWarning?: boolean;
maxAttempts?: number;
connectionValidation?: OutboundUrlConnectionValidationOptions;
+ s3RequestTimeoutMs?: number;
}): StorageService {
if (
params.useAzureBlob !== undefined
@@ -785,6 +794,7 @@ class S3StorageService implements StorageService {
logSlowDownAsWarning?: boolean;
maxAttempts?: number;
connectionValidation?: OutboundUrlConnectionValidationOptions;
+ s3RequestTimeoutMs?: number;
}) {
// Use accessKeyId and secretAccessKey if provided or fallback to default credentials
const { accessKeyId, secretAccessKey } = params;
@@ -796,7 +806,10 @@ class S3StorageService implements StorageService {
}
: undefined;
- const requestHandler = createS3RequestHandler(params.connectionValidation);
+ const requestHandler = createS3RequestHandler(
+ params.connectionValidation,
+ params.s3RequestTimeoutMs,
+ );
const region =
params.region === undefined
? undefined