🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(media): time out S3 media requests (#18636)
Valery Meleshkindc02d2364031
diff --git a/.env.dev.example b/.env.dev.exampleindex 4f45290cbf..fc1bf5937a 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -157,6 +157,8 @@ LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090## Necessary for minio compatibilityLANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=trueLANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/+# Request timeout in milliseconds; defaults to 30000.+# LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS=30000# Opt in to extracting and uploading media embedded in OTEL span attributes.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED=false# Early extraction requires both the toggle and a project selection.diff --git a/.env.prod.example b/.env.prod.exampleindex 601cad8072..ee29393cdf 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -205,6 +205,9 @@ OTEL_SERVICE_NAME="langfuse"# with the default value first, then setting it to a desired limit.# LANGFUSE_S3_EVENT_KEY_MAX_SEGMENT_BYTES=2048#+# S3 media request timeout in milliseconds; defaults to 30000.+# LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS=30000+## Whether to use blob_storage_file_log table to manage blob storage events# Can be set to `false` if `event` entities are managed using lifecycle policies in the blob storage bucket.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=truediff --git a/docker-compose.yml b/docker-compose.ymlindex db4651ad88..090a4e2f7d 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -56,6 +56,7 @@ services:LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}+ LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS: ${LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS:-30000}LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED: ${LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED:-false}LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES: ${LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES:-2097152}LANGFUSE_S3_BATCH_EXPORT_ENABLED: ${LANGFUSE_S3_BATCH_EXPORT_ENABLED:-false}diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex 2412473f56..0664fddf5e 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -312,6 +312,11 @@ const EnvSchema = z.object({.default("false"),LANGFUSE_S3_MEDIA_UPLOAD_SSE: z.enum(["AES256", "aws:kms"]).optional(),LANGFUSE_S3_MEDIA_UPLOAD_SSE_KMS_KEY_ID: z.string().optional(),+ LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS: z.coerce+ .number()+ .int()+ .positive()+ .default(30_000),LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),LANGFUSE_AZURE_SKIP_CONTAINER_CHECK: z.enum(["true", "false"])diff --git a/packages/shared/src/server/s3/index.ts b/packages/shared/src/server/s3/index.tsindex 0b393d70d7..c50ccb14cb 100644--- a/packages/shared/src/server/s3/index.ts+++ b/packages/shared/src/server/s3/index.ts@@ -34,6 +34,7 @@ export const getS3MediaStorageClient = (bucketName: string): StorageService => {forcePathStyle: env.LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE === "true",awsSse: env.LANGFUSE_S3_MEDIA_UPLOAD_SSE,awsSseKmsKeyId: env.LANGFUSE_S3_MEDIA_UPLOAD_SSE_KMS_KEY_ID,+ s3RequestTimeoutMs: env.LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS,});}return s3MediaStorageClient;diff --git a/packages/shared/src/server/services/StorageService.test.ts b/packages/shared/src/server/services/StorageService.test.tsindex bbb4208e47..be10098d61 100644--- a/packages/shared/src/server/services/StorageService.test.ts+++ b/packages/shared/src/server/services/StorageService.test.ts@@ -1,9 +1,11 @@import { createHash, getFips } from "crypto";+import { createServer } from "http";import { PassThrough, Readable } from "stream";import { afterEach, describe, expect, it, vi } from "vitest";import type { S3Client } from "@aws-sdk/client-s3";+import type { NodeHttpHandler } from "@smithy/node-http-handler";import { BLOB_STORAGE_REGION_INVALID_MESSAGE } from "../../utils/stringChecks";import { env } from "../../env";@@ -29,6 +31,53 @@ vi.mock("@aws-sdk/lib-storage", () => ({}));describe("S3StorageService region normalization", () => {+ it("configures a failing request timeout when requested", async () => {+ const server = createServer(() => {});+ await new Promise<void>((resolve) =>+ server.listen(0, "127.0.0.1", resolve),+ );+ const address = server.address();+ if (!address || typeof address === "string") {+ throw new Error("Expected a local HTTP port");+ }++ try {+ const service = StorageServiceFactory.getInstance({+ accessKeyId: "test-access-key",+ secretAccessKey: "test-secret-key",+ bucketName: "test-bucket",+ endpoint: undefined,+ region: "us-west-2",+ forcePathStyle: false,+ useAzureBlob: false,+ useGoogleCloudStorage: false,+ useOCIObjectStorage: false,+ awsSse: undefined,+ awsSseKmsKeyId: undefined,+ s3RequestTimeoutMs: 50,+ });+ const client = (service as unknown as { client: S3Client }).client;+ const handler = client.config.requestHandler as NodeHttpHandler;+ await expect(+ handler.handle({+ protocol: "http:",+ hostname: "127.0.0.1",+ port: address.port,+ method: "GET",+ path: "/",+ } as Parameters<NodeHttpHandler["handle"]>[0]),+ ).rejects.toMatchObject({ name: "TimeoutError" });++ expect(handler.httpHandlerConfigs()).toMatchObject({+ requestTimeout: 50,+ throwOnRequestTimeout: true,+ });+ } finally {+ server.closeAllConnections();+ server.close();+ }+ });+it("trims a persisted region before configuring the AWS client", async () => {const service = StorageServiceFactory.getInstance({accessKeyId: "test-access-key",diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.tsindex 220f839596..ddcb2d90ab 100644--- a/packages/shared/src/server/services/StorageService.ts+++ b/packages/shared/src/server/services/StorageService.ts@@ -129,13 +129,19 @@ function handleStorageError(err: unknown, operation: string): never {function createS3RequestHandler(connectionValidation?: OutboundUrlConnectionValidationOptions,+ s3RequestTimeoutMs?: number,): NodeHttpHandler {const maxSockets = env.LANGFUSE_S3_CONCURRENT_WRITES;+ const timeout =+ s3RequestTimeoutMs === undefined+ ? {}+ : { requestTimeout: s3RequestTimeoutMs, throwOnRequestTimeout: true };if (!connectionValidation) {// Both agents need the limit: plain-HTTP endpoints (e.g. MinIO) go through// httpAgent, which otherwise stays at the SDK default of 50 sockets.return new NodeHttpHandler({+ ...timeout,httpAgent: { maxSockets },httpsAgent: { maxSockets },});@@ -147,6 +153,7 @@ function createS3RequestHandler();return new NodeHttpHandler({+ ...timeout,httpAgent,httpsAgent,});@@ -360,6 +367,7 @@ export class StorageServiceFactory {* @param params.logSlowDownAsWarning - Log S3 SlowDown as a warning. Set only when the caller escalates to an error after its own retries are spent.* @param params.maxAttempts - Maximum AWS SDK attempts for each S3 operation.* @param params.connectionValidation - Optional connection-time DNS/IP validation for user-controlled endpoints.+ * @param params.s3RequestTimeoutMs - Optional S3 HTTP request deadline; timeout rejects the request.*/public static getInstance(params: {accessKeyId: string | undefined;@@ -378,6 +386,7 @@ export class StorageServiceFactory {logSlowDownAsWarning?: boolean;maxAttempts?: number;connectionValidation?: OutboundUrlConnectionValidationOptions;+ s3RequestTimeoutMs?: number;}): StorageService {if (params.useAzureBlob !== undefined@@ -785,6 +794,7 @@ class S3StorageService implements StorageService {logSlowDownAsWarning?: boolean;maxAttempts?: number;connectionValidation?: OutboundUrlConnectionValidationOptions;+ s3RequestTimeoutMs?: number;}) {// Use accessKeyId and secretAccessKey if provided or fallback to default credentialsconst { accessKeyId, secretAccessKey } = params;@@ -796,7 +806,10 @@ class S3StorageService implements StorageService {}: undefined;- const requestHandler = createS3RequestHandler(params.connectionValidation);+ const requestHandler = createS3RequestHandler(+ params.connectionValidation,+ params.s3RequestTimeoutMs,+ );const region =params.region === undefined? undefined