🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): increase S3 upload retry attempts (#18614)
Tobias Wochinger69a84e0e955a
diff --git a/packages/shared/src/server/s3/index.ts b/packages/shared/src/server/s3/index.tsindex ceab3d1db7..0b393d70d7 100644--- a/packages/shared/src/server/s3/index.ts+++ b/packages/shared/src/server/s3/index.ts@@ -48,6 +48,7 @@ export const getS3MediaStorageClient = (bucketName: string): StorageService => {*/export function createEventUploadStorageService(bucketName: string,+ options?: { maxAttempts?: number },): StorageService {return StorageServiceFactory.getInstance({bucketName,@@ -59,6 +60,7 @@ export function createEventUploadStorageService(awsSse: env.LANGFUSE_S3_EVENT_UPLOAD_SSE,awsSseKmsKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_SSE_KMS_KEY_ID,logSlowDownAsWarning: true,+ maxAttempts: options?.maxAttempts,});}diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.tsindex 72f4d599c8..220f839596 100644--- a/packages/shared/src/server/services/StorageService.ts+++ b/packages/shared/src/server/services/StorageService.ts@@ -358,6 +358,7 @@ export class StorageServiceFactory {* @param params.awsSse - Server-side encryption method (e.g., "aws:kms")* @param params.awsSseKmsKeyId - SSE KMS Key ID when using KMS encryption* @param params.logSlowDownAsWarning - Log S3 SlowDown as a warning. Set only when the caller escalates to an error after its own retries are spent.+ * @param params.maxAttempts - Maximum AWS SDK attempts for each S3 operation.* @param params.connectionValidation - Optional connection-time DNS/IP validation for user-controlled endpoints.*/public static getInstance(params: {@@ -375,6 +376,7 @@ export class StorageServiceFactory {awsSse: string | undefined;awsSseKmsKeyId: string | undefined;logSlowDownAsWarning?: boolean;+ maxAttempts?: number;connectionValidation?: OutboundUrlConnectionValidationOptions;}): StorageService {if (@@ -781,6 +783,7 @@ class S3StorageService implements StorageService {awsSse: string | undefined;awsSseKmsKeyId: string | undefined;logSlowDownAsWarning?: boolean;+ maxAttempts?: number;connectionValidation?: OutboundUrlConnectionValidationOptions;}) {// Use accessKeyId and secretAccessKey if provided or fallback to default credentials@@ -817,6 +820,7 @@ class S3StorageService implements StorageService {requestChecksumCalculation: "WHEN_REQUIRED",responseChecksumValidation: "WHEN_REQUIRED",requestHandler,+ maxAttempts: params.maxAttempts,});addS3DiagnosticsMiddleware(this.client, {diff --git a/worker/src/features/evaluation/__tests__/s3StorageClient.test.ts b/worker/src/features/evaluation/__tests__/s3StorageClient.test.tsnew file mode 100644index 0000000000..c9bb3a10b4--- /dev/null+++ b/worker/src/features/evaluation/__tests__/s3StorageClient.test.ts@@ -0,0 +1,42 @@+import { expect, it, vi } from "vitest";++const { createEventUploadStorageService } = vi.hoisted(() => ({+ createEventUploadStorageService: vi+ .fn()+ .mockImplementation(+ (_bucketName: string, options?: { maxAttempts?: number }) => ({+ options,+ }),+ ),+}));++vi.mock("@langfuse/shared/src/server", () => ({+ createEventUploadStorageService,+}));++vi.mock("../../../env", () => ({+ env: {+ LANGFUSE_S3_EVENT_UPLOAD_BUCKET: "event-bucket",+ },+}));++import {+ getEvalS3StorageClient,+ getEvalScoreS3StorageClient,+} from "../s3StorageClient";++it("raises retries only for evaluator score uploads", () => {+ const observationClient = getEvalS3StorageClient();+ const scoreClient = getEvalScoreS3StorageClient();++ expect(observationClient).not.toBe(scoreClient);+ expect(createEventUploadStorageService).toHaveBeenNthCalledWith(+ 1,+ "event-bucket",+ );+ expect(createEventUploadStorageService).toHaveBeenNthCalledWith(+ 2,+ "event-bucket",+ { maxAttempts: 6 },+ );+});diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.tsindex b0dc745a45..6c20c134f6 100644--- a/worker/src/features/evaluation/evalExecutionDeps.ts+++ b/worker/src/features/evaluation/evalExecutionDeps.ts@@ -25,7 +25,7 @@ import {writeInternalTraceViaOtelIngestion,} from "@langfuse/shared/src/server";import { UnrecoverableError } from "../../errors/UnrecoverableError";-import { getEvalS3StorageClient } from "./s3StorageClient";+import { getEvalScoreS3StorageClient } from "./s3StorageClient";import { createInternalEventsWriter } from "../internal-tracing/createInternalEventsWriter";import { recordExportVolume } from "../../services/exportVolumeMetric";@@ -200,7 +200,7 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {});const bucketPath = `${bucketPrefix}${params.eventId}.json`;- await getEvalS3StorageClient().uploadJson(bucketPath, [+ await getEvalScoreS3StorageClient().uploadJson(bucketPath, [params.event as unknown as Record<string, unknown>,]);},diff --git a/worker/src/features/evaluation/s3StorageClient.ts b/worker/src/features/evaluation/s3StorageClient.tsindex 09ae6e12ed..20b642f6e0 100644--- a/worker/src/features/evaluation/s3StorageClient.ts+++ b/worker/src/features/evaluation/s3StorageClient.ts@@ -4,18 +4,19 @@ import {} from "@langfuse/shared/src/server";import { env } from "../../env";+const EVAL_S3_MAX_ATTEMPTS = 6;+/**- * Singleton S3 storage client for eval operations.- * Used by both eval execution (score upload) and observation eval scheduling (observation upload).+ * Singleton S3 storage client for observation eval scheduling and processing.+ * These operations keep the AWS SDK's configured retry budget because+ * scheduling runs inline with ingestion.*/let s3StorageServiceClient: StorageService | null = null;+let s3ScoreStorageServiceClient: StorageService | null = null;/**- * Gets the singleton S3 storage client for eval operations.+ * Gets the singleton S3 storage client for observation eval operations.* Creates the client on first call using environment configuration.- *- * @param bucketName - The S3 bucket name to use- * @returns The S3 storage service client*/export function getEvalS3StorageClient(): StorageService {if (!s3StorageServiceClient) {@@ -26,3 +27,18 @@ export function getEvalS3StorageClient(): StorageService {return s3StorageServiceClient;}++/**+ * Gets the singleton S3 client used for evaluator score uploads.+ * It raises the SDK attempt budget to six.+ */+export function getEvalScoreS3StorageClient(): StorageService {+ if (!s3ScoreStorageServiceClient) {+ s3ScoreStorageServiceClient = createEventUploadStorageService(+ env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,+ { maxAttempts: EVAL_S3_MAX_ATTEMPTS },+ );+ }++ return s3ScoreStorageServiceClient;+}