Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): increase S3 upload retry attempts (#18614)

Tobias Wochinger69a84e0e955a
diff --git a/packages/shared/src/server/s3/index.ts b/packages/shared/src/server/s3/index.ts
index ceab3d1db7..0b393d70d7 100644
--- a/packages/shared/src/server/s3/index.ts
+++ b/packages/shared/src/server/s3/index.ts
@@ -48,6 +48,7 @@ export const getS3MediaStorageClient = (bucketName: string): StorageService => {
*/
export function createEventUploadStorageService(
bucketName: string,
+ options?: { maxAttempts?: number },
): StorageService {
return StorageServiceFactory.getInstance({
bucketName,
@@ -59,6 +60,7 @@ export function createEventUploadStorageService(
awsSse: env.LANGFUSE_S3_EVENT_UPLOAD_SSE,
awsSseKmsKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_SSE_KMS_KEY_ID,
logSlowDownAsWarning: true,
+ maxAttempts: options?.maxAttempts,
});
}
diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.ts
index 72f4d599c8..220f839596 100644
--- a/packages/shared/src/server/services/StorageService.ts
+++ b/packages/shared/src/server/services/StorageService.ts
@@ -358,6 +358,7 @@ export class StorageServiceFactory {
* @param params.awsSse - Server-side encryption method (e.g., "aws:kms")
* @param params.awsSseKmsKeyId - SSE KMS Key ID when using KMS encryption
* @param params.logSlowDownAsWarning - Log S3 SlowDown as a warning. Set only when the caller escalates to an error after its own retries are spent.
+ * @param params.maxAttempts - Maximum AWS SDK attempts for each S3 operation.
* @param params.connectionValidation - Optional connection-time DNS/IP validation for user-controlled endpoints.
*/
public static getInstance(params: {
@@ -375,6 +376,7 @@ export class StorageServiceFactory {
awsSse: string | undefined;
awsSseKmsKeyId: string | undefined;
logSlowDownAsWarning?: boolean;
+ maxAttempts?: number;
connectionValidation?: OutboundUrlConnectionValidationOptions;
}): StorageService {
if (
@@ -781,6 +783,7 @@ class S3StorageService implements StorageService {
awsSse: string | undefined;
awsSseKmsKeyId: string | undefined;
logSlowDownAsWarning?: boolean;
+ maxAttempts?: number;
connectionValidation?: OutboundUrlConnectionValidationOptions;
}) {
// Use accessKeyId and secretAccessKey if provided or fallback to default credentials
@@ -817,6 +820,7 @@ class S3StorageService implements StorageService {
requestChecksumCalculation: "WHEN_REQUIRED",
responseChecksumValidation: "WHEN_REQUIRED",
requestHandler,
+ maxAttempts: params.maxAttempts,
});
addS3DiagnosticsMiddleware(this.client, {
diff --git a/worker/src/features/evaluation/__tests__/s3StorageClient.test.ts b/worker/src/features/evaluation/__tests__/s3StorageClient.test.ts
new file mode 100644
index 0000000000..c9bb3a10b4
--- /dev/null
+++ b/worker/src/features/evaluation/__tests__/s3StorageClient.test.ts
@@ -0,0 +1,42 @@
+import { expect, it, vi } from "vitest";
+
+const { createEventUploadStorageService } = vi.hoisted(() => ({
+ createEventUploadStorageService: vi
+ .fn()
+ .mockImplementation(
+ (_bucketName: string, options?: { maxAttempts?: number }) => ({
+ options,
+ }),
+ ),
+}));
+
+vi.mock("@langfuse/shared/src/server", () => ({
+ createEventUploadStorageService,
+}));
+
+vi.mock("../../../env", () => ({
+ env: {
+ LANGFUSE_S3_EVENT_UPLOAD_BUCKET: "event-bucket",
+ },
+}));
+
+import {
+ getEvalS3StorageClient,
+ getEvalScoreS3StorageClient,
+} from "../s3StorageClient";
+
+it("raises retries only for evaluator score uploads", () => {
+ const observationClient = getEvalS3StorageClient();
+ const scoreClient = getEvalScoreS3StorageClient();
+
+ expect(observationClient).not.toBe(scoreClient);
+ expect(createEventUploadStorageService).toHaveBeenNthCalledWith(
+ 1,
+ "event-bucket",
+ );
+ expect(createEventUploadStorageService).toHaveBeenNthCalledWith(
+ 2,
+ "event-bucket",
+ { maxAttempts: 6 },
+ );
+});
diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.ts
index b0dc745a45..6c20c134f6 100644
--- a/worker/src/features/evaluation/evalExecutionDeps.ts
+++ b/worker/src/features/evaluation/evalExecutionDeps.ts
@@ -25,7 +25,7 @@ import {
writeInternalTraceViaOtelIngestion,
} from "@langfuse/shared/src/server";
import { UnrecoverableError } from "../../errors/UnrecoverableError";
-import { getEvalS3StorageClient } from "./s3StorageClient";
+import { getEvalScoreS3StorageClient } from "./s3StorageClient";
import { createInternalEventsWriter } from "../internal-tracing/createInternalEventsWriter";
import { recordExportVolume } from "../../services/exportVolumeMetric";
@@ -200,7 +200,7 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
});
const bucketPath = `${bucketPrefix}${params.eventId}.json`;
- await getEvalS3StorageClient().uploadJson(bucketPath, [
+ await getEvalScoreS3StorageClient().uploadJson(bucketPath, [
params.event as unknown as Record<string, unknown>,
]);
},
diff --git a/worker/src/features/evaluation/s3StorageClient.ts b/worker/src/features/evaluation/s3StorageClient.ts
index 09ae6e12ed..20b642f6e0 100644
--- a/worker/src/features/evaluation/s3StorageClient.ts
+++ b/worker/src/features/evaluation/s3StorageClient.ts
@@ -4,18 +4,19 @@ import {
} from "@langfuse/shared/src/server";
import { env } from "../../env";
+const EVAL_S3_MAX_ATTEMPTS = 6;
+
/**
- * Singleton S3 storage client for eval operations.
- * Used by both eval execution (score upload) and observation eval scheduling (observation upload).
+ * Singleton S3 storage client for observation eval scheduling and processing.
+ * These operations keep the AWS SDK's configured retry budget because
+ * scheduling runs inline with ingestion.
*/
let s3StorageServiceClient: StorageService | null = null;
+let s3ScoreStorageServiceClient: StorageService | null = null;
/**
- * Gets the singleton S3 storage client for eval operations.
+ * Gets the singleton S3 storage client for observation eval operations.
* Creates the client on first call using environment configuration.
- *
- * @param bucketName - The S3 bucket name to use
- * @returns The S3 storage service client
*/
export function getEvalS3StorageClient(): StorageService {
if (!s3StorageServiceClient) {
@@ -26,3 +27,18 @@ export function getEvalS3StorageClient(): StorageService {
return s3StorageServiceClient;
}
+
+/**
+ * Gets the singleton S3 client used for evaluator score uploads.
+ * It raises the SDK attempt budget to six.
+ */
+export function getEvalScoreS3StorageClient(): StorageService {
+ if (!s3ScoreStorageServiceClient) {
+ s3ScoreStorageServiceClient = createEventUploadStorageService(
+ env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+ { maxAttempts: EVAL_S3_MAX_ATTEMPTS },
+ );
+ }
+
+ return s3ScoreStorageServiceClient;
+}