Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ingestion): return 503 with Retry-After when S3 throttles event uploads (#18398)

Niklas Semmlercc5be2cdf866
diff --git a/packages/shared/src/errors/ServiceUnavailableError.ts b/packages/shared/src/errors/ServiceUnavailableError.ts
index 299fdad07d..ec52808b7e 100644
--- a/packages/shared/src/errors/ServiceUnavailableError.ts
+++ b/packages/shared/src/errors/ServiceUnavailableError.ts
@@ -1,7 +1,14 @@
import { BaseError } from "./BaseError";
export class ServiceUnavailableError extends BaseError {
- constructor(description = "Service Temporarily Unavailable") {
+ /** Sent as the `Retry-After` response header when set. */
+ public readonly retryAfterSeconds?: number;
+
+ constructor(
+ description = "Service Temporarily Unavailable",
+ options?: { retryAfterSeconds?: number },
+ ) {
super("ServiceUnavailableError", 503, description, true);
+ this.retryAfterSeconds = options?.retryAfterSeconds;
}
}
diff --git a/packages/shared/src/server/ingestion/processEventBatch.ts b/packages/shared/src/server/ingestion/processEventBatch.ts
index e5d81fb530..9e2c212517 100644
--- a/packages/shared/src/server/ingestion/processEventBatch.ts
+++ b/packages/shared/src/server/ingestion/processEventBatch.ts
@@ -39,6 +39,7 @@ import {
import { buildEventBucketPrefix } from "./eventBucketPath";
import { isTraceIdInSample } from "./sampling";
import {
+ createS3ThrottledIngestionError,
isS3SlowDownError,
markProjectS3Slowdown,
} from "../redis/s3SlowdownTracking";
@@ -253,6 +254,8 @@ export const processEventBatch = async (
* ASYNC PROCESSING *
********************/
let s3UploadErrored = false;
+ let s3UploadThrottled = false;
+ let s3UploadFailedOtherwise = false;
await instrumentAsync({ name: "s3-upload-events" }, async () => {
// S3 Event Upload is blocking, but non-failing.
// If a promise rejects, we log it below, but do not throw an error.
@@ -275,6 +278,7 @@ export const processEventBatch = async (
// Check if this is a SlowDown error and mark the project for secondary queue
if (isS3SlowDownError(result.reason)) {
+ s3UploadThrottled = true;
logger.warn(
"S3 SlowDown error during upload, marking project for secondary queue",
{
@@ -288,6 +292,7 @@ export const processEventBatch = async (
reason: "s3_slowdown",
});
} else {
+ s3UploadFailedOtherwise = true;
markProjectIngestFailure(authCheck.scope.projectId!, {
source: "process_event_batch",
reason: "s3_upload_error",
@@ -301,6 +306,10 @@ export const processEventBatch = async (
});
});
+ if (s3UploadThrottled && !s3UploadFailedOtherwise) {
+ throw createS3ThrottledIngestionError(source);
+ }
+
// Send each event individually to IngestionQueue for ClickHouse processing
if (s3UploadErrored) {
throw new Error(
diff --git a/packages/shared/src/server/redis/s3SlowdownTracking.ts b/packages/shared/src/server/redis/s3SlowdownTracking.ts
index 0f37f4c5be..c688ad62b5 100644
--- a/packages/shared/src/server/redis/s3SlowdownTracking.ts
+++ b/packages/shared/src/server/redis/s3SlowdownTracking.ts
@@ -3,7 +3,10 @@ import { logger } from "../logger";
import { traceException, recordIncrement } from "../instrumentation";
import { env } from "../../env";
-export { isS3SlowDownError } from "../services/s3ThrottleError";
+export {
+ createS3ThrottledIngestionError,
+ isS3SlowDownError,
+} from "../services/s3ThrottleError";
const S3_SLOWDOWN_PREFIX = "langfuse:s3-slowdown";
diff --git a/packages/shared/src/server/services/s3ThrottleError.ts b/packages/shared/src/server/services/s3ThrottleError.ts
index 2032b24319..63e22b7276 100644
--- a/packages/shared/src/server/services/s3ThrottleError.ts
+++ b/packages/shared/src/server/services/s3ThrottleError.ts
@@ -1,4 +1,8 @@
+import { ServiceUnavailableError } from "../../errors";
+import { recordIncrement } from "../instrumentation";
+
const MAX_CAUSE_DEPTH = 8;
+const S3_THROTTLE_RETRY_AFTER_SECONDS = 2;
/**
* S3 throttling (`SlowDown` / "reduce your request rate"). Storage helpers wrap
@@ -31,3 +35,19 @@ function matchesSlowDown(err: object): boolean {
}
return false;
}
+
+/**
+ * Rejects an ingestion request whose upload S3 throttled. A 503 with
+ * `Retry-After` is retried by OTel exporters and SDKs, unlike a 500, and keeps
+ * the rejection in 5xx-based error monitors (429 would read as a customer rate
+ * limit).
+ */
+export function createS3ThrottledIngestionError(
+ source: string,
+): ServiceUnavailableError {
+ recordIncrement("langfuse.ingestion.s3_throttle_rejections", 1, { source });
+ return new ServiceUnavailableError(
+ "Blob storage is throttling uploads, please retry",
+ { retryAfterSeconds: S3_THROTTLE_RETRY_AFTER_SECONDS },
+ );
+}
diff --git a/web/src/__tests__/server/withMiddlewares.servertest.ts b/web/src/__tests__/server/withMiddlewares.servertest.ts
index ba724a19de..5e0bd01163 100644
--- a/web/src/__tests__/server/withMiddlewares.servertest.ts
+++ b/web/src/__tests__/server/withMiddlewares.servertest.ts
@@ -128,6 +128,24 @@ describe("withMiddlewares error handling", () => {
// Should trace 5xx errors
expect(traceException).toHaveBeenCalledWith(error);
});
+ it("should set Retry-After when a ServiceUnavailableError carries one", async () => {
+ const handler = withMiddlewares({
+ POST: async () => {
+ throw new ServiceUnavailableError("Throttled", {
+ retryAfterSeconds: 2,
+ });
+ },
+ });
+
+ const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
+ method: "POST",
+ });
+
+ await handler(req, res);
+
+ expect(res._getStatusCode()).toBe(503);
+ expect(res.getHeader("Retry-After")).toBe("2");
+ });
});
describe("LangfuseNotFoundError handling", () => {
diff --git a/web/src/features/public-api/server/withMiddlewares.ts b/web/src/features/public-api/server/withMiddlewares.ts
index b7a90b0490..704b0317b4 100644
--- a/web/src/features/public-api/server/withMiddlewares.ts
+++ b/web/src/features/public-api/server/withMiddlewares.ts
@@ -23,6 +23,7 @@ import {
type PublicApiErrorContract,
} from "./structuredPublicApiErrorContract";
import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";
+import { setRetryAfterHeader } from "@/src/features/public-api/server/writeError";
// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via typeof
const httpMethods = ["GET", "POST", "PUT", "DELETE", "PATCH"] as const;
@@ -113,6 +114,8 @@ export function withMiddlewares(
return await finalHandlers[method](req, res);
} catch (error) {
+ setRetryAfterHeader(res, error);
+
if (error instanceof ClickHouseResourceError) {
const errorMessage =
options?.clickHouseResourceErrorMessage ??
diff --git a/web/src/features/public-api/server/writeError.ts b/web/src/features/public-api/server/writeError.ts
index 6f695f9b0c..40fb89f68f 100644
--- a/web/src/features/public-api/server/writeError.ts
+++ b/web/src/features/public-api/server/writeError.ts
@@ -1,6 +1,6 @@
import { type NextApiResponse } from "next";
-import { type BaseError } from "@langfuse/shared";
+import { type BaseError, ServiceUnavailableError } from "@langfuse/shared";
/** ErrorOrgApiKeyRequired is the 403 message when a non-organization key hits an organization-scoped operation. */
export const ErrorOrgApiKeyRequired =
@@ -31,3 +31,16 @@ export function writeProjectError(
message: error.httpCode === 403 ? ErrorOrgApiKeyRequired : error.message,
});
}
+
+/** setRetryAfterHeader forwards a ServiceUnavailableError's retry hint as `Retry-After`. */
+export function setRetryAfterHeader(
+ res: NextApiResponse,
+ error: unknown,
+): void {
+ if (
+ error instanceof ServiceUnavailableError &&
+ error.retryAfterSeconds !== undefined
+ ) {
+ res.setHeader("Retry-After", String(error.retryAfterSeconds));
+ }
+}
diff --git a/web/src/pages/api/public/ingestion.ts b/web/src/pages/api/public/ingestion.ts
index 95355ed329..e2e0d0c097 100644
--- a/web/src/pages/api/public/ingestion.ts
+++ b/web/src/pages/api/public/ingestion.ts
@@ -24,6 +24,7 @@ import {
} from "@langfuse/shared";
import { isPrismaException } from "@/src/utils/exceptions";
import { RateLimitService } from "@/src/features/public-api/server/RateLimitService";
+import { setRetryAfterHeader } from "@/src/features/public-api/server/writeError";
import * as opentelemetry from "@opentelemetry/api";
import { env } from "@/src/env.mjs";
import {
@@ -254,6 +255,7 @@ export default async function handler(
}
}
+ setRetryAfterHeader(res, error);
return res.status(error.httpCode).json({
error: error.name,
message: error.message,
diff --git a/web/src/server/otel/processOtelIngestion.ts b/web/src/server/otel/processOtelIngestion.ts
index b97dccad45..ea94ea95d4 100644
--- a/web/src/server/otel/processOtelIngestion.ts
+++ b/web/src/server/otel/processOtelIngestion.ts
@@ -1,6 +1,8 @@
import { $root } from "@/src/pages/api/public/otel/otlp-proto/generated/root";
import {
+ createS3ThrottledIngestionError,
getCurrentSpan,
+ isS3SlowDownError,
logger,
markProjectIngestFailure,
OtelIngestionProcessor,
@@ -264,6 +266,13 @@ export async function processOtelIngestion(
source: "public_otel_api",
reason: "publish_failed",
});
+ if (isS3SlowDownError(error)) {
+ logger.warn("S3 SlowDown error during OTel upload", {
+ projectId: config.projectId,
+ error,
+ });
+ throw createS3ThrottledIngestionError("otel");
+ }
throw error;
}
}
diff --git a/worker/src/__tests__/processEventBatch.test.ts b/worker/src/__tests__/processEventBatch.test.ts
index e40acaeea7..12d5c56644 100644
--- a/worker/src/__tests__/processEventBatch.test.ts
+++ b/worker/src/__tests__/processEventBatch.test.ts
@@ -1,6 +1,7 @@
import { beforeEach, describe, it, expect, assert, vi } from "vitest";
import { eventTypes } from "../../../packages/shared/src/server/ingestion/types";
import { processEventBatch } from "../../../packages/shared/src/server/ingestion/processEventBatch";
+import { ServiceUnavailableError } from "../../../packages/shared/src/errors";
import {
createUnknownSdkIngestionAttribution,
UNKNOWN_INGESTION_SDK_VALUE,
@@ -147,4 +148,65 @@ describe("processEventBatch", () => {
ingestionSdkVersion: UNKNOWN_INGESTION_SDK_VALUE,
});
});
+ it("rejects with a retryable 503 when S3 throttles the upload", async () => {
+ const authCheck = {
+ validKey: true as const,
+ scope: {
+ projectId: "project-id",
+ accessLevel: "project" as const,
+ publicKey: "pk-lf-public",
+ },
+ };
+ uploadJsonMock.mockRejectedValue(
+ new Error("Failed to upload JSON", {
+ cause: Object.assign(new Error("Please reduce your request rate."), {
+ name: "SlowDown",
+ }),
+ }),
+ );
+
+ const error = await processEventBatch(
+ [createTraceCreateEvent()],
+ authCheck,
+ {
+ delay: 0,
+ attribution: createUnknownSdkIngestionAttribution({ authCheck }),
+ },
+ ).catch((e: unknown) => e);
+
+ expect(error).toBeInstanceOf(ServiceUnavailableError);
+ expect(error).toMatchObject({ httpCode: 503, retryAfterSeconds: 2 });
+ expect(queueAddMock).not.toHaveBeenCalled();
+ });
+
+ it("does not report a retryable 503 when another upload failed for a different reason", async () => {
+ const authCheck = {
+ validKey: true as const,
+ scope: {
+ projectId: "project-id",
+ accessLevel: "project" as const,
+ publicKey: "pk-lf-public",
+ },
+ };
+ const otherTrace = createTraceCreateEvent();
+ otherTrace.id = "other-event-id";
+ otherTrace.body.id = "other-trace-id";
+ uploadJsonMock
+ .mockRejectedValueOnce(
+ Object.assign(new Error("throttled"), { name: "SlowDown" }),
+ )
+ .mockRejectedValueOnce(new Error("Access Denied"));
+
+ const error = await processEventBatch(
+ [createTraceCreateEvent(), otherTrace],
+ authCheck,
+ {
+ delay: 0,
+ attribution: createUnknownSdkIngestionAttribution({ authCheck }),
+ },
+ ).catch((e: unknown) => e);
+
+ expect(error).toBeInstanceOf(Error);
+ expect(error).not.toBeInstanceOf(ServiceUnavailableError);
+ });
});