🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ingestion): return 503 with Retry-After when S3 throttles event uploads (#18398)
Niklas Semmlercc5be2cdf866
diff --git a/packages/shared/src/errors/ServiceUnavailableError.ts b/packages/shared/src/errors/ServiceUnavailableError.tsindex 299fdad07d..ec52808b7e 100644--- a/packages/shared/src/errors/ServiceUnavailableError.ts+++ b/packages/shared/src/errors/ServiceUnavailableError.ts@@ -1,7 +1,14 @@import { BaseError } from "./BaseError";export class ServiceUnavailableError extends BaseError {- constructor(description = "Service Temporarily Unavailable") {+ /** Sent as the `Retry-After` response header when set. */+ public readonly retryAfterSeconds?: number;++ constructor(+ description = "Service Temporarily Unavailable",+ options?: { retryAfterSeconds?: number },+ ) {super("ServiceUnavailableError", 503, description, true);+ this.retryAfterSeconds = options?.retryAfterSeconds;}}diff --git a/packages/shared/src/server/ingestion/processEventBatch.ts b/packages/shared/src/server/ingestion/processEventBatch.tsindex e5d81fb530..9e2c212517 100644--- a/packages/shared/src/server/ingestion/processEventBatch.ts+++ b/packages/shared/src/server/ingestion/processEventBatch.ts@@ -39,6 +39,7 @@ import {import { buildEventBucketPrefix } from "./eventBucketPath";import { isTraceIdInSample } from "./sampling";import {+ createS3ThrottledIngestionError,isS3SlowDownError,markProjectS3Slowdown,} from "../redis/s3SlowdownTracking";@@ -253,6 +254,8 @@ export const processEventBatch = async (* ASYNC PROCESSING *********************/let s3UploadErrored = false;+ let s3UploadThrottled = false;+ let s3UploadFailedOtherwise = false;await instrumentAsync({ name: "s3-upload-events" }, async () => {// S3 Event Upload is blocking, but non-failing.// If a promise rejects, we log it below, but do not throw an error.@@ -275,6 +278,7 @@ export const processEventBatch = async (// Check if this is a SlowDown error and mark the project for secondary queueif (isS3SlowDownError(result.reason)) {+ s3UploadThrottled = true;logger.warn("S3 SlowDown error during upload, marking project for secondary queue",{@@ -288,6 +292,7 @@ export const processEventBatch = async (reason: "s3_slowdown",});} else {+ s3UploadFailedOtherwise = true;markProjectIngestFailure(authCheck.scope.projectId!, {source: "process_event_batch",reason: "s3_upload_error",@@ -301,6 +306,10 @@ export const processEventBatch = async (});});+ if (s3UploadThrottled && !s3UploadFailedOtherwise) {+ throw createS3ThrottledIngestionError(source);+ }+// Send each event individually to IngestionQueue for ClickHouse processingif (s3UploadErrored) {throw new Error(diff --git a/packages/shared/src/server/redis/s3SlowdownTracking.ts b/packages/shared/src/server/redis/s3SlowdownTracking.tsindex 0f37f4c5be..c688ad62b5 100644--- a/packages/shared/src/server/redis/s3SlowdownTracking.ts+++ b/packages/shared/src/server/redis/s3SlowdownTracking.ts@@ -3,7 +3,10 @@ import { logger } from "../logger";import { traceException, recordIncrement } from "../instrumentation";import { env } from "../../env";-export { isS3SlowDownError } from "../services/s3ThrottleError";+export {+ createS3ThrottledIngestionError,+ isS3SlowDownError,+} from "../services/s3ThrottleError";const S3_SLOWDOWN_PREFIX = "langfuse:s3-slowdown";diff --git a/packages/shared/src/server/services/s3ThrottleError.ts b/packages/shared/src/server/services/s3ThrottleError.tsindex 2032b24319..63e22b7276 100644--- a/packages/shared/src/server/services/s3ThrottleError.ts+++ b/packages/shared/src/server/services/s3ThrottleError.ts@@ -1,4 +1,8 @@+import { ServiceUnavailableError } from "../../errors";+import { recordIncrement } from "../instrumentation";+const MAX_CAUSE_DEPTH = 8;+const S3_THROTTLE_RETRY_AFTER_SECONDS = 2;/*** S3 throttling (`SlowDown` / "reduce your request rate"). Storage helpers wrap@@ -31,3 +35,19 @@ function matchesSlowDown(err: object): boolean {}return false;}++/**+ * Rejects an ingestion request whose upload S3 throttled. A 503 with+ * `Retry-After` is retried by OTel exporters and SDKs, unlike a 500, and keeps+ * the rejection in 5xx-based error monitors (429 would read as a customer rate+ * limit).+ */+export function createS3ThrottledIngestionError(+ source: string,+): ServiceUnavailableError {+ recordIncrement("langfuse.ingestion.s3_throttle_rejections", 1, { source });+ return new ServiceUnavailableError(+ "Blob storage is throttling uploads, please retry",+ { retryAfterSeconds: S3_THROTTLE_RETRY_AFTER_SECONDS },+ );+}diff --git a/web/src/__tests__/server/withMiddlewares.servertest.ts b/web/src/__tests__/server/withMiddlewares.servertest.tsindex ba724a19de..5e0bd01163 100644--- a/web/src/__tests__/server/withMiddlewares.servertest.ts+++ b/web/src/__tests__/server/withMiddlewares.servertest.ts@@ -128,6 +128,24 @@ describe("withMiddlewares error handling", () => {// Should trace 5xx errorsexpect(traceException).toHaveBeenCalledWith(error);});+ it("should set Retry-After when a ServiceUnavailableError carries one", async () => {+ const handler = withMiddlewares({+ POST: async () => {+ throw new ServiceUnavailableError("Throttled", {+ retryAfterSeconds: 2,+ });+ },+ });++ const { req, res } = createMocks<NextApiRequest, NextApiResponse>({+ method: "POST",+ });++ await handler(req, res);++ expect(res._getStatusCode()).toBe(503);+ expect(res.getHeader("Retry-After")).toBe("2");+ });});describe("LangfuseNotFoundError handling", () => {diff --git a/web/src/features/public-api/server/withMiddlewares.ts b/web/src/features/public-api/server/withMiddlewares.tsindex b7a90b0490..704b0317b4 100644--- a/web/src/features/public-api/server/withMiddlewares.ts+++ b/web/src/features/public-api/server/withMiddlewares.ts@@ -23,6 +23,7 @@ import {type PublicApiErrorContract,} from "./structuredPublicApiErrorContract";import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";+import { setRetryAfterHeader } from "@/src/features/public-api/server/writeError";// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via typeofconst httpMethods = ["GET", "POST", "PUT", "DELETE", "PATCH"] as const;@@ -113,6 +114,8 @@ export function withMiddlewares(return await finalHandlers[method](req, res);} catch (error) {+ setRetryAfterHeader(res, error);+if (error instanceof ClickHouseResourceError) {const errorMessage =options?.clickHouseResourceErrorMessage ??diff --git a/web/src/features/public-api/server/writeError.ts b/web/src/features/public-api/server/writeError.tsindex 6f695f9b0c..40fb89f68f 100644--- a/web/src/features/public-api/server/writeError.ts+++ b/web/src/features/public-api/server/writeError.ts@@ -1,6 +1,6 @@import { type NextApiResponse } from "next";-import { type BaseError } from "@langfuse/shared";+import { type BaseError, ServiceUnavailableError } from "@langfuse/shared";/** ErrorOrgApiKeyRequired is the 403 message when a non-organization key hits an organization-scoped operation. */export const ErrorOrgApiKeyRequired =@@ -31,3 +31,16 @@ export function writeProjectError(message: error.httpCode === 403 ? ErrorOrgApiKeyRequired : error.message,});}++/** setRetryAfterHeader forwards a ServiceUnavailableError's retry hint as `Retry-After`. */+export function setRetryAfterHeader(+ res: NextApiResponse,+ error: unknown,+): void {+ if (+ error instanceof ServiceUnavailableError &&+ error.retryAfterSeconds !== undefined+ ) {+ res.setHeader("Retry-After", String(error.retryAfterSeconds));+ }+}diff --git a/web/src/pages/api/public/ingestion.ts b/web/src/pages/api/public/ingestion.tsindex 95355ed329..e2e0d0c097 100644--- a/web/src/pages/api/public/ingestion.ts+++ b/web/src/pages/api/public/ingestion.ts@@ -24,6 +24,7 @@ import {} from "@langfuse/shared";import { isPrismaException } from "@/src/utils/exceptions";import { RateLimitService } from "@/src/features/public-api/server/RateLimitService";+import { setRetryAfterHeader } from "@/src/features/public-api/server/writeError";import * as opentelemetry from "@opentelemetry/api";import { env } from "@/src/env.mjs";import {@@ -254,6 +255,7 @@ export default async function handler(}}+ setRetryAfterHeader(res, error);return res.status(error.httpCode).json({error: error.name,message: error.message,diff --git a/web/src/server/otel/processOtelIngestion.ts b/web/src/server/otel/processOtelIngestion.tsindex b97dccad45..ea94ea95d4 100644--- a/web/src/server/otel/processOtelIngestion.ts+++ b/web/src/server/otel/processOtelIngestion.ts@@ -1,6 +1,8 @@import { $root } from "@/src/pages/api/public/otel/otlp-proto/generated/root";import {+ createS3ThrottledIngestionError,getCurrentSpan,+ isS3SlowDownError,logger,markProjectIngestFailure,OtelIngestionProcessor,@@ -264,6 +266,13 @@ export async function processOtelIngestion(source: "public_otel_api",reason: "publish_failed",});+ if (isS3SlowDownError(error)) {+ logger.warn("S3 SlowDown error during OTel upload", {+ projectId: config.projectId,+ error,+ });+ throw createS3ThrottledIngestionError("otel");+ }throw error;}}diff --git a/worker/src/__tests__/processEventBatch.test.ts b/worker/src/__tests__/processEventBatch.test.tsindex e40acaeea7..12d5c56644 100644--- a/worker/src/__tests__/processEventBatch.test.ts+++ b/worker/src/__tests__/processEventBatch.test.ts@@ -1,6 +1,7 @@import { beforeEach, describe, it, expect, assert, vi } from "vitest";import { eventTypes } from "../../../packages/shared/src/server/ingestion/types";import { processEventBatch } from "../../../packages/shared/src/server/ingestion/processEventBatch";+import { ServiceUnavailableError } from "../../../packages/shared/src/errors";import {createUnknownSdkIngestionAttribution,UNKNOWN_INGESTION_SDK_VALUE,@@ -147,4 +148,65 @@ describe("processEventBatch", () => {ingestionSdkVersion: UNKNOWN_INGESTION_SDK_VALUE,});});+ it("rejects with a retryable 503 when S3 throttles the upload", async () => {+ const authCheck = {+ validKey: true as const,+ scope: {+ projectId: "project-id",+ accessLevel: "project" as const,+ publicKey: "pk-lf-public",+ },+ };+ uploadJsonMock.mockRejectedValue(+ new Error("Failed to upload JSON", {+ cause: Object.assign(new Error("Please reduce your request rate."), {+ name: "SlowDown",+ }),+ }),+ );++ const error = await processEventBatch(+ [createTraceCreateEvent()],+ authCheck,+ {+ delay: 0,+ attribution: createUnknownSdkIngestionAttribution({ authCheck }),+ },+ ).catch((e: unknown) => e);++ expect(error).toBeInstanceOf(ServiceUnavailableError);+ expect(error).toMatchObject({ httpCode: 503, retryAfterSeconds: 2 });+ expect(queueAddMock).not.toHaveBeenCalled();+ });++ it("does not report a retryable 503 when another upload failed for a different reason", async () => {+ const authCheck = {+ validKey: true as const,+ scope: {+ projectId: "project-id",+ accessLevel: "project" as const,+ publicKey: "pk-lf-public",+ },+ };+ const otherTrace = createTraceCreateEvent();+ otherTrace.id = "other-event-id";+ otherTrace.body.id = "other-trace-id";+ uploadJsonMock+ .mockRejectedValueOnce(+ Object.assign(new Error("throttled"), { name: "SlowDown" }),+ )+ .mockRejectedValueOnce(new Error("Access Denied"));++ const error = await processEventBatch(+ [createTraceCreateEvent(), otherTrace],+ authCheck,+ {+ delay: 0,+ attribution: createUnknownSdkIngestionAttribution({ authCheck }),+ },+ ).catch((e: unknown) => e);++ expect(error).toBeInstanceOf(Error);+ expect(error).not.toBeInstanceOf(ServiceUnavailableError);+ });});